线性回归、岭回归、Lasso回归
前言
本章主要介绍线性回归、岭回归、Lasso回归,tensorflow实现的线性回归的简单例子代码。
原理篇看这里
[机器学习-原理篇]学习之线性回归、岭回归、Lasso回归
1.线性回归
from sklearn import linear_model
def test_linearRegression(X, y):
clf = linear_model.LinearRegression()
clf.fit(X, y)
print('linearRegression coef:',clf.coef_)
print('linearRegression intercept:',clf.intercept_)
if __name__ == '__main__':
X = [[0, 0], [1, 1], [2, 2.]]
y = [[0], [1], [2.]]
test_linearRegression(X, y)
linearRegression coef: [[0.5 0.5]]
linearRegression intercept: [1.11022302e-16]
2. 岭回归
from sklearn import linear_model
def test_ridge(X, y):
clf = linear_model.Ridge(alpha=.1)
clf.fit(X, y)
print('ridge coef:',clf.coef_)
print('ridge intercept',clf.intercept_)
if __name__ == '__main__':
X = [[0, 0], [1, 1], [2, 2.]]
y = [[0], [1], [2.]]
test_ridge(X, y)
ridge coef: [[0.48780488 0.48780488]]
ridge intercept [0.02439024]
3. Lasso回归
from sklearn import linear_model
def test_lasso(X, y):
clf = linear_model.Lasso(alpha=0.1)
clf.fit(X, y)
print('lasso coef:',clf.coef_)
print('lasso intercept: ',clf.intercept_)
if __name__ == '__main__':
X = [[0, 0], [1, 1], [2, 2.]]
y = [[0], [1], [2.]]
test_lasso(X, y)
从这个例子看出,第二特征的权重直接是0, 由此可以进一步得出结论
lasso 可以用来做 feature selection,而 ridge 不行。或者说,lasso 更容易使得权重变为 0,而 ridge 更容易使得权重接近 0。
lasso coef: [0.85 0. ]
lasso intercept: [0.15]
4. tensorflow利用梯度下降实现的线性回归
import tensorflow as tf
TRAIN_STEPS = 10
def test_tensorflow1(X, y):
w = tf.Variable(initial_value=[[1.0],[1.0]])
#w2 = tf.Variable(initial_value=1.0)
b = tf.Variable(initial_value=0.)
optimizer = tf.keras.optimizers.SGD(0.1)
mse = tf.keras.losses.MeanSquaredError()
for i in range(TRAIN_STEPS):
#print("epoch:", i)
#print("w1:", w.numpy())
#print("b:", b.numpy())
with tf.GradientTape(persistent=True,watch_accessed_variables=True) as g:
logit = tf.matmul(X, w) + b
loss = mse(y, logit)
#loss = (y - logit)*(y - logit)
gradients = g.gradient(target=loss, sources=[w, b]) # 计算梯度
#print(gradients)
optimizer.apply_gradients(zip(gradients, [w, b])) # 更新梯度
print("test_tensorflow1 w1:", w.numpy())
print("test_tensorflow1 b:", b.numpy())
def test_tensorflow2(X, y):
w = tf.Variable(initial_value=[[1.0],[1.0]])
#w2 = tf.Variable(initial_value=1.0)
b = tf.Variable(initial_value=0.)
optimizer = tf.keras.optimizers.SGD(0.1)
mse = tf.keras.losses.MeanSquaredError()
for i in range(TRAIN_STEPS):
#print("epoch:", i)
#print("w1:", w.numpy())
#print("b:", b.numpy())
with tf.GradientTape(persistent=True,watch_accessed_variables=True) as g:
logit = tf.matmul(X, w) + b
loss = tf.square((y - logit))
loss = tf.reduce_sum(loss)/3
gradients = g.gradient(target=loss, sources=[w, b]) # 计算梯度
#print(gradients)
optimizer.apply_gradients(zip(gradients, [w, b])) # 更新梯度
print("test_tensorflow2 w1:", w.numpy())
print("test_tensorflow2 b:", b.numpy())
if __name__ == '__main__':
X = [[0, 0], [1, 1], [2, 2.]]
#X = [[0], [1], [2]]
y = [[0], [1], [2.]]
test_linearRegression(X, y)
test_lasso(X, y)
test_ridge(X, y)
test_tensorflow1(X, y)
test_tensorflow2(X, y)
test_tensorflow1 w1: [[0.5456011]
[0.5456011]]
test_tensorflow1 b: -0.13680318
test_tensorflow2 w1: [[0.5456011]
[0.5456011]]
test_tensorflow2 b: -0.13680318