機器學習算法之多項式回歸

多項式回歸，采用升維的方式，把x的冪當作新的特征，再利用線性回歸方法解決

import numpy as np 
import matplotlib.pyplot as plt
​
x = np.random.uniform(-4,4,100)
y = 0.6*x**2 + x + 2 + np.random.normal(size=100)
​
# 單線性回歸
from sklearn.linear_model import LinearRegression
lin_reg = LinearRegression()
lin_reg.fit(X,y)
y_predict = lin_reg.predict(X)
​
＃　多線性回歸
X2 = np.hstack([X,X**2])
lin_reg = LinearRegression()
lin_reg.fit(X2,y)
y_predict2 = lin_reg.predict(X2)
​
plt.scatter(X,y)
plt.plot(np.sort(x),y_predict2[np.argsort(x)],color='r')
plt.show()
​
lin_reg.coef_   # array([0.98046078, 0.59747765])
lin_reg.intercept_  # 2.0771588970176973

Scikit-learn中實現

from sklearn.preprocessing import PolynomialFeatures
​
poly = PolynomialFeatures(degree=2) # degree表示設置的最高冪的次數
X2 = poly.fit_transform(X)
X2.shape　   # (100, 3)
​
lin_reg = LinearRegression()
lin_reg.fit(X2,y)
y_predict2 = lin_reg.predict(X2)
​
plt.scatter(X,y)
plt.plot(np.sort(x),y_predict2[np.argsort(x)],color='r')
plt.show()
​
lin_reg.coef_   # array([0. ,0.98046078, 0.59747765])
lin_reg.intercept_  # 2.0771588970176964

Pipeline實現

from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
​
pipeline = Pipeline([
    ('poly',PolynomialFeatures(degree=2)),
    ('std_scaler',StandardScaler()),
    ('lin_reg',LinearRegression())
])
X3 = pipeline.fit(X,y)
y_predict3 = pipeline.predict(X)
​
# pipeline.coef_ # 該項報錯！ 放入管道后，不能直接取系數？？？

過擬合和欠擬合

def plot_learning_curve(algo, X_train, X_test, y_train, y_test):
    train_score = []
    test_score = []
    for i in range(1, len(X_train)+1):
        algo.fit(X_train[:i], y_train[:i])
    
        y_train_predict = algo.predict(X_train[:i])
        train_score.append(mean_squared_error(y_train[:i], y_train_predict))
    
        y_test_predict = algo.predict(X_test)
        test_score.append(mean_squared_error(y_test, y_test_predict))
        
    plt.plot([i for i in range(1, len(X_train)+1)], 
                               np.sqrt(train_score), label="train")
    plt.plot([i for i in range(1, len(X_train)+1)], 
                               np.sqrt(test_score), label="test")
    plt.legend()
    plt.axis([0, len(X_train)+1, 0, 4])
    plt.show()
    
plot_learning_curve(LinearRegression(), X_train, X_test, y_train, y_test)
​
poly2_reg = PolynomialRegression(degree=2)
plot_learning_curve(poly2_reg, X_train, X_test, y_train, y_test)

交叉驗證

 

默認分成3份,要自定義,則修改參數 cv

cross_val_score(estimator,X_train,y_train) 結果返回３組（默認cv=3）的評分

import numpy as np
from sklearn import datasets
digits = datasets.load_digits()
X,y = digits.data,digits.target
from sklearn.model_selection import train_test_split
​
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.4, random_state=666)
​
from sklearn.model_selection import cross_val_score
from sklearn.neighbors import KNeighborsClassifier
​
knn_clf = KNeighborsClassifier()
cross_val_score(knn_clf,X_train,y_train)
# array([0.98895028, 0.97777778, 0.96629213])
​
best_k, best_p, best_score = 0, 0, 0
for k in range(2, 11):
    for p in range(1, 6):
        knn_clf = KNeighborsClassifier(weights="distance", n_neighbors=k, p=p)
        # knn_clf.fit(X_train, y_train)
        # score = knn_clf.score(X_test, y_test)
        scores = cross_val_score(knn_clf,X_train,y_train,cv=3)
        score = np.mean(scores)
        if score > best_score:
            best_k, best_p, best_score = k, p, score
            
print("Best K =", best_k)
print("Best P =", best_p)
print("Best Score =", best_score)
'''
Best K = 2
Best P = 2
Best Score = 0.9823599874006478
'''
best_knn_clf = KNeighborsClassifier(weights="distance", n_neighbors=2, p=2)
best_knn_clf.fit(X_train,y_train)
# y_predict = best_knn_clf.predict(X_test)
best_knn_clf.score(X_test,y_test) # 0.980528511821975

回顧網格搜索

from sklearn.model_selection import GridSearchCV
​
grid_params = {
    'weights': ['distance'],
    'n_neighbors': [i for i in range(2, 11)], 
    'p': [i for i in range(1, 6)]
}
knn2_clf = KNeighborsClassifier()
clf = GridSearchCV(knn2_clf,grid_params,cv=3,n_jobs=-1)
clf.fit(X_train,y_train)
clf.best_score_ # 0.9823747680890538
clf.best_params_ # {'n_neighbors': 2, 'p': 2, 'weights': 'distance'}
clf.best_estimator_ 
# KNeighborsClassifier(n_neighbors=2, p=2,weights='distance')

偏差和方差

 

偏差和方差通常是矛盾的.

減低偏差,會提高方差.

減低方差,會提高偏差.

有些算法天生是高方差的算法,如kNN.

非參數學習通常都是高方差算法,因為不對數據進行任何假設.

有些算法天生是高偏差的算法,如線性回歸.

參數學習通常都是高偏差算法,因為對數據具有及強的假設.

 

機器學習算法,通常要解決的是高方差問題,比如過擬合,通常手段:

1. 降低模型復雜度

2. 減少數據維度;降噪

3. 增加樣本數

4. 使用驗證集

5. 模型正則化

模型正則化--嶺回歸

參數--alpha

import numpy as np
import matplotlib.pyplot as plt
​
# 構建數據集
np.random.seed(42)
x = np.random.uniform(-3.0, 3.0, size=100)
X = x.reshape(-1, 1)
y = 0.5 * x + 3 + np.random.normal(0, 1, size=100)
plt.scatter(x, y)
plt.show()
​
# 創建管道－－多項式線性回歸
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import PolynomialFeatures
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import LinearRegression
​
def PolynomialRegression(degree):
    return Pipeline([
        ("poly", PolynomialFeatures(degree=degree)),
        ("std_scaler", StandardScaler()),
        ("lin_reg", LinearRegression())
    ])
​
# 切割數據集為訓練和測試集
from sklearn.model_selection import train_test_split
​
np.random.seed(666)
X_train, X_test, y_train, y_test = train_test_split(X, y)
​
# fit訓練數據集，predict測試數據集，求取均方誤差MSE
from sklearn.metrics import mean_squared_error
​
poly_reg = PolynomialRegression(degree=20)
poly_reg.fit(X_train, y_train)
​
y_poly_predict = poly_reg.predict(X_test)
mean_squared_error(y_test, y_poly_predict) # 167.94010867293571
# 繪制擬合函數曲線圖
def plot_model(model):
    X_plot = np.linspace(-3, 3, 100).reshape(100, 1)
    y_plot = model.predict(X_plot)
    plt.scatter(x, y)
    plt.plot(X_plot[:,0], y_plot, color='r')
    plt.axis([-3, 3, 0, 6])
    plt.show()
​
plot_model(poly_reg)
​
# 使用嶺回歸
from sklearn.linear_model import Ridge
​
def RidgeRegression(degree, alpha):
    return Pipeline([
        ("poly", PolynomialFeatures(degree=degree)),
        ("std_scaler", StandardScaler()),
        ("ridge_reg", Ridge(alpha=alpha))
    ])
​
ridge1_reg = RidgeRegression(20, 0.0001)  # 1.3233492754051845  崎嶇的線條
# ridge1_reg = RidgeRegression(20, 1)  # 1.1888759304218448 崎嶇的光滑曲線
# ridge1_reg = RidgeRegression(20, 100)  # 1.3196456113086197   光滑的曲線
# ridge1_reg = RidgeRegression(20, 10000) #  1.8408455590998372 和x軸平行的直線
# 解析：　當alpha無窮大時，theta(i,i>=1)的平方必須都趨於０，當theta(i)都等於０時，即y=theta(0),和x軸平行
ridge1_reg.fit(X_train, y_train)
​
y1_predict = ridge1_reg.predict(X_test)
mean_squared_error(y_test, y1_predict) # 1.3233492754051845
plot_model(ridge1_reg)

模型正則化--LASSO回歸

 lasso回歸和嶺回歸比較

# 創建lasso管道
from sklearn.linear_model import Lasso
def LassoRegression(degree, alpha):
    return Pipeline([
        ("poly", PolynomialFeatures(degree=degree)),
        ("std_scaler", StandardScaler()),
        ("lasso_reg", Lasso(alpha=alpha))
    ])
lasso1_reg = LassoRegression(20, 0.01) # 1.1496080843259966
# lasso1_reg = LassoRegression(20, 0.1) # 1.1213911351818648
# lasso1_reg = LassoRegression(20, 1) # 1.8408939659515595
lasso1_reg.fit(X_train, y_train)
​
y1_predict = lasso1_reg.predict(X_test)
mean_squared_error(y_test, y1_predict)
​
plot_model(lasso1_reg)

模型泛化--更具普遍性

我們訓練機器學習的模型,不是為了在我們的訓練集上有很好的測試結果,而是為了應對未來,在未知的數據集下有更好的預測結果.

L1&L2&彈性網絡

 

 

注: 上述筆記是自己在 慕課網學習 劉宇波老師<Python3入門機器學習 經典算法與應用>課程時整理的, 截圖皆是來自視頻中的ppt. 附網課視頻鏈接: https://s.imooc.com/Sk8Yr5g