python大戰機器學習——模型評估、選擇與驗證

1、損失函數和風險函數

(1)損失函數：常見的有 0-1損失函數  絕對損失函數  平方損失函數  對數損失函數

(2)風險函數：損失函數的期望      經驗風險：模型在數據集T上的平均損失

　 根據大數定律，當N趨向於∞時，經驗風險趨向於風險函數

2、模型評估方法

(1)訓練誤差與測試誤差

　　訓練誤差：關於訓練集的平均損失

　　測試誤差：定義模型關於測試集的平均損失。其反映了學習方法對未知測試數據集的預測能力

(2)泛化誤差：學到的模型對未知數據的預測能力。其越小，該模型越有效。泛化誤差定義為所學習模型的期望風險

(3)過擬合：對已知數據預測得很好，對未知數據預測得很差的現象。原因是將訓練樣本本身的一些特點當做了所有潛在樣本都具有的一般性質，這會造成泛化能力的下降。常用的防止過擬合的辦法為正則化。正則化是基於結構化風險最小化策略的實現。

3、模型評估

(1)留出法：直接將數據划分為三個互斥的部分，然后在訓練集上訓練模型，在驗證集上選擇模型，最后用測試集上的誤差作為泛化誤差的估計。

(2)交叉驗證法(S折交叉驗證法)：數據隨機划分為S個互不相交且大小相同的子集，利用S-1個子集數據訓練模型，利用余下的一個子集測試模型。對S種組合依次重復進行，獲取測試誤差的均值。

(3)留一法：留出一個樣例作為測試集。其缺點就是當數據集比較大時計算量太大

(4)自助法：先從T中隨機取出一個樣本放入采樣集T_S中，再把該樣本放回T中。經過N次隨機采樣操作，得到包含N個樣本的采樣集T_S。將TS用作訓練集，T-T_S用過測試集。

4、性能度量

(1)測試准確率和測試錯誤率

(2)混淆矩陣

　　查准率：P=TP/(TP+FP)  ，即所有預測為正類的結果中，真正的正類的比例

　　查全率：R=TP/(TP+FN)，即正真的正類中，被分類器找出來的比例

　　不同的問題中，判別標准不同。對於推薦系統，更側重於查准率（即推薦的結果中，用戶真正感興趣的比例）；對於醫學診斷系統，更側重於查全率（即疾病被發現的比例）

　　2/F1=1/P+1/R

5、ROC曲線

　　真正例率：TPR=TP/(TP+FN)

　　假正例率：FPR=FP/(TN+FP)，刻畫的是分類器錯認為正類的負實例占所有負實例的比例

　　以真正例率為縱軸、假正例率為橫軸作圖，就得到ROC曲線。在ROC圖中，對角線對應於隨機猜想模型。點(0,1)對應於理想模型。通常ROC曲線越靠近點(0,1)越好。

6、偏差方差分解

代碼如下：

from sklearn.metrics import zero_one_loss,log_loss
from sklearn.model_selection import train_test_split,KFold,StratifiedKFold,LeaveOneOut,cross_val_score
from sklearn.datasets import load_digits,load_iris
from sklearn.svm import LinearSVC,SVC
from sklearn.metrics import accuracy_score,precision_score,recall_score,f1_score,classification_report
from sklearn.metrics import confusion_matrix,precision_recall_curve,roc_curve
from sklearn.metrics import mean_absolute_error,mean_squared_error,classification_report
from sklearn.multiclass import OneVsRestClassifier
from sklearn.model_selection import validation_curve,learning_curve,GridSearchCV,RandomizedSearchCV
import matplotlib.pyplot as plt
from sklearn.preprocessing import label_binarize
from sklearn.linear_model import LogisticRegression
import numpy as np
#zero_one_loss
# y_true=[1,1,1,1,1,0,0,0,0,0]
# y_pred=[0,0,0,1,1,1,1,1,0,0]
# print("zero_one_loss<fraction>:",zero_one_loss(y_true,y_pred,normalize=True))
# print("zero_one_loss<num>:",zero_one_loss(y_true,y_pred,normalize=False))

#log_loss
# y_true=[1,1,1,0,0,0]
# y_pred=[[0.1,0.9],
#         [0.2,0.8],
#         [0.3,0.7],
#         [0.7,0.3],
#         [0.8,0.2],
#         [0.9,0.1]
#         ]
# print("log_loss<average>:",log_loss(y_true,y_pred,normalize=True))
# print("log_loss<total>:",log_loss(y_true,y_pred,normalize=False))

#train_test_split
# X=[
#     [1,2,3,4],
#     [11,12,13,14],
#     [21,22,23,24],
#     [31,32,33,34],
#     [41,42,43,44],
#     [51,52,53,54],
#     [61,62,63,64],
#     [71,72,73,74]
# ]
# Y=[1,1,0,0,1,1,0,0]
# X_train,X_test,Y_train,Y_test=train_test_split(X,Y,test_size=0.4,random_state=0)
# print("X_train=",X_train)
# print("X_test=",X_test)
# print("Y_train=",Y_train)
# print("Y_test=",Y_test)
# X_train,X_test,Y_train,Y_test=train_test_split(X,Y,test_size=0.4,random_state=0,stratify=Y)
# print("X_train=",X_train)
# print("X_test=",X_test)
# print("Y_train=",Y_train)
# print("Y_test=",Y_test)

#KFold
# X=np.array([
#     [1,2,3,4],
#     [11,12,13,14],
#     [21,22,23,24],
#     [31,32,33,34],
#     [41,42,43,44],
#     [51,52,53,54],
#     [61,62,63,64],
#     [71,72,73,74],
#     [81,82,83,84]
# ])
# Y=np.array([1,1,0,0,1,1,0,0,1])
#
# folder=KFold(n_splits=3,random_state=0,shuffle=False)
# for train_index,test_index in folder.split(X,Y):
#     print("Train Index:",train_index)
#     print("Test Index:",test_index)
#     print("X_train:",X[train_index])
#     print("X_test:",X[test_index])
#     print("")
#
# shuffle_folder=KFold(n_splits=3,random_state=0,shuffle=True)
# for train_index,test_index in shuffle_folder.split(X,Y):
#     print("Train Index:",train_index)
#     print("Test Index:",test_index)
#     print("X_train:",X[train_index])
#     print("X_test:",X[test_index])
#     print("")

#StratifiedKFold
# stratified_folder=StratifiedKFold(n_splits=4,random_state=0,shuffle=False)
#as the operation is similar to the above,pass

#LeaveOneOut,too easy,pass
# loo=LeaveOneOut(len(Y))

#cross_val_score
# digits=load_digits()
# X=digits.data
# Y=digits.target
#
# result=cross_val_score(LinearSVC(),X,Y,cv=10)
# print("Cross Val Score is:",result)

#accuracy_score,pass
# accuracy_score(y_true,y_pred,normalize=True/False)

#precision_score,pass
# precision_socre(y_true,y_pred)

#recall_score,pass
# recall_score(y_true,y_pred)

#f1_score,pass
# f1_score(y_true,y_pred)

#fbeta_score,pass
# fbeta_score(y_true,y_pred,beta=num_beta)

#classification_report
# y_true=[1,1,1,1,1,0,0,0,0,0]
# y_pred=[0,0,1,1,0,0,0,0,0,0]
# print("Classification Report:\n",classification_report(y_true,y_pred,target_names=["class_0","class_1"]))

#confusion_matrix,pass
# confusion_matrix(y_true,y_pred,labels=[0,1])

#precision_recall_curve
# iris=load_iris()
# X=iris.data
# Y=iris.target
# #print(X,'\n',Y)
# Y=label_binarize(Y,classes=[0,1,2])
# n_classes=Y.shape[1]
# # print(n_classes,'\n',Y)
# np.random.seed(0)
# n_samples,n_features=X.shape
# # print(n_samples,'\n',n_features)
# X=np.c_[X,np.random.randn(n_samples,200*n_features)]
# # n_samples,n_features=X.shape
# # print(n_samples,'\n',n_features)
# x_train,x_test,y_train,y_test=train_test_split(X,Y,test_size=0.5,random_state=0)
# clf=OneVsRestClassifier(SVC(kernel='linear',probability=True,random_state=0))
# clf.fit(x_train,y_train)
# y_score=clf.fit(x_train,y_train).decision_function(x_test)
# # print(y_score)
# fig=plt.figure()
# ax=fig.add_subplot(1,1,1)
# precision=dict()
# recall=dict()
# for i in range(n_classes):
#     precision[i],recall[i],_=precision_recall_curve(y_test[:,i],y_score[:,i])
#     ax.plot(recall[i],precision[i],label="target=%s"%i)
# ax.set_xlabel("Recall Score")
# ax.set_ylabel("Precision Score")
# ax.set_title("P-R")
# ax.legend(loc="best")
# ax.set_xlim(0,1.1)
# ax.set_ylim(0,1.1)
# ax.grid()
# plt.show()

#roc_curve,roc_auc_score,pass
# roc_curve(y_test,y_score)

#mean_absolute_error,pass
# mean_absolute_error(y_true,y_pred)

#mean_squared_error,pass
# mean_squared_error(y_true,y_pred)

#validation_curve,pass
# validation_curve(LinearSVC(),X,Y,param_name="C",param_range=np.logspace(-2,2),cv=10,scoring="accuracy")

#learning_curve,pass
# train_size=np.linspace(0.1,1.0,endpoint=True,dtype='float')
# learning_curve(LinearSVC(),X,Y,cv=10,scoring="accuracy",train_sizes=train_size)

#GridSearcgCV
# digits=load_digits()
# x_train,x_test,y_train,y_test=train_test_split(digits.data,digits.target,test_size=0.25,random_state=0,stratify=digits.target)
# tuned_parameters=[{'penalty':['l1','l2'],'C':[0.01,0.05,0.1,0.5,1,5,10,50,100],'solver':['liblinear'],'multi_class':['ovr']},
#                   {'penalty':['l2'],'C':[0.01,0.05,0.1,0.5,1,5,10,50,100],'solver':['lbfgs'],'multi_class':['ovr','multinomial']},
#                   ]
# clf=GridSearchCV(LogisticRegression(tol=1e-6),tuned_parameters,cv=10)
# clf.fit(x_train,y_train)
# print("Best parameters set found:",clf.best_params_)
# print("Grid scores:")
# for params,mean_score,scores in clf.grid_scores_:
#     print("\t%0.3f(+/-%0.03f) for %s"%(mean_score,scores.std()*2,params))
# 
# print("Optimized Score:",clf.score(x_test,y_test))
# print("Detailed classification report:")
# y_true,y_pred=y_test,clf.predict(x_test)
# print(classification_report(y_true,y_pred))

#RandomizedSearchCV
# RandomizedSearchCV(LogisticRegression(penalty='l2',solver='lbfgs',tol=1e-6,tuned_parameters,cv=10,scoring='accuracy',n_iter=100))