評估指標【交叉驗證&ROC曲線】

# -*- coding: utf-8 -*-
"""
Created on Mon Sep 10 11:21:27 2018

@author: zhen
"""
from sklearn.datasets import fetch_mldata
import numpy as np
from sklearn.linear_model import SGDClassifier
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import cross_val_predict
from sklearn.metrics import precision_recall_curve
import matplotlib
import matplotlib.pyplot as plt
from sklearn.metrics import roc_curve
from sklearn.metrics import roc_auc_score
from sklearn.ensemble import RandomForestClassifier

mnist = fetch_mldata('MNIST original', data_home='D:/AnalyseData學習資源庫/人工智能開發/分類評估/資料/test_data_home')

x, y = mnist['data'], mnist['target']
some_digit = x[36000]  #獲取第36000行數據

some_digit_image = some_digit.reshape(28, 28)

plt.imshow(some_digit_image, cmap=matplotlib.cm.binary,
           interpolation='nearest', vmin=0, vmax=1)
plt.axis('off')
plt.show()

x_train, x_test, y_train, y_test = x[:60000], x[60000:], y[:60000], y[60000:]
shuffle_index = np.random.permutation(60000)

x_train, y_train = x_train[shuffle_index], y_train[shuffle_index]

y_train_5 = (y_train == 5)
y_test_5 = (y_test == 5)

sgd_clf = SGDClassifier(loss='log', random_state=42, max_iter=1000, tol=1e-4)
sgd_clf.fit(x_train, y_train_5)  

result = sgd_clf.predict([some_digit])

print(cross_val_score(sgd_clf, x_train, y_train_5, cv=3, scoring='accuracy'))
print(cross_val_score(sgd_clf, x_train, y_train_5, cv=3, scoring='precision'))
print(cross_val_score(sgd_clf, x_train, y_train_5, cv=3, scoring='recall'))

sgd_clf.fit(x_train, y_train_5)

y_scores = sgd_clf.decision_function([some_digit])

threshold = 0
y_some_digit_pred = (y_scores > threshold)

threshold = 200000
y_some_digit_pred = (y_scores > threshold)

# cv 數據集划分的個數
y_scores = cross_val_predict(sgd_clf, x_train, y_train_5, cv=3, method='decision_function')

precisions, recalls, thresholds = precision_recall_curve(y_train_5, y_scores)


def plot_precision_recall_vs_threshold(precisions, recalls, thresholds):
    plt.plot(thresholds, precisions[:-1], 'b--',label='Precision')
    plt.plot(thresholds, recalls[:-1], 'r--', label='Recall')
    plt.xlabel("Threshold")
    plt.legend(loc='upper left')
    plt.ylim([0, 1])
    plt.show()  
    
    
def plot_roc_curve(fpr, tpr, label=None):
    plt.plot(fpr, tpr, linewidth=2, label='roc')
    plt.plot([0, 1], [0, 1], 'k--', label='mid')
    plt.legend(loc='lower right')
    # plt.axes([0, 1, 0, 1]) : 前兩個參數表示坐標原點的位置，后兩個表示x,y軸的長度
    plt.xlabel('fpr')
    plt.ylabel('tpr')
    plt.show()  


plot_precision_recall_vs_threshold(precisions, recalls, thresholds)

fpr, tpr, thresholds = roc_curve(y_train_5, y_scores)
plot_roc_curve(fpr, tpr)
   
print(roc_auc_score(y_train_5, y_scores))

forest_clf = RandomForestClassifier(random_state=42)
y_probas_forest = cross_val_predict(forest_clf, x_train, y_train_5, cv=3, method='predict_proba')
y_scores_forest = y_probas_forest[:, 1]
fpr_forest, tpr_forest, thresholds_forest = roc_curve(y_train_5, y_scores_forest)
plt.plot(fpr, tpr, 'b:', label='SGD')
plt.plot(fpr_forest, tpr_forest, label='Random Forest')
plt.legend(loc='lower right')
plt.show()

print(roc_auc_score(y_train_5, y_scores_forest))

　　　　　　　　　　

總結：正向准確率和召回率在整體上成反比，可知在使用相同數據集，相同驗證方式的情況下，隨機森林要優於隨機梯度下降！