python scipy stats學習筆記

from scipy.stats import chi2                 # 卡方分布
from scipy.stats import norm                 # 正態分布
from scipy.stats import t                    # t分布
from scipy.stats import f                    # F分布
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import scipy.stats as stats
from scipy.stats import chi2_contingency     # 列聯表分析

# matplotlib畫圖注釋中文需要設置
from matplotlib.font_manager import FontProperties
xy_font_set = FontProperties(fname=r"c:\windows\fonts\方正稚藝簡體.ttf", size=12)
zhushi_font_set = FontProperties(fname=r"c:\windows\fonts\方正粗倩簡體.ttf", size=12)
titleYW_font_set = FontProperties(fname=r"c:\windows\fonts\Gabriola.ttf", size=20)
titleZW_font_set = FontProperties(fname=r"c:\windows\fonts\漢儀細行楷簡.ttf", size=18)

# rvs: Random Variates
# pdf: Probability Density Function                         概率密度函數
# cdf: Cumulative Distribution Function                     概率密度函數的積分函數
# sf: Survival Function (1-CDF)
# ppf: Percent Point Function (Inverse of CDF)              百分點函數，概率密度函數的積分值
# isf: Inverse Survival Function (Inverse of SF)
# stats: Return mean, variance, (Fisher’s) skew, or (Fisher’s) kurtosis
# moment: non-central moments of the distribution

# ppf以概率的形式，查詢函數值-----------類似分布臨界表

plt.figure()
# example ------------------------------------------- 卡方分布(右側單邊)
plt.subplot2grid((2, 2), (0, 0))
df = 20                                                   # 自由度
# print(chi2.ppf(0.01, df))                               # 計算函q=0.01概率時數值。其中 q = 1-a
# print(chi2.cdf(8.260, df))                              # 知道x值求a
x = np.linspace(chi2.ppf(0.01, df),                       # 繪制概率密度圖
                 chi2.ppf(0.99, df), 100)
plt.plot(x, chi2.pdf(x, df), alpha=0.6, label='chi2 pdf')
plt.title(u'自由度為20時的卡方概率密度函數圖',  fontproperties=titleZW_font_set, size=10)
# 計算平均數、方差、標准差
# print(chi2.mean(df))
# print(chi2.var(df))
# print(chi2.std(df))


# example ---------------------------------------------------- 標准正態分布(左側單邊)
plt.subplot2grid((2, 2), (0, 1))
# print(norm.ppf(0.6179))                                         # 知道q時求x, q=a
# print(norm.cdf(0.3))                                            # 知道x時求q
x = np.linspace(norm.ppf(0.01), norm.ppf(0.99), 100)
plt.plot(x, norm.pdf(x), alpha=0.6, label='norm pdf')
plt.title(u'標准正態分布概率密度函數圖',  fontproperties=titleZW_font_set, size=10)


# example ----------------------------------------------------- t分布(雙邊分布)
plt.subplot2grid((2, 2), (1, 0))
df = 15
x = np.linspace(t.ppf(0.01, df), t.ppf(0.99, df), 100)
# print(t.ppf(0.95, df))                                           # q=0.95,a=(1-q)*2
# print(t.cdf(1.753, df))
plt.plot(x, t.pdf(x, df), alpha=0.6, label='t pdf')
plt.title(u'自由度為15時的t分布概率密度函數圖',  fontproperties=titleZW_font_set, size=10)


# example ------------------------------------------------------ F分布(右側單邊分布)
plt.subplot2grid((2, 2), (1, 1))
df = 5
dn = 8
x = np.linspace(f.ppf(0.01, df, dn), f.ppf(0.99, df, dn), 100)
# print(f.ppf(0.95, df, dn))
plt.plot(x, f.pdf(x, df, dn), alpha=0.6, label='f pdf')
plt.title(u'自由度為5和8時的f分布概率密度函數圖',  fontproperties=titleZW_font_set, size=10)


# example ------------------------------------------------------- 非標准正態分布
plt.figure()
std = 1
mean = 2
normalDistribution = stats.norm(mean, std)                        # 構建統計分布
x = np.linspace(normalDistribution.ppf(0.01), normalDistribution.ppf(0.99), 100)
plt.plot(x, normalDistribution.pdf(x))
# plt.show()

# example -------------------------------------------------------- 對連續數據進行正態擬合
plt.figure()
train = pd.read_csv("csv/Titanic/train.csv")
train_Age = train.dropna(subset=['Age'])
M_S = stats.norm.fit(train_Age['Age'])                            # 正態擬合的平均值與標准差
train_Age['Age'].plot(kind='kde')                                 # 原本的概率密度分布圖

normalDistribution = stats.norm(M_S[0], M_S[1])                   # 繪制擬合的正態分布圖
x = np.linspace(normalDistribution.ppf(0.01), normalDistribution.ppf(0.99), 100)
plt.plot(x, normalDistribution.pdf(x), c='orange')
plt.xlabel('Age about Titanic')
plt.title('Titanic[Age] on NormalDistribution', size=20)
plt.legend(['Origin', 'NormDistribution'])


# ----------------------------------------------------------------- R x C列聯表，獨立性檢驗
# 建立關於性別與存活
train_pclass_0 = train['Pclass'][train['Survived'] == 0].value_counts()
train_pclass_1 = train['Pclass'][train['Survived'] == 1].value_counts()
train_pclass_01 = pd.concat([train_pclass_0, train_pclass_1], axis=1, sort=True)
train_pclass_01.columns = ['Not_Survived', 'Survived']
g, p, dof, expctd = chi2_contingency(train_pclass_01.values)             # g為chi2值，服從自由度為dof的卡方分布

print(g)
# 擬合優度檢驗，判斷兩個分類型變量是否獨立
# 首先繪制卡方自由度為dof的曲線
plt.figure()
x = np.linspace(chi2.ppf(0.01, dof), chi2.ppf(0.99, dof), 100)
plt.plot(x, chi2.pdf(x, dof))

# 以95%置信區間，查看小概率事件區間
plt.axvline(chi2.ppf(0.975, dof), color='r')
plt.axvline(chi2.ppf(0.025, dof), color='r')
plt.title('chi2 distribution'+'whose dof is '+str(dof))
plt.text(chi2.ppf(0.975, dof), 0.02, 'q=0.95,z='+str(round(chi2.ppf(0.975, dof), 2)), ha='right', va='top', color='g', alpha=0.8, size=15)
plt.text(chi2.ppf(0.025, dof), 0.02, 'q=0.05,z='+str(round(chi2.ppf(0.025, dof), 2)), ha='left', va='top', color='g', alpha=0.8, size=15)

plt.show()