转载：https://www.cnblogs.com/jasonfreak/p/5448385.html
特征选择主要从两个方面入手：
- 特征是否发散：特征发散说明特征的方差大，能够根据取值的差异化度量目标信息.
- 特征与目标相关性：优先选取与目标高度相关性的.
- 对于特征选择，有时候我们需要考虑分类变量和连续变量的不同.
1.过滤法：按照发散性或者相关性对各个特征进行评分，设定阈值或者待选择阈值的个数选择特征
方差选择法：建议作为数值特征的筛选方法
计算各个特征的方差，然后根据阈值，选择方差大于阈值的特征
from sklearn.feature_selection import VarianceThreshold from sklearn.datasets import load_iris import pandas as pd X,y = load_iris(return_X_y=True) X_df = pd.DataFrame(X,columns=list("ABCD")) #建议作为数值特征的筛选方法，对于分类特征可以考虑每个类别的占比问题
ts = 0.5 vt = VarianceThreshold(threshold=ts) vt.fit(X_df) #查看各个特征的方差
dict_variance = {} for i,j in zip(X_df.columns.values,vt.variances_): dict_variance[i] = j
#获取保留了的特征的特征名 ls = list() for i,j in dict_variance.items(): if j >= ts: ls.append(i) X_new = pd.DataFrame(vt.fit_transform(X_df),columns=ls)
卡方检验：建议作为分类问题的分类变量的筛选方法
经典的卡方检验是检验定性自变量对定性因变量的相关性。假设自变量有N种取值，因变量有M种取值，考虑自变量等于i且因变量等于j的样本频数的观察值与期望的差距，构建统计量：
from sklearn.feature_selection import VarianceThreshold,SelectKBest,chi2 from sklearn.datasets import load_iris import pandas as pd X,y = load_iris(return_X_y=True) X_df = pd.DataFrame(X,columns=list("ABCD")) (chi2,pval) = chi2(X_df,y) dict_feature = {} for i,j in zip(X_df.columns.values,chi2): dict_feature[i]=j #对字典按照values排序
ls = sorted(dict_feature.items(),key=lambda item:item[1],reverse=True) #特征选取数量
k =2 ls_new_feature=[] for i in range(k): ls_new_feature.append(ls[i][0]) X_new = X_df[ls_new_feature]
互信息法：建议作为分类问题的分类变量的筛选方法
经典的互信息也是评价定性自变量对定性因变量的相关性的，为了处理定量数据，最大信息系数法被提出，互信息计算公式如下：
from sklearn.feature_selection import VarianceThreshold,SelectKBest,chi2 from sklearn.datasets import load_iris import pandas as pd from sklearn.feature_selection import mutual_info_classif #用于度量特征和离散目标的互信息
X,y = load_iris(return_X_y=True) X_df = pd.DataFrame(X,columns=list("ABCD")) feature_cat = ["A","D"] discrete_features = [] feature = X_df.columns.values.tolist() for k in feature_cat: if k in feature: discrete_features.append(feature.index(k)) mu = mutual_info_classif(X_df,y,discrete_features=discrete_features, n_neighbors=3, copy=True, random_state=None) dict_feature = {} for i,j in zip(X_df.columns.values,mu): dict_feature[i]=j #对字典按照values排序
ls = sorted(dict_feature.items(),key=lambda item:item[1],reverse=True) #特征选取数量
k =2 ls_new_feature=[] for i in range(k): ls_new_feature.append(ls[i][0]) X_new = X_df[ls_new_feature]
from sklearn.feature_selection import VarianceThreshold,SelectKBest,chi2 from sklearn.datasets import load_iris import pandas as pd from sklearn.feature_selection import mutual_info_classif,mutual_info_regression #用于度量特征和连续目标的互信息
X,y = load_iris(return_X_y=True) X_df = pd.DataFrame(X,columns=list("ABCD")) feature_cat = ["A","D"] discrete_features = [] feature = X_df.columns.values.tolist() for k in feature_cat: if k in feature: discrete_features.append(feature.index(k)) mu = mutual_info_regression(X_df,y,discrete_features=discrete_features, n_neighbors=3, copy=True, random_state=None) dict_feature = {} for i,j in zip(X_df.columns.values,mu): dict_feature[i]=j #对字典按照values排序
ls = sorted(dict_feature.items(),key=lambda item:item[1],reverse=True) #特征选取数量
k =2 ls_new_feature=[] for i in range(k): ls_new_feature.append(ls[i][0]) X_new = X_df[ls_new_feature]
2.包装法
递归特征消除法：用一个基模型来进行多轮训练，每轮训练后，消除若干权值系数的特征，再基于新的特征集进行下一轮训练
from sklearn.datasets import load_iris import pandas as pd from sklearn.feature_selection import RFE,RFECV from sklearn.ensemble import RandomForestClassifier X,y = load_iris(return_X_y=True) X_df = pd.DataFrame(X,columns=list("ABCD")) refCV = RFECV(estimator=RandomForestClassifier(), step=0.5, cv =5, scoring=None, n_jobs=-1) refCV.fit(X_df,y) #保留特征的数量
refCV.n_features_ #保留特征的False、True标记
refCV.support_ feature_new = X_df.columns.values[refCV.support_] #交叉验证分数
refCV.grid_scores_
3.嵌入的方法
基于L1范数：使用带惩罚项的基模型，除了筛选出特征外，同时也进行了降维
from sklearn.datasets import load_iris import pandas as pd from sklearn.feature_selection import SelectFromModel from sklearn.linear_model import LogisticRegression X,y = load_iris(return_X_y=True) X_df = pd.DataFrame(X,columns=list("ABCD")) sf = SelectFromModel(estimator=LogisticRegression(penalty="l1", C=0.1), threshold=None, prefit=False, norm_order=1) sf.fit(X_df,y) X_new = X_df[X_df.columns.values[sf.get_support()]]
基于树模型的特征选择法：
树模型中GBDT也可用来作为基模型进行特征选择，使用feature_selection库的SelectFromModel类结合GBDT模型
from sklearn.datasets import load_iris import pandas as pd from sklearn.feature_selection import SelectFromModel from sklearn.ensemble import GradientBoostingClassifier X,y = load_iris(return_X_y=True) X_df = pd.DataFrame(X,columns=list("ABCD")) sf = SelectFromModel(estimator=GradientBoostingClassifier(), threshold=None, prefit=False, norm_order=1) sf.fit(X_df,y) X_new = X_df[X_df.columns.values[sf.get_support()]]