通過移動設備行為數據預測性別年齡

1. 通過行為習慣對移動用戶人口屬性(年齡+性別)進行預測。

2. 數據及包含~20萬用戶數據，分成12組，同時提供了用戶行為屬性，如:手機品牌、型號、APP的類型等。

3. 通過logloss評價

 

main.py

# -*- coding: utf-8 -*-


import pandas as pd
import os
from pd_tools import split_train_test, get_part_data
import numpy as np
from sklearn.preprocessing import LabelEncoder, OneHotEncoder
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import LogisticRegression
from sklearn import svm
from sklearn.decomposition import PCA
from ml_tools import get_best_model
from sklearn.metrics import log_loss
from sklearn.feature_selection import VarianceThreshold

# 數據集變量聲明
dataset_path = './dataset'
gender_age_filename = 'gender_age.csv'
phone_brand_device_model_filename = 'phone_brand_device_model.csv'
events_filename = 'events.csv'
app_events_filename = 'app_events.csv'
app_labels_filename = 'app_labels.csv'
label_categories_filename = 'label_categories.csv'

train_gender_age_filename = 'gender_age_train.csv'
test_gender_age_filename = 'gender_age_test.csv'

is_first_run = False


def run_main():
    """
        主函數
    """
    if is_first_run:
        # 1. 分割數據集
        print('分割數據集')
        all_gender_age = pd.read_csv(os.path.join(dataset_path, gender_age_filename))
        df_train, df_test = split_train_test(all_gender_age)
        # 查看訓練集測試集基本信息
        print('訓練集中各類的數據個數：', df_train.groupby('group').size())
        print('測試集中各類的數據個數：', df_test.groupby('group').size())

        # 保存分割的數據集
        df_train.to_csv(os.path.join(dataset_path, train_gender_age_filename),
                        index=False)
        df_test.to_csv(os.path.join(dataset_path, test_gender_age_filename),
                       index=False)

    # 2. 加載數據
    print('加載數據')
    # 加載數據
    gender_age_train = pd.read_csv(os.path.join(dataset_path, train_gender_age_filename),
                                   index_col='device_id')
    gender_age_test = pd.read_csv(os.path.join(dataset_path, test_gender_age_filename),
                                  index_col='device_id')

    # 選取部分數據用於實驗
    percent = 0.1
    gender_age_train = get_part_data(gender_age_train, percent=percent)
    gender_age_test = get_part_data(gender_age_test, percent=percent)

    phone_brand_device_model = pd.read_csv(os.path.join(dataset_path, phone_brand_device_model_filename))
    # 去掉重復數據
    phone_brand_device_model = phone_brand_device_model.drop_duplicates('device_id').set_index('device_id')

    events = pd.read_csv(os.path.join(dataset_path, events_filename),
                         usecols=['device_id', 'event_id'], index_col='event_id')
    app_events = pd.read_csv(os.path.join(dataset_path, app_events_filename),
                             usecols=['event_id', 'app_id'])
    # app_labels = pd.read_csv(os.path.join(dataset_path, app_labels_filename))

    # 3. 特征工程
    # 3.1 手機品牌特征
    # 使用LabelEncoder將類別轉換為數字
    brand_label_encoder = LabelEncoder()
    brand_label_encoder.fit(phone_brand_device_model['phone_brand'].values)
    phone_brand_device_model['brand_label_code'] = \
        brand_label_encoder.transform(phone_brand_device_model['phone_brand'].values)
    gender_age_train['brand_label_code'] = phone_brand_device_model['brand_label_code']
    gender_age_test['brand_label_code'] = phone_brand_device_model['brand_label_code']

    # 使用OneHotEncoder將數字轉換為OneHot碼
    brand_onehot_encoder = OneHotEncoder()
    brand_onehot_encoder.fit(phone_brand_device_model['brand_label_code'].values.reshape(-1, 1))
    tr_brand_feat = brand_onehot_encoder.transform(gender_age_train['brand_label_code'].values.reshape(-1, 1))
    te_brand_feat = brand_onehot_encoder.transform(gender_age_test['brand_label_code'].values.reshape(-1, 1))

    print('[手機品牌]特征維度：', tr_brand_feat.shape[1])

    # 3.2 手機型號特征
    # 合並手機品牌與型號字符串
    phone_brand_device_model['brand_model'] = \
        phone_brand_device_model['phone_brand'].str.cat(phone_brand_device_model['device_model'])

    # 使用LabelEncoder將類別轉換為數字
    model_label_encoder = LabelEncoder()
    model_label_encoder.fit(phone_brand_device_model['brand_model'].values)
    phone_brand_device_model['brand_model_label_code'] = \
        model_label_encoder.transform(phone_brand_device_model['brand_model'].values)
    gender_age_train['brand_model_label_code'] = phone_brand_device_model['brand_model_label_code']
    gender_age_test['brand_model_label_code'] = phone_brand_device_model['brand_model_label_code']

    # 使用OneHotEncoder將數字轉換為OneHot碼
    model_onehot_encoder = OneHotEncoder()
    model_onehot_encoder.fit(phone_brand_device_model['brand_model_label_code'].values.reshape(-1, 1))
    tr_model_feat = model_onehot_encoder.transform(gender_age_train['brand_model_label_code'].values.reshape(-1, 1))
    te_model_feat = model_onehot_encoder.transform(gender_age_test['brand_model_label_code'].values.reshape(-1, 1))

    print('[手機型號]特征維度：', tr_model_feat.shape[1])

    # 3.3 安裝app特征
    device_app = app_events.merge(events, how='left', left_on='event_id', right_index=True)
    # 運行app的總次數
    n_run_s = device_app['app_id'].groupby(device_app['device_id']).size()

    # 運行app的個數
    n_app_s = device_app['app_id'].groupby(device_app['device_id']).nunique()

    gender_age_train['n_run'] = n_run_s
    gender_age_train['n_app'] = n_app_s

    # 填充缺失數據
    gender_age_train['n_run'].fillna(0, inplace=True)
    gender_age_train['n_app'].fillna(0, inplace=True)

    gender_age_test['n_run'] = n_run_s
    gender_age_test['n_app'] = n_app_s

    # 填充缺失數據
    gender_age_test['n_run'].fillna(0, inplace=True)
    gender_age_test['n_app'].fillna(0, inplace=True)

    tr_run_feat = gender_age_train['n_run'].values.reshape(-1, 1)
    tr_app_feat = gender_age_train['n_app'].values.reshape(-1, 1)

    te_run_feat = gender_age_test['n_run'].values.reshape(-1, 1)
    te_app_feat = gender_age_test['n_app'].values.reshape(-1, 1)

    # 3.4 合並所有特征
    tr_feat = np.hstack((tr_brand_feat.toarray(), tr_model_feat.toarray(), tr_run_feat, tr_app_feat))
    te_feat = np.hstack((te_brand_feat.toarray(), te_model_feat.toarray(), te_run_feat, te_app_feat))
    print('特征提取結束')
    print('每個樣本特征維度：', tr_feat.shape[1])

    # 3.5 特征范圍歸一化
    scaler = StandardScaler()
    tr_feat_scaled = scaler.fit_transform(tr_feat)
    te_feat_scaled = scaler.transform(te_feat)

    # 3.6 特征選擇
    sel = VarianceThreshold(threshold=(.8 * (1 - .8)))
    tr_feat_scaled_sel = sel.fit_transform(tr_feat_scaled)
    te_feat_scaled_sel = sel.transform(te_feat_scaled)

    # 3.7 PCA降維操作
    pca = PCA(n_components=0.95)  # 保留95%共享率的特征向量
    tr_feat_scaled_sel_pca = pca.fit_transform(tr_feat_scaled_sel)
    te_feat_scaled_sel_pca = pca.transform(te_feat_scaled_sel)
    print('特征處理結束')
    print('處理后每個樣本特征維度：', tr_feat_scaled_sel_pca.shape[1])

    # 4 為數據添加標簽
    group_label_encoder = LabelEncoder()
    group_label_encoder.fit(gender_age_train['group'].values)
    y_train = group_label_encoder.transform(gender_age_train['group'].values)
    y_test = group_label_encoder.transform(gender_age_test['group'].values)

    # 5. 訓練模型
    # 5.1 邏輯回歸模型
    print('訓練邏輯回歸模型...')
    lr_param_grid = [
        {'C': [1e-3, 1e-2, 1e-1, 1, 10, 100]}
    ]
    lr_model = LogisticRegression()
    best_lr_model = get_best_model(lr_model,
                                   tr_feat_scaled_sel_pca, y_train,
                                   lr_param_grid, cv=3)
    y_pred_lr = best_lr_model.predict_proba(te_feat_scaled_sel_pca)

    # 5.2 SVM
    print('訓練SVM模型...')
    svm_param_grid = [
        {'C': [1e-2, 1e-1, 1, 10, 100], 'gamma': [0.001, 0.0001], 'kernel': ['rbf']},
    ]

    # 設置probability=True用於輸出預測概率
    svm_model = svm.SVC(probability=True)
    best_svm_model = get_best_model(svm_model,
                                    tr_feat_scaled_sel_pca, y_train,
                                    svm_param_grid, cv=3)
    y_pred_svm = best_svm_model.predict_proba(te_feat_scaled_sel_pca)

    # 6. 查看結果
    print('邏輯回歸模型 logloss:', log_loss(y_test, y_pred_lr))
    print('SVM logloss:', log_loss(y_test, y_pred_svm))


if __name__ == '__main__':
    run_main()

 

ml_tools.py

# -*- coding: utf-8 -*-

from sklearn.model_selection import GridSearchCV


def get_best_model(model, X_train, y_train, params, cv=5):
    """
        交叉驗證獲取最優模型
        默認5折交叉驗證
    """
    clf = GridSearchCV(model, params, cv=cv)
    clf.fit(X_train, y_train)
    return clf.best_estimator_

pd_tools.py

# -*- coding: utf-8 -*-

import pandas as pd
import math


def split_train_test(df_data, size=0.8):
    """
        分割訓練集和測試集
    """
    # 為保證每個類中的數據能在訓練集中和測試集中的比例相同，所以需要依次對每個類進行處理
    df_train = pd.DataFrame()
    df_test = pd.DataFrame()

    labels = df_data['group'].unique().tolist()
    for label in labels:
        # 找出group的記錄
        df_w_label = df_data[df_data['group'] == label]
        # 重新設置索引，保證每個類的記錄是從0開始索引，方便之后的拆分
        df_w_label = df_w_label.reset_index()

        # 默認按80%訓練集，20%測試集分割
        # 這里為了簡化操作，取前80%放到訓練集中，后20%放到測試集中
        # 當然也可以隨機拆分80%，20%（嘗試實現下DataFrame中的隨機拆分）

        # 該類數據的行數
        n_lines = df_w_label.shape[0]
        split_line_no = math.floor(n_lines * size)
        text_df_w_label_train = df_w_label.iloc[:split_line_no, :]
        text_df_w_label_test = df_w_label.iloc[split_line_no:, :]

        # 放入整體訓練集，測試集中
        df_train = df_train.append(text_df_w_label_train)
        df_test = df_test.append(text_df_w_label_test)

    df_train = df_train.reset_index()
    df_test = df_test.reset_index()
    return df_train, df_test


def get_part_data(df_data, percent=1):
    """
        從df_data中按percent選取部分數據
    """
    df_result = pd.DataFrame()
    grouped = df_data.groupby('group')
    for group_name, group in grouped:
        n_group_size = group.shape[0]
        n_part_size = math.floor(n_group_size * percent)
        part_df = group.iloc[:n_part_size, :]
        df_result = df_result.append(part_df)

    return df_result

dataset下載地址
鏈接：http://pan.baidu.com/s/1dE7D0bf 
密碼：yapd