Python數據分析與挖掘----收入的預測分析

Python數據分析與挖掘之收入的預測分析

數據集形式

# 導入第三方包
import pandas as pd
import numpy as np
import seaborn as sns
# 導入繪圖模塊
import matplotlib.pyplot as plt
# 導入模型評估模塊
from sklearn import metrics
# 導入網格搜索法的函數
from sklearn.model_selection import GridSearchCV
# 數據讀取
income = pd.read_excel(r'C:\Users\Administrator.SKY-20180518VHY\Desktop\shujufenxi\第2章 從收入預測分析開始\income.xlsx')

# 查看數據集是否存在缺失值
print(income.apply(lambda x:np.sum(x.isnull())))

在此處可見workclass，occupation，native-country處值有缺失

缺失值處理

刪除法：若確實比例非常小刪除法較爲合理。
替換法：若缺失爲離散型考慮用衆數替換，數值型，則考慮用均值或中位數替換缺失值
插補法：基於未缺失的變量預測缺失變量的值，如常見的迴歸插補法，多重插補法，拉格朗日插補法等
此處用衆數進行替換

income.fillna(value = {'workclass':income.workclass.mode()[0],
                              'occupation':income.occupation.mode()[0],
                              'native-country':income['native-country'].mode()[0]}, inplace = True)
print(income.head())#輸出前五行

# 數據的探索性分析、數值型的統計描述
print(income.describe())

# 數據的探索性分析、離散型的統計描述
print(income.describe(include =[ 'object']))

# 設置繪圖風格
plt.style.use('ggplot')
# 設置多圖形的組合
fig, axes = plt.subplots(2, 1)
# 繪製不同收入水平下的年齡核密度圖，    針對數值型
income['age'][income.income == ' <=50K'].plot(kind = 'kde', label = '<=50K', ax = axes[0], legend = True, linestyle = '-')
income['age'][income.income == ' >50K'].plot(kind = 'kde', label = '>50K', ax = axes[0], legend = True, linestyle = '--')
# 繪製不同收入水平下的周工作小時數和密度圖
income['hours-per-week'][income.income == ' <=50K'].plot(kind = 'kde', label = '<=50K', ax = axes[1], legend = True, linestyle = '-')
income['hours-per-week'][income.income == ' >50K'].plot(kind = 'kde', label = '>50K', ax = axes[1], legend = True, linestyle = '--')
# 顯示圖形
plt.show()

此圖是描繪僅以被調查居民的年齡和每週工作小時爲例，繪製各自的分佈形狀圖

# 構造不同收入水平下各種族人數的數據    針對離散型
race = pd.DataFrame(income.groupby(by = ['race','income']).aggregate(np.size).loc[:,'age'])
# 重設行索引
race = race.reset_index()
# 變量重命名
race.rename(columns={'age':'counts'}, inplace=True)
# 排序
race.sort_values(by = ['race','counts'], ascending=False, inplace=True)

# 構造不同收入水平下各家庭關係人數的數據
relationship = pd.DataFrame(income.groupby(by = ['relationship','income']).aggregate(np.size).loc[:,'age'])
relationship = relationship.reset_index()
relationship.rename(columns={'age':'counts'}, inplace=True)
relationship.sort_values(by = ['relationship','counts'], ascending=False, inplace=True)

# 設置圖框比例，並繪圖
plt.figure(figsize=(9,5))
sns.barplot(x="race", y="counts", hue = 'income', data=race)
plt.show()

plt.figure(figsize=(9,5))
sns.barplot(x="relationship", y="counts", hue = 'income', data=relationship)
plt.show()

# 離散變量的重編碼
for feature in income.columns:
    if income[feature].dtype == 'object':
        income[feature] = pd.Categorical(income[feature]).codes
print(income.head())

# 刪除變量
income.drop(['education','fnlwgt'], axis = 1, inplace = True)
print(income.head())

# 數據拆分
from sklearn.model_selection import train_test_split

X_train, X_test, y_train, y_test = train_test_split(income.loc[:,'age':'native-country'], 
                                                    income['income'], train_size = 0.75, 
                                                    random_state = 1234)
print('訓練數據集共有%d條觀測' %X_train.shape[0])
print('測試數據集共有%d條觀測' %X_test.shape[0])

# 導入k近鄰模型的類
from sklearn.neighbors import KNeighborsClassifier
# 構建k近鄰模型
kn = KNeighborsClassifier()
kn.fit(X_train, y_train)
print(kn)

# 預測測試集
kn_pred = kn.predict(X_test)
print(pd.crosstab(kn_pred, y_test))

# 模型得分
print('模型在訓練集上的準確率%f' %kn.score(X_train,y_train))
print('模型在測試集上的準確率%f' %kn.score(X_test,y_test))

# 計算ROC曲線的x軸和y軸數據
fpr, tpr, _ = metrics.roc_curve(y_test, kn.predict_proba(X_test)[:,1])
# 繪製ROC曲線
plt.plot(fpr, tpr, linestyle = 'solid', color = 'red')
# 添加陰影
plt.stackplot(fpr, tpr, color = 'steelblue')
# 繪製參考線
plt.plot([0,1],[0,1], linestyle = 'dashed', color = 'black')
# 往圖中添加文本
plt.text(0.6,0.4,'AUC=%.3f' % metrics.auc(fpr,tpr), fontdict = dict(size = 18))
plt.show()

# 導入GBDT模型的類
from sklearn.ensemble import GradientBoostingClassifier
# 構建GBDT模型
gbdt = GradientBoostingClassifier()
gbdt.fit(X_train, y_train)
print(gbdt)

# 預測測試集
gbdt_pred = gbdt.predict(X_test)
print(pd.crosstab(gbdt_pred, y_test))

# 模型得分
print('模型在訓練集上的準確率%f' %gbdt.score(X_train,y_train))
print('模型在測試集上的準確率%f' %gbdt.score(X_test,y_test))

# 繪製ROC曲線
fpr, tpr, _ = metrics.roc_curve(y_test, gbdt.predict_proba(X_test)[:,1])
plt.plot(fpr, tpr, linestyle = 'solid', color = 'red')
plt.stackplot(fpr, tpr, color = 'steelblue')
plt.plot([0,1],[0,1], linestyle = 'dashed', color = 'black')
plt.text(0.6,0.4,'AUC=%.3f' % metrics.auc(fpr,tpr), fontdict = dict(size = 18))
plt.show()

# K近鄰模型的網格搜索法
# 導入網格搜索法的函數
from sklearn.model_selection import GridSearchCV
# 選擇不同的參數
k_options = list(range(1,12))
parameters = {'n_neighbors':k_options}
# 搜索不同的K值
grid_kn = GridSearchCV(estimator = KNeighborsClassifier(), param_grid = parameters, cv=10, scoring='accuracy', verbose=0)
grid_kn.fit(X_train, y_train)
print(grid_kn)
# 結果輸出
print(grid_kn.grid_scores_, grid_kn.best_params_, grid_kn.best_score_) 

# 預測測試集
grid_kn_pred = grid_kn.predict(X_test)
print(pd.crosstab(grid_kn_pred, y_test))

# 模型得分
print('模型在訓練集上的準確率%f' %grid_kn.score(X_train,y_train))
print('模型在測試集上的準確率%f' %grid_kn.score(X_test,y_test))

# 繪製ROC曲線
fpr, tpr, _ = metrics.roc_curve(y_test, grid_kn.predict_proba(X_test)[:,1])
plt.plot(fpr, tpr, linestyle = 'solid', color = 'red')
plt.stackplot(fpr, tpr, color = 'steelblue')
plt.plot([0,1],[0,1], linestyle = 'dashed', color = 'black')
plt.text(0.6,0.4,'AUC=%.3f' % metrics.auc(fpr,tpr), fontdict = dict(size = 18))
plt.show()



# GBDT模型的網格搜索法
# 選擇不同的參數
learning_rate_options = [0.01,0.05,0.1]
max_depth_options = [3,5,7,9]
n_estimators_options = [100,300,500]
parameters = {'learning_rate':learning_rate_options,'max_depth':max_depth_options,'n_estimators':n_estimators_options}

grid_gbdt = GridSearchCV(estimator = GradientBoostingClassifier(), param_grid = parameters, cv=10, scoring='accuracy')
grid_gbdt.fit(X_train, y_train)

# 結果輸出
grid_gbdt.grid_scores_, grid_gbdt.best_params_, grid_gbdt.best_score_  


# 預測測試集
grid_gbdt_pred = grid_gbdt.predict(X_test)
print(pd.crosstab(grid_gbdt_pred, y_test))

# 模型得分
print('模型在訓練集上的準確率%f' %grid_gbdt.score(X_train,y_train))
print('模型在測試集上的準確率%f' %grid_gbdt.score(X_test,y_test))

# 繪製ROC曲線
fpr, tpr, _ = metrics.roc_curve(y_test, grid_gbdt.predict_proba(X_test)[:,1])
plt.plot(fpr, tpr, linestyle = 'solid', color = 'red')
plt.stackplot(fpr, tpr, color = 'steelblue')
plt.plot([0,1],[0,1], linestyle = 'dashed', color = 'black')
plt.text(0.6,0.4,'AUC=%.3f' % metrics.auc(fpr,tpr), fontdict = dict(size = 18))
plt.show()

注：來源劉順祥《從零開始學Python數據分析與挖掘》，版權歸原作者所有，僅供學習使用，不用於商業用途，如有侵權請留言聯繫刪除，感謝合作。