2.【數據挖掘】二手車交易價格預測大賽-探索性數據分析（EDA）

2.1 什麼是探索性數據分析

探索性數據分析(EDA ，Exploratory Data Analysis)，是指對已有的數據（特別是調查或觀察得來的原始數據）在儘量少的先驗假定下進行探索，通過作圖、製表、方程擬合、計算特徵量等手段探索數據的結構和規律的一種數據分析方法。

2.2 本案例的探索性分析流程

實驗工具：Jupyter Notebook

（1）導入各種數據科學庫以及可視化庫

#coding:utf-8
import warnings
warnings.filterwarnings('ignore') # 導入warnings包，利用過濾器來實現忽略警告語句

import numpy as np
import pandas as pd 
import matplotlib.pyplot as plt
import seaborn as sns
import missingno as msno

（2）載入數據

Train_data = pd.read_csv('data/train.csv', sep=' ')
Test_data = pd.read_csv('data/testA.csv', sep=' ')

（3）查看數據基本情況

Train_data.head().append(Train_data.tail())  # 簡要查看前五行和後五行；
Train_data.shape 

Test_data.head().append(Test_data.tail())
Test_data.shape

Train_data.describe()
Test_data.describe()

Train_data.info()  # 查看數據的基本信息
Test_data.info()

（4）判斷數據異常情況

缺失值總數統計

Train_data.isnull().sum()  # 缺失值總數統計
Test_data.isnull().sum()

nan可視化

missing = Train_data.isnull().sum()  
missing = missing[missing > 0]
missing.sort_values(inplace=True)
missing.plot.bar()

msno.matrix(Train_data.sample(250))
msno.bar(Train_data.sample(1000))

msno.matrix(Test_data.sample(250))
msno.bar(Test_data.sample(1000))

查看異常值

通過以上返回結果知notRepairedDamage列需要糾正。

Train_data['notRepairedDamage'].value_counts()  # 對該列的值進行統計
Train_data['notRepairedDamage'].replace('-', np.nan, inplace=True)  # 對未知值’-‘用nan表示
Train_data['notRepairedDamage'].value_counts()  # 查看修改後結果

Test_data['notRepairedDamage'].value_counts()
Test_data['notRepairedDamage'].replace('-', np.nan, inplace=True)

清理嚴重傾斜的數據

Test_data['seller'].value_counts()
Train_data['offerType'].value_counts()

del Train_data['seller']
del Train_data['offerType']

del Test_data['seller']
del Test_data['offerType']

（5）瞭解預測值分佈情況

Train_data['price']  # 數據price列
Train_data['price'].value_counts()

總體分佈情況

import scipy.stats as st

y = Train_data['price']
plt.figure(1); plt.title('Johnson SU')
sns.distplot(y, kde=False, fit=st.johnsonsu)  #無界約翰遜分佈
plt.figure(2); plt.title('Normal')
sns.distplot(y, kde=False, fit=st.norm)
plt.figure(3); plt.title('Log Normal')
sns.distplot(y, kde=False, fit=st.lognorm)

查看skewness and kurtosis,即偏度與峯值

sns.distplot(Train_data['price']);
print("Skewness: %f" % Train_data['price'].skew())
print("Kurtosis: %f" % Train_data['price'].kurt())

Train_data.skew(), Train_data.kurt()
sns.distplot(Train_data.skew(),color='blue',axlabel ='Skewness')
sns.distplot(Train_data.kurt(),color='blue',axlabel ='Skewness')

查看預測值具體頻數

plt.hist(Train_data['price'], orientation = 'vertical',histtype = 'bar', color ='red')
plt.show()

（6）特徵分析

特徵分爲類別特徵和數字特徵

Y_train = Train_data['price']

numeric_features = ['power', 'kilometer', 'v_0', 'v_1', 'v_2', 'v_3', 
                    'v_4', 'v_5', 'v_6', 'v_7', 'v_8', 'v_9', 'v_10', 'v_11', 'v_12', 'v_13','v_14' ]
categorical_features = ['name', 'model', 'brand', 'bodyType', 'fuelType',
                        'gearbox', 'notRepairedDamage', 'regionCode',]
                        
                        
for cat_fea in categorical_features:
    print(cat_fea + "的特徵分佈如下：")
    print("{}特徵有個{}不同的值".format(cat_fea, Train_data[cat_fea].nunique()))
    print(Train_data[cat_fea].value_counts())

1）數字特徵分析

numeric_features.append('price')
Train_data.head()

相關性分析

price_numeric = Train_data[numeric_features]
correlation = price_numeric.corr()
print(correlation['price'].sort_values(ascending = False),'\n')

f , ax = plt.subplots(figsize = (7, 7))
plt.title('Correlation of Numeric Features with Price',y=1,size=16)
sns.heatmap(correlation,square = True,  vmax=0.8)

del price_numeric['price']

查看幾個特徵的偏度與峯值

for col in numeric_features:
    print('{:15}'.format(col), 
          'Skewness: {:05.2f}'.format(Train_data[col].skew()) , 
          '   ' ,
          'Kurtosis: {:06.2f}'.format(Train_data[col].kurt())  
         )

每個數字特徵分佈可視化

f = pd.melt(Train_data, value_vars=numeric_features)
g = sns.FacetGrid(f, col="variable",  col_wrap=2, sharex=False, sharey=False)
g = g.map(sns.distplot, "value")

數字特徵相互之間的關係可視化

sns.set()
columns = ['price', 'v_12', 'v_8' , 'v_0', 'power', 'v_5',  'v_2', 'v_6', 'v_1', 'v_14']
sns.pairplot(Train_data[columns],size = 2 ,kind ='scatter',diag_kind='kde')
plt.show()

多變量互相迴歸關係可視化

fig, ((ax1, ax2), (ax3, ax4), (ax5, ax6), (ax7, ax8), (ax9, ax10))= plt.subplots(nrows=5, ncols=2, figsize=(24, 20))

v_12_scatter_plot = pd.concat([Y_train,Train_data['v_12']],axis = 1)
sns.regplot(x='v_12',y = 'price', data = v_12_scatter_plot,scatter= True, fit_reg=True, ax=ax1)

v_8_scatter_plot = pd.concat([Y_train,Train_data['v_8']],axis = 1)
sns.regplot(x='v_8',y = 'price',data = v_8_scatter_plot,scatter= True, fit_reg=True, ax=ax2)

v_0_scatter_plot = pd.concat([Y_train,Train_data['v_0']],axis = 1)
sns.regplot(x='v_0',y = 'price',data = v_0_scatter_plot,scatter= True, fit_reg=True, ax=ax3)

power_scatter_plot = pd.concat([Y_train,Train_data['power']],axis = 1)
sns.regplot(x='power',y = 'price',data = power_scatter_plot,scatter= True, fit_reg=True, ax=ax4)

v_5_scatter_plot = pd.concat([Y_train,Train_data['v_5']],axis = 1)
sns.regplot(x='v_5',y = 'price',data = v_5_scatter_plot,scatter= True, fit_reg=True, ax=ax5)

v_2_scatter_plot = pd.concat([Y_train,Train_data['v_2']],axis = 1)
sns.regplot(x='v_2',y = 'price',data = v_2_scatter_plot,scatter= True, fit_reg=True, ax=ax6)

v_6_scatter_plot = pd.concat([Y_train,Train_data['v_6']],axis = 1)
sns.regplot(x='v_6',y = 'price',data = v_6_scatter_plot,scatter= True, fit_reg=True, ax=ax7)

v_1_scatter_plot = pd.concat([Y_train,Train_data['v_1']],axis = 1)
sns.regplot(x='v_1',y = 'price',data = v_1_scatter_plot,scatter= True, fit_reg=True, ax=ax8)

v_14_scatter_plot = pd.concat([Y_train,Train_data['v_14']],axis = 1)
sns.regplot(x='v_14',y = 'price',data = v_14_scatter_plot,scatter= True, fit_reg=True, ax=ax9)

v_13_scatter_plot = pd.concat([Y_train,Train_data['v_13']],axis = 1)
sns.regplot(x='v_13',y = 'price',data = v_13_scatter_plot,scatter= True, fit_reg=True, ax=ax10)

2）類別特徵分析

unique分佈

for fea in categorical_features:
    print(Train_data[fea].nunique())

類型特徵箱圖可視化

categorical_features = ['model',
 'brand',
 'bodyType',
 'fuelType',
 'gearbox',
 'notRepairedDamage']
for c in categorical_features:
    Train_data[c] = Train_data[c].astype('category')
    if Train_data[c].isnull().any():
        Train_data[c] = Train_data[c].cat.add_categories(['MISSING'])
        Train_data[c] = Train_data[c].fillna('MISSING')

def boxplot(x, y, **kwargs):
    sns.boxplot(x=x, y=y)
    x=plt.xticks(rotation=90)

f = pd.melt(Train_data, id_vars=['price'], value_vars=categorical_features)
g = sns.FacetGrid(f, col="variable",  col_wrap=2, sharex=False, sharey=False, size=5)
g = g.map(boxplot, "value", "price")

類別特徵的小提琴圖可視化

catg_list = categorical_features
target = 'price'
for catg in catg_list :
    sns.violinplot(x=catg, y=target, data=Train_data)
    plt.show()

類別特徵的柱形圖可視化

def bar_plot(x, y, **kwargs):
    sns.barplot(x=x, y=y)
    x=plt.xticks(rotation=90)

f = pd.melt(Train_data, id_vars=['price'], value_vars=categorical_features)
g = sns.FacetGrid(f, col="variable",  col_wrap=2, sharex=False, sharey=False, size=5)
g = g.map(bar_plot, "value", "price")

類別特徵的每個類別頻數可視化

def count_plot(x,  **kwargs):
    sns.countplot(x=x)
    x=plt.xticks(rotation=90)

f = pd.melt(Train_data,  value_vars=categorical_features)
g = sns.FacetGrid(f, col="variable",  col_wrap=2, sharex=False, sharey=False, size=5)
g = g.map(count_plot, "value")

（6）生成數據報告

import pandas_profiling

pfr = pandas_profiling.ProfileReport(Train_data)
pfr.to_file("./example.html")

2.3 附錄

遇到的問題以及解決方法：

Q1:

安裝pandas-profiling庫

A1:

終端輸入pip install pandas-profiling[notebook,html]

參考鏈接：pandas-profiling

Q2:

返回錯誤：ERROR: Cannot uninstall 'wrapt'. It is a distutils installed project and thus we cannot accurately determine which files belong to it which would lead to only a partial uninstall.

A2:

終端輸入 pip install -U --ignore-installed wrapt enum34 simplejson netaddr

Q3:

返回錯誤：ImportError: Numpy version 1.16.0 or later must be installed to use Astropy.

[外鏈圖片轉存失敗,源站可能有防盜鏈機制,建議將圖片保存下來直接上傳(img-pyXftmV9-1585043230056)(image/image-20200324161048300.png)]

A3:

輸入numpy.version.version查看中numpy的版本；
（解決後更新）

【數據挖掘】二手車交易價格預測大賽-探索性數據分析（EDA）

2.【數據挖掘】二手車交易價格預測大賽-探索性數據分析（EDA）

2.1 什麼是探索性數據分析

2.2 本案例的探索性分析流程

（1）導入各種數據科學庫以及可視化庫

（2）載入數據

（3）查看數據基本情況

（4）判斷數據異常情況

（5）瞭解預測值分佈情況

（6）特徵分析

1）數字特徵分析

2）類別特徵分析

（6）生成數據報告

2.3 附錄

遇到的問題以及解決方法：

【數據挖掘】二手車交易價格預測大賽-賽題理解

【數據挖掘】模型融合

【數據挖掘】（三）特徵工程

【SAR系列】（一）合成孔徑雷達基礎

【數據挖掘】二手車交易價格預測大賽-建模與調參

Mac下配置sublime實現LaTeX

https://yachay.unat.edu.pe/blog/index.php?comment_area=format_blog&comment_component=blog&comment_co

linux以太網驅動總結