数据挖掘实战--二手车交易价格预测（三）模型训练和预测

异常值分析与处理

test_df = pd.read_csv('D:/DataMining/Test Data/used_car_testA_20200313.csv', sep=' ')
#将price变换为正态分布
train_df['price'] = np.log1p(train_df['price'])
# 删除部分异常值
train_df.drop(train_df[train_df['price'] < 2].index, inplace=True)
# 超过边界部分进行限制
train_df['power'] = train_df['power'].map(lambda x: 600 if x>600 else x)
test_df['power'] = test_df['power'].map(lambda x: 600 if x>600 else x)
# 整合训练集测试集以便后续特征工程
all_features = pd.concat([train_df, test_df], sort=False).reset_index(drop=True)

填充缺失值

将存在空值的部分填充为均值

def fill_missing(df):
    df['fuelType'] = df['fuelType'].fillna(train_df['fuelType'].mean())
    df['gearbox'] = df['gearbox'].fillna(train_df['gearbox'].mean())
    df['bodyType'] = df['bodyType'].fillna(train_df['bodyType'].mean())
    df['model'] = df['model'].fillna(train_df['model'].mean())
    return df

all_features = fill_missing(all_features)

数据类型转换

对一些分类特征存储成数值需要进行转化为字符型数值

def data_astype(df):
    # string
    df['SaleID'] = df['SaleID'].astype(int).astype(str)
    df['name'] = df['name'].astype(int).astype(str)
    df['model'] = df['model'].astype(str)
    df['brand'] = df['brand'].astype(str)
    df['bodyType'] = df['bodyType'].astype(str)
    df['fuelType'] = df['fuelType'].astype(str)
    df['gearbox'] = df['gearbox'].astype(str)
    df['notRepairedDamage'] = df['notRepairedDamage'].astype(str)
    df['regionCode'] = df['regionCode'].astype(int).astype(str)
    df['seller'] = df['seller'].astype(int).astype(str)
    df['offerType'] = df['offerType'].astype(int).astype(str)

    return df
    
all_features = data_astype(all_features)

提取年份和月份

# 提取年份
all_features['regYear'] = all_features['regDate'].map(lambda x:int(str(x)[:4]))
all_features['createYear'] = all_features['creatDate'].map(lambda x:int(str(x)[:4]))
# 提取月份
all_features['regMonth'] = all_features['regDate'].map(lambda x:int(str(x)[4:6]))
all_features['createMonth'] = all_features['creatDate'].map(lambda x:int(str(x)[4:6]))

# 计算上线日期与注册日期想差月份数
all_features['months'] = (all_features['createYear']-all_features['regYear'])*12+(all_features['createMonth']-all_features['regMonth'])
all_features['years'] = all_features['months'] / 12

# 查看月份数统计值
all_features['months'].describe()
# 月份数分布
all_features['months'].hist()

编码分类变量

删除一些不要的特征。

all_features = all_features.drop(['SaleID', 'name', 'regDate', 'model', 'seller',
                                  'offerType', 'creatDate', 'regYear', 'regionCode',
                                  'createYear', 'regMonth', 'createMonth', 'months'], axis=1)

查看剩余的特征类型

all_features.dtypes

创建相关性组合，为之后的删除相关性高的变量做准备

corr = all_features.corr()
#创建相关性系数组合
feature_group = list(itertools.combinations(corr.columns, 2))