數據挖掘之京東購買意向預測

一. 數據檢查

利用pd.Merge連接兩個數據, 觀察數據是否減少

1.1 檢查用戶是否一致

import pandas as pd
def user_action_check():
    df_user = pd.read_csv(r'data/JData_User.csv', encoding = 'gbk')
    df_sku = df_user.loc[:, 'user_id'].to_frame()
    df_month2 = pd.read_csv(r'data\JData_Action_201602.csv', encoding = 'gbk')
    print('Is action of Feb. from User file? ', len(df_month2) == len(pd.merge(df_sku, df_month2)))
    df_month3 = pd.read_csv(r'data\JData_Action_201603.csv', encoding = 'gbk')
    print('Is action of Mar. from User file? ', len(df_month3) == len(pd.merge(df_sku, df_month3)))
    df_month4 = pd.read_csv(r'data\JData_Action_201604.csv', encoding = 'gbk')
    print('Is action of Apr. from User file? ', len(df_month4) == len(pd.merge(df_sku, df_month4)))
    
user_action_check()

Is action of Feb. from User file? True
Is action of Mar. from User file? True
Is action of Apr. from User file? True

1.2 檢查是否有重複記錄

def deduplicate(filepath, filename, newpath):
    df_file = pd.read_csv(filepath, encoding = 'gbk')
    before = df_file.shape[0]
    df_file.drop_duplicates(inplace = True)
    after = df_file.shape[0]
    n_dup = before - after
    print('No. of duplicate records for ' + filename + ' is: ' + str(n_dup))
    if n_dup != 0:
        df_file.to_csv(newpath, index = None)
    else:
        print('No duplicate records in ' + filename)
        
deduplicate(r'data\JData_Action_201603.csv', 'Mar. action', '京東/JData_Action_201603_dedup.csv')
deduplicate(r'data\JData_Action_201604.csv', 'Feb. action', '京東/JData_Action_201604_dedup.csv')
deduplicate(r'data\JData_Comment.csv', 'Comment', '京東/JData_Comment_dedup.csv')
deduplicate(r'data\JData_Product.csv', 'Product', '京東/JData_Product_dedup.csv')
deduplicate(r'data\JData_User.csv', 'User', '京東/JData_User_dedup.csv')

df_month2 = pd.read_csv(r'Fdata\JData_Action_201602.csv', encoding = 'gbk')
IsDuplicated = df_month2.duplicated()
df_d = df_month2[IsDuplicated]
df_d.groupby('type').count()
#發現重複數據大多數都是由於瀏覽（1），或者點擊(6)產生

1.3 檢查註冊時間

df_user = pd.read_csv(r'data\JData_User.csv', encoding = 'gbk')
df_user['user_reg_tm'] = pd.to_datetime(df_user['user_reg_tm'])
df_user.loc[df_user.user_reg_tm >= '2016-4-15']

df_month = pd.read_csv(r'data\JData_Action_201604.csv')
df_month['time'] = pd.to_datetime(df_month['time'])
df_month.loc[df_month.time >= '2016-4-16']

結論：說明用戶沒有異常操作數據，所以這一批用戶不刪除

1.3 INT類型轉換

df_month = pd.read_csv(r'data\JData_Action_201602.csv', encoding = 'gbk')
df_month['user_id'] = df_month['user_id'].apply(lambda x: int(x))
print(df_month['user_id'].dtype)
df_month.to_csv(r'京東\JData_Action_201602.csv', index = None)

df_month = pd.read_csv(r'data\JData_Action_201603.csv', encoding = 'gbk')
df_month['user_id'] = df_month['user_id'].apply(lambda x: int(x))
print(df_month['user_id'].dtype)
df_month.to_csv(r'京東\JData_Action_201603.csv', index = None)

df_month = pd.read_csv(r'data\JData_Action_201604.csv', encoding = 'gbk')
df_month['user_id'] = df_month['user_id'].apply(lambda x: int(x))
print(df_month['user_id'].dtype)
df_month.to_csv(r'京東\JData_Action_201604.csv', index = None)

二. 構建特徵表單

2.1 構建User_table

#定義文件名
ACTION_201602_FILE = "京東/JData_Action_201602.csv"
ACTION_201603_FILE = "京東/JData_Action_201603.csv"
ACTION_201604_FILE = "京東/JData_Action_201604.csv"
COMMENT_FILE = "京東/JData_Comment.csv"
PRODUCT_FILE = "京東/JData_Product.csv"
USER_FILE = "京東/JData_User.csv"
USER_TABLE_FILE = "京東/User_table.csv"
ITEM_TABLE_FILE = "京東/Item_table.csv"

import pandas as pd
import numpy as np
from collections import Counter

# 功能函數: 對每一個user分組的數據進行統計
def add_type_count(group):
    behavior_type = group.type.astype(int)
    type_cnt = Counter(behavior_type)
    group['browse_num'] = type_cnt[1]
    group['addcart_num'] = type_cnt[2]
    group['delcart_num'] = type_cnt[3]
    group['buy_num'] = type_cnt[4]
    group['favor_num'] = type_cnt[5]
    group['click_num'] = type_cnt[6]
    return group[['user_id', 'browse_num', 'addcart_num', 'delcart_num', 'buy_num', 'favor_num', 'click_num']]

# 由於用戶行爲數據量較大,一次性讀入可能造成內存錯誤(Memory Error),因而使用pandas的分塊(chunk)讀取.
def get_from_action_data(fname, chunk_size = 50000):
    reader = pd.read_csv(fname, header = 0, iterator = True, encoding = 'gbk')
    chunks = []
    loop = True
    while loop:
        try:
            chunk = reader.get_chunk(chunk_size)[['user_id', 'type']]
            chunks.append(chunk)
        except StopIteration:
            loop = False
            print('Iteration is stopped')
    df_ac = pd.concat(chunks, ignore_index = True)
    df_ac = df_ac.groupby(['user_id'], as_index = False).apply(add_type_count)
    df_ac = df_ac.drop_duplicates('user_id')
    return df_ac

# 將各個action數據的統計量進行聚合
def merge_action_data():
    df_ac = []
    df_ac.append(get_from_action_data(fname = ACTION_201602_FILE))
    df_ac.append(get_from_action_data(fname = ACTION_201603_FILE))
    df_ac.append(get_from_action_data(fname = ACTION_201604_FILE))
    df_ac = pd.concat(df_ac, ignore_index = True)
    # 用戶在不同action表中統計量求和
    df_ac = df_ac.groupby(['user_id'], as_index = False).sum()
    # 構造轉化率字段
    df_ac['buy_addcart_ratio'] = df_ac['buy_num'] / df_ac['addcart_num']
    df_ac['buy_browse_ratio'] = df_ac['buy_num'] / df_ac['browse_num']
    df_ac['buy_click_ratio'] = df_ac['buy_num'] / df_ac['click_num']
    df_ac['buy_favor_ratio'] = df_ac['buy_num'] / df_ac['favor_num']
    # 將大於１的轉化率字段置爲１(100%)
    df_ac.loc[df_ac['buy_addcart_ratio'] > 1, 'buy_addcart_ratio'] = 1.
    df_ac.loc[df_ac['buy_browse_ratio'] > 1, 'buy_browse_ratio'] = 1.
    df_ac.loc[df_ac['buy_click_ratio'] > 1, 'buy_click_ratio'] = 1.
    df_ac.loc[df_ac['buy_favor_ratio'] > 1, 'buy_favor_ratio'] = 1.
    return df_ac

# 從FJData_User表中抽取需要的字段
def get_from_jdata_user():
    df_user = pd.read_csv(USER_FILE, header = 0)
    df_user = df_user[['user_id', 'age', 'sex', 'user_lv_cd']]
    return df_user

user_base = get_from_jdata_user()
user_behavior = merge_action_data()

user_behavior = pd.merge(user_base, user_behavior, on = ['user_id'], how = 'left')
user_behavior.to_csv(USER_TABLE_FILE, index = False)

user_table = pd.read_csv(USER_TABLE_FILE)
print(user_table.shape)
user_table.head()

2.2 構建Item_table

# 讀取Product中商品
def get_from_jdata_product():
    df_item = pd.read_csv(PRODUCT_FILE, header = 0, encoding = 'gbk')
    return df_item

# 對每一個商品分組進行統計
def add_type_count(group):
    behavior_type = group.type.astype(int)
    type_cnt = Counter(behavior_type)
    group['browse_num'] = type_cnt[1]
    group['addcart_num'] = type_cnt[2]
    group['delcart_num'] = type_cnt[3]
    group['buy_num'] = type_cnt[4]
    group['favor_num'] = type_cnt[5]
    group['click_num'] = type_cnt[6]
    return group[['sku_id', 'browse_num', 'addcart_num', 'delcart_num', 'buy_num', 'favor_num', 'click_num']]

# 對action中的數據進行統計
def get_from_action_data(fname, chunk_size = 50000):
    reader = pd.read_csv(fname, header = 0, iterator = True)
    chunks = []
    loop = True
    while loop:
        try:
            chunk = reader.get_chunk(chunk_size)[['sku_id', 'type']]
            chunks.append(chunk)
        except StopIteration:
            loop = False
            print('Iteration is stopped')
    df_ac = pd.concat(chunks, ignore_index = True)
    df_ac = df_ac.groupby('sku_id', as_index = False).apply(add_type_count)
    df_ac = df_ac.drop_duplicates('sku_id')
    return df_ac

# 獲取評論中的商品數據,如果存在某一個商品有兩個日期的評論，取最晚的那一個
def get_from_jdata_comment():
    df_cmt = pd.read_csv(COMMENT_FILE, header = 0)
    df_cmt['dt'] = pd.to_datetime(df_cmt['dt'])
    idx = df_cmt.groupby(['sku_id'])['dt'].transform(max) == df_cmt['dt']
    df_cmt = df_cmt[idx]
    return df_cmt[['sku_id', 'comment_num', 'has_bad_comment', 'bad_comment_rate']]

# 數據合併
def merge_action_data():
    df_ac = []
    df_ac.append(get_from_action_data(fname = ACTION_201602_FILE))
    df_ac.append(get_from_action_data(fname = ACTION_201603_FILE))
    df_ac.append(get_from_action_data(fname = ACTION_201604_FILE))
    df_ac = pd.concat(df_ac, ignore_index = True)
    # 用戶在不同action表中統計量求和
    df_ac = df_ac.groupby(['sku_id'], as_index = False).sum()
    # 構造轉化率字段
    df_ac['buy_addcart_ratio'] = df_ac['buy_num'] / df_ac['addcart_num']
    df_ac['buy_browse_ratio'] = df_ac['buy_num'] / df_ac['browse_num']
    df_ac['buy_click_ratio'] = df_ac['buy_num'] / df_ac['click_num']
    df_ac['buy_favor_ratio'] = df_ac['buy_num'] / df_ac['favor_num']
    # 將大於１的轉化率字段置爲１(100%)
    df_ac.loc[df_ac['buy_addcart_ratio'] > 1, 'buy_addart_ratio'] = 1.
    df_ac.loc[df_ac['buy_browse_ratio'] > 1, 'buy_browse_ratio'] = 1.
    df_ac.loc[df_ac['buy_click_ratio'] > 1, 'buy_click_ratio'] = 1.
    df_ac.loc[df_ac['buy_favor_ratio'] > 1, 'buy_favor_ratio'] = 1.
    return df_ac

item_base = get_from_jdata_product()
item_behavior = merge_action_data()
item_comment = get_from_jdata_comment()

item_behavior = pd.merge(item_base, item_behavior, on = 'sku_id', how = 'left')
item_behavior = pd.merge(item_behavior, item_comment, on = 'sku_id', how = 'left')
item_behavior.to_csv(ITEM_TABLE_FILE, index = False)

item_tabel = pd.read_csv(ITEM_TABLE_FILE)
print(item_tabel.shape)
item_tabel.head()

三. 數據清洗

import pandas as pd
df_user = pd.read_csv('京東/User_table.csv', header = 0)
pd.options.display.float_format = '{:.3f}'.format #輸出格式設置，保留三位小數
df_user.describe()

df_user[df_user['age'].isnull()]

# 刪除沒有age,sex的用戶
delete_index = df_user[df_user['age'].isnull()].index
df_user.drop(delete_index, axis = 0, inplace = True)

# 刪除無交互記錄的用戶
df_naction = df_user[df_user['browse_num'].isnull() & df_user['addcart_num'].isnull() & df_user['delcart_num'].isnull() & df_user['buy_num'].isnull() & df_user['favor_num'].isnull() & df_user['click_num'].isnull()]
df_user.drop(df_naction.index, axis = 0, inplace = True)
print(len(df_user))

# 統計並刪除無購買記錄的用戶
df_bzero = df_user[df_user['buy_num'] == 0]
print(len(df_bzero))
df_user = df_user[df_user['buy_num'] != 0]
df_user.describe()

105177
75694

# 刪除爬蟲及惰性用戶
bindex = df_user[df_user['buy_browse_ratio'] < 0.0005].index
print(len(bindex))
df_user.drop(bindex, axis = 0, inplace = True)

cindex = df_user[df_user['buy_click_ratio'] < 0.0005].index
print(len(cindex))
df_user.drop(cindex, axis = 0, inplace = True)

df_user.describe()

df_user.to_csv('京東/User_table.csv', index = None)

四. 數據探索

4.1 一週各天購買情況

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline

ACTION_201602_FILE = "京東/JData_Action_201602.csv"
ACTION_201603_FILE = "京東/JData_Action_201603.csv"
ACTION_201604_FILE = "京東/JData_Action_201604.csv"
COMMENT_FILE = "京東/JData_Comment.csv"
PRODUCT_FILE = "京東/JData_Product.csv"
USER_FILE = "京東/JData_User.csv"
USER_TABLE_FILE = "京東/User_table.csv"
ITEM_TABLE_FILE = "京東/Item_table.csv"

# 提取購買(type=4)的行爲數據
def get_from_action_data(fname, chunk_size = 50000):
    reader = pd.read_csv(fname, header = 0, iterator = True)
    chunks = []
    loop = True
    while loop:
        try:
            chunk = reader.get_chunk(chunk_size)[['user_id', 'sku_id', 'type', 'time']]
            chunks.append(chunk)
        except StopIteration:
            loop = False
            print('Iteration is stopped')
    df_ac = pd.concat(chunks, ignore_index = True)
    df_ac = df_ac[df_ac['type'] == 4]
    return df_ac[['user_id', 'sku_id', 'time']]

df_ac = []
df_ac.append(get_from_action_data(fname = ACTION_201602_FILE))
df_ac.append(get_from_action_data(fname = ACTION_201603_FILE))
df_ac.append(get_from_action_data(fname = ACTION_201604_FILE))
df_ac = pd.concat(df_ac, ignore_index = True)
print(df_ac.dtypes)
df_ac.head()

# 將time字段轉換爲datetime類型
df_ac['time'] = pd.to_datetime(df_ac['time'])
# 使用lambda匿名函數將時間time轉換爲星期(週一爲1, 週日爲７)
df_ac['time'] = df_ac['time'].apply(lambda x: x.weekday() + 1)

# 週一到週日每天購買用戶個數
df_user = df_ac.groupby('time')['user_id'].nunique()
df_user = df_user.to_frame().reset_index()
df_user.columns = ['weekday', 'user_num']

# 週一到週日每天購買商品個數
df_item = df_ac.groupby('time')['sku_id'].nunique()
df_item = df_item.to_frame().reset_index()
df_item.columns = ['weekday', 'item_num']

# 週一到週日每天購買記錄個數
df_ui = df_ac.groupby('time', as_index = False).size()
df_ui = df_ui.to_frame().reset_index()
df_ui.columns = ['weekday', 'user_item_num']

bar_width = 0.2  # 條形寬度
opacity = 0.4    # 透明度

plt.figure(figsize = (8, 6))
plt.bar(df_user['weekday'], df_user['user_num'], bar_width, alpha = opacity, color = 'b', label = 'user')
plt.bar(df_item['weekday'] + bar_width, df_item['item_num'], bar_width, alpha = opacity, color = 'g', label = 'item')
plt.bar(df_ui['weekday'] + bar_width*2, df_ui['user_item_num'], bar_width, alpha = opacity, color = 'r', label = 'user_item')
plt.xlabel('Weekday')
plt.ylabel('Number')
plt.title('A Week Purchase Table')
plt.xticks(df_user['weekday'] + bar_width, (1, 2, 3, 4, 5, 6, 7))
plt.tight_layout()
plt.legend(prop = {'size': 10})
plt.grid()

4.2 一個月中各天購買量

4.2.1 2016年2月

df_ac = get_from_action_data(fname = ACTION_201602_FILE)
df_ac['time'] = pd.to_datetime(df_ac['time']).apply(lambda x: x.day)

df_user = df_ac.groupby('time')['user_id'].nunique()
df_user = df_user.to_frame().reset_index()
df_user.columns = ['day', 'user_num']

df_item = df_ac.groupby('time')['sku_id'].nunique()
df_item = df_item.to_frame().reset_index()
df_item.columns = ['day', 'item_num']

df_ui = df_ac.groupby('time', as_index = False).size()
df_ui = df_ui.to_frame().reset_index()
df_ui.columns = ['day', 'user_item_num']

bar_width = 0.2
opacith = 0.4
day_range = range(1, len(df_user['day']) + 1)

plt.figure(figsize = (10, 7))
plt.bar(df_user['day'], df_user['user_num'], bar_width, alpha = opacity, color = 'b', label = 'user')
plt.bar(df_item['day'] + bar_width, df_item['item_num'], bar_width, alpha = opacity, color = 'g', label = 'item')
plt.bar(df_ui['day'] + bar_width*2, df_ui['user_item_num'], bar_width, alpha = opacith, color = 'r', label = 'user_item')

plt.xlabel('Day')
plt.ylabel('Number')
plt.title('February Purchase Table')
plt.xticks(df_user['day'] + bar_width, day_range)
plt.tight_layout()
plt.legend(prop = {'size': 12})
plt.grid()

4.2.2 2016年3月

df_ac = get_from_action_data(fname = ACTION_201603_FILE)
df_ac['time'] = pd.to_datetime(df_ac['time']).apply(lambda x: x.day)

df_user = df_ac.groupby('time')['user_id'].nunique()
df_user = df_user.to_frame().reset_index()
df_user.columns = ['day', 'user_num']

df_item = df_ac.groupby('time')['sku_id'].nunique()
df_item = df_item.to_frame().reset_index()
df_item.columns = ['day', 'item_num']

df_ui = df_ac.groupby('time', as_index = False).size()
df_ui = df_ui.to_frame().reset_index()
df_ui.columns = ['day', 'user_item_num']

bar_width = 0.2
opacith = 0.4
day_range = range(1, len(df_user['day']) + 1)

plt.figure(figsize = (10, 7))
plt.bar(df_user['day'], df_user['user_num'], bar_width, alpha = opacity, color = 'b', label = 'user')
plt.bar(df_item['day'] + bar_width, df_item['item_num'], bar_width, alpha = opacity, color = 'g', label = 'item')
plt.bar(df_ui['day'] + bar_width*2, df_ui['user_item_num'], bar_width, alpha = opacith, color = 'r', label = 'user_item')

plt.xlabel('Day')
plt.ylabel('Number')
plt.title('March Purchase Table')
plt.xticks(df_user['day'] + bar_width, day_range)
plt.tight_layout()
plt.legend(prop = {'size': 12})
#plt.grid()

4.2.3 2016年4月

df_ac = get_from_action_data(fname = ACTION_201604_FILE)
df_ac['time'] = pd.to_datetime(df_ac['time']).apply(lambda x: x.day)

df_user = df_ac.groupby('time')['user_id'].nunique()
df_user = df_user.to_frame().reset_index()
df_user.columns = ['day', 'user_num']

df_item = df_ac.groupby('time')['sku_id'].nunique()
df_item = df_item.to_frame().reset_index()
df_item.columns = ['day', 'item_num']

df_ui = df_ac.groupby('time', as_index = False).size()
df_ui = df_ui.to_frame().reset_index()
df_ui.columns = ['day', 'user_item_num']

bar_width = 0.2
opacith = 0.4
day_range = range(1, len(df_user['day']) + 1)

plt.figure(figsize = (12, 8))
plt.bar(df_user['day'], df_user['user_num'], bar_width, alpha = opacity, color = 'b', label = 'user')
plt.bar(df_item['day'] + bar_width, df_item['item_num'], bar_width, alpha = opacity, color = 'g', label = 'item')
plt.bar(df_ui['day'] + bar_width*2, df_ui['user_item_num'], bar_width, alpha = opacith, color = 'r', label = 'user_item')

plt.xlabel('Day')
plt.ylabel('Number')
plt.title('March Purchase Table')
plt.xticks(df_user['day'] + bar_width, day_range)
plt.tight_layout()
plt.legend(prop = {'size': 12})
#plt.grid()

4.3 商品類別銷售統計

4.3.1 一週各商品類別銷售情況

def get_from_action_data(fname, chunk_size = 50000):
    reader = pd.read_csv(fname, header = 0, iterator = True)
    chunks = []
    loop = True
    while loop:
        try:
            chunk = reader.get_chunk(chunk_size)[['cate', 'brand', 'type', 'time']]
            chunks.append(chunk)
        except StopIteration:
            loop = False
            print('Iteration is stopped')
    df_ac = pd.concat(chunks, ignore_index = True)
    df_ac = df_ac[df_ac['type'] == 4]
    return df_ac[['cate', 'brand', 'type', 'time']]

df_ac = []
df_ac.append(get_from_action_data(fname = ACTION_201602_FILE))
df_ac.append(get_from_action_data(fname = ACTION_201603_FILE))
df_ac.append(get_from_action_data(fname = ACTION_201604_FILE))
df_ac = pd.concat(df_ac, ignore_index = True)

df_ac['time'] = pd.to_datetime(df_ac['time']).apply(lambda x: x.weekday() + 1)
print(df_ac.shape)
df_ac.head()

# 週一到週日每天購買商品類別數量統計
df_product = df_ac.groupby(['time', 'cate']).brand.count()
df_product = df_product.unstack()  # unstack() 將行旋轉到列
df_product.plot(kind = 'bar', title = 'Cate Purchase Table in a Week', figsize = (8, 6))

4.3.2 每月各類商品銷售情況（只關注商品8）

df_ac2 = get_from_action_data(fname = ACTION_201602_FILE)
df_ac2['time'] = pd.to_datetime(df_ac2['time']).apply(lambda x: x.day)

df_ac3 = get_from_action_data(fname = ACTION_201603_FILE)
df_ac3['time'] = pd.to_datetime(df_ac3['time']).apply(lambda x: x.day)

df_ac4 = get_from_action_data(fname = ACTION_201604_FILE)
df_ac4['time'] = pd.to_datetime(df_ac4['time']).apply(lambda x: x.day)

dc_cate2 = df_ac2[df_ac2['cate'] == 8]
dc_cate2 = dc_cate2.groupby('time').brand.count()
dc_cate2 = dc_cate2.to_frame().reset_index()
dc_cate2.columns = ['day', 'product_num']

dc_cate3 = df_ac3[df_ac3['cate'] == 8]
dc_cate3 = dc_cate3.groupby('time').brand.count()
dc_cate3 = dc_cate3.to_frame().reset_index()
dc_cate3.columns = ['day', 'product_num']

dc_cate4 = df_ac4[df_ac4['cate'] == 8]
dc_cate4 = dc_cate4.groupby('time').brand.count()
dc_cate4 = dc_cate4.to_frame().reset_index()
dc_cate4.columns = ['day', 'product_num']

bar_width = 0.2
opacity = 0.4
day_range = range(1, len(dc_cate3['day']) + 1)
plt.figure(figsize = (12, 8))

plt.bar(dc_cate2['day'], dc_cate2['product_num'], bar_width, alpha = opacity, color = 'b', label = 'February')
plt.bar(dc_cate3['day'] + bar_width, dc_cate3['product_num'], bar_width, alpha = opacity, color = 'r', label = 'March')
plt.bar(dc_cate4['day'] + bar_width*2, dc_cate4['product_num'], bar_width, alpha = opacity, color = 'g', label = 'April')

plt.xlabel('Day')
plt.ylabel('Number')
plt.title('Cate-8 Purchase Table')
plt.xticks(dc_cate3['day'] + bar_width, day_range)
plt.tight_layout()
plt.legend(prop = {'size': 12})

4.4 查看特定用戶對特定商品的的軌跡

def special_ui_action_data(fname, user_id, item_id, chunk_size = 50000):
    reader = pd.read_csv(fname, header = 0, iterator = True)
    chunks = []
    loop = True
    while loop:
        try:
            chunk = reader.get_chunk(chunk_size)[['user_id', 'sku_id', 'type', 'time']]
            chunks.append(chunk)
        except StopIteration:
            loop = False
            print('Iteration is stopped')
    df_ac = pd.concat(chunks, ignore_index = True)
    df_ac = df_ac[(df_ac['user_id'] == user_id) & (df_ac['sku_id'] == item_id)]
    return df_ac

def explore_user_item_via_time():
    user_id = 266079
    item_id = 138778
    df_ac = []
    df_ac.append(special_ui_action_data(ACTION_201602_FILE, user_id, item_id))
    df_ac.append(special_ui_action_data(ACTION_201603_FILE, user_id, item_id))
    df_ac.append(special_ui_action_data(ACTION_201604_FILE, user_id, item_id))
    df_ac = pd.concat(df_ac, ignore_index = False)
    print(df_ac.sort_values('time'))
    
explore_user_item_via_time()

五. 特徵工程

import pandas as pd
import numpy as np

action_02_path = '京東/JData_Action_201602.csv'
action_03_path = '京東/JData_Action_201603.csv'
action_04_path = '京東/JData_Action_201604.csv'
comment_path = '京東/JData_Comment.csv'
product_path = '京東/JData_Product.csv'
user_path = '京東/JData_User.csv'

def get_actions(action_path):
    reader = pd.read_csv(action_path, iterator = True)
    #reader[['user_id', 'sku_id', 'model_id', 'type', 'cate', 'brand']] = reader[['user_id', 'sku_id', 'model_id', 'type', 'cate', 'brand']].astype('float32')
    chunks = []
    loop = True
    while loop:
        try:
            chunk = reader.get_chunk(50000)
            chunks.append(chunk)
        except StopIteration:
            loop = False
            print('Iteration is stopped.')
    action = pd.concat(chunks, ignore_index = True)
    action[['user_id', 'sku_id', 'model_id', 'type', 'cate', 'brand']] = action[['user_id', 'sku_id', 'model_id', 'type', 'cate', 'brand']].astype('float32')
    return action

def get_all_action():
    action_02 = get_actions(action_02_path)
    action_03 = get_actions(action_03_path)
    action_04 = get_actions(action_04_path)
    actions = pd.concat([action_02, action_03, action_04])
    return actions

5.1 用戶基本特徵

from sklearn.preprocessing import LabelEncoder

def get_basic_user_feat():
    user = pd.read_csv(user_path, encoding = 'gbk')
    user.dropna(axis = 0, how = 'any', inplace = True)
    user[['sex', 'age']] = user[['sex', 'age']].astype(int)
    age_df = LabelEncoder().fit_transform(user['age'])
    age_df = pd.get_dummies(age_df, prefix = 'age')
    sex_df = pd.get_dummies(user['sex'], prefix = 'sex')
    user_lv_df = pd.get_dummies(user['user_lv_cd'], prefix = 'user_lv_cd')
    user = pd.concat([user['user_id'], age_df, sex_df, user_lv_df], axis = 1)
    return user

5.2 商品基本特徵

def get_basic_product_feat():
    product = pd.read_csv(product_path)
    attr1_df = pd.get_dummies(product['a1'], prefix = 'a1')
    attr2_df = pd.get_dummies(product['a2'], prefix = 'a2')
    attr3_df = pd.get_dummies(product['a3'], prefix = 'a3')
    product = pd.concat([product[['sku_id', 'cate', 'brand']], attr1_df, attr2_df, attr3_df], axis = 1)
    return product

5.3 評論特徵

comment_date = ["2016-02-01", "2016-02-08", "2016-02-15", "2016-02-22", "2016-02-29", "2016-03-07",
                "2016-03-14", "2016-03-21", "2016-03-28", "2016-04-04","2016-04-11", "2016-04-15"]

def get_comments_product_feat(end_date):
    comments = pd.read_csv(comment_path)
    comment_date_end = end_date
    comment_date_begin = comment_date[0]
    for date in reversed(comment_date):
        if date < comment_date_end:
            comment_date_begin = date
            break
    comments = comments[comments.dt == comment_date_begin]
    df = pd.get_dummies(comments['comment_num'], prefix = 'comment_num')
    # 爲了防止某個時間段不具備評論數爲0的情況（測試集出現過這種情況）
    for i in range(0, 5):
        if 'comment_num_' + str(i) not in df.columns:
            df['comment_num_' + str(i)] = 0
    df = df[['comment_num_0', 'comment_num_1', 'comment_num_2', 'comment_num_3', 'comment_num_4']]
    comments = pd.concat([comments, df], axis = 1)
    comments = comments[['sku_id', 'has_bad_comment', 'bad_comment_rate','comment_num_0',
                         'comment_num_1', 'comment_num_2', 'comment_num_3', 'comment_num_4']]
    return comments

5.4 行爲特徵

# 獲取某個時間段的行爲記錄
def get_time_action(start_date, end_date, all_actions):
    actions = all_actions[(all_actions.time >= start_date) & (all_actions.time < end_date)].copy()
    return actions

def get_action_feat(start_date, end_date, all_actions, i):
    actions = get_time_action(start_date, end_date, all_actions)
    actions = actions[['user_id', 'sku_id', 'cate', 'type']]
    before_date = 'action_before_%s' % i
    df = pd.get_dummies(actions['type'], prefix = before_date)
    actions = pd.concat([actions, df], axis = 1)
    
    # 分組統計，用戶-類別-商品,不同用戶對不同類別下商品的行爲計數
    actions = actions.groupby(['user_id', 'sku_id', 'cate'], as_index = False).sum()
    # 分組統計，用戶-類別，不同用戶對不同商品類別的行爲計數
    user_cate = actions.groupby(['user_id', 'cate'], as_index = False).sum()
    del user_cate['sku_id']
    del user_cate['type']
    actions = pd.merge(actions, user_cate, how = 'left', on = ['user_id', 'cate'])
    
    #本類別下其他商品點擊量
    # 前述兩種分組含有相同名稱的不同行爲的計數，系統會自動針對名稱調整添加後綴,x,y，所以這裏作差統計的是同一類別下其他商品的行爲計數
    actions[before_date + '_1.0_y'] = actions[before_date + '_1.0_y'] - actions[before_date + '_1.0_x']
    actions[before_date + '_2.0_y'] = actions[before_date + '_2.0_y'] - actions[before_date + '_2.0_x']
    actions[before_date + '_3.0_y'] = actions[before_date + '_3.0_y'] - actions[before_date + '_3.0_x']
    actions[before_date + '_4.0_y'] = actions[before_date + '_4.0_y'] - actions[before_date + '_4.0_x']
    actions[before_date + '_5.0_y'] = actions[before_date + '_5.0_y'] - actions[before_date + '_5.0_x']
    actions[before_date + '_6.0_y'] = actions[before_date + '_6.0_y'] - actions[before_date + '_6.0_x']
    
    # 統計用戶對不同類別下商品計數與該類別下商品行爲計數均值（對時間）的差值
    actions[before_date + 'minus_mean_1'] = actions[before_date + '_1.0_x'] - actions[before_date + '_1.0_x'] / i
    actions[before_date + 'minus_mean_2'] = actions[before_date + '_2.0_x'] - actions[before_date + '_2.0_x'] / i
    actions[before_date + 'minus_mean_3'] = actions[before_date + '_3.0_x'] - actions[before_date + '_3.0_x'] / i
    actions[before_date + 'minus_mean_4'] = actions[before_date + '_4.0_x'] - actions[before_date + '_4.0_x'] / i
    actions[before_date + 'minus_mean_5'] = actions[before_date + '_5.0_x'] - actions[before_date + '_5.0_x'] / i
    actions[before_date + 'minus_mean_6'] = actions[before_date + '_6.0_x'] - actions[before_date + '_6.0_x'] / i
    del actions['type']
    return actions

5.5 累積用戶特徵

分時間段用戶不同行爲的購買轉化率和均值

from datetime import datetime
from datetime import timedelta

def get_accumulate_user_feat(end_date, all_actions, day):
    start_date = datetime.strptime(end_date, '%Y-%m-%d') - timedelta(days = day)
    start_date = start_date.strftime('%Y-%m-%d')
    before_date = 'user_action_%s' % day

    actions = get_time_action(start_date, end_date, all_actions)
    df = pd.get_dummies(actions['type'], prefix = before_date)
    actions['date'] = pd.to_datetime(actions['time']).apply(lambda x: x.date())
    actions = pd.concat([actions[['user_id', 'date']], df], axis = 1)
    actions = actions.groupby('user_id', as_index = False).sum()
    
    actions[before_date + '_1_ratio'] = np.log(1 + actions[before_date + '_4.0']) - np.log(1 + actions[before_date + '_1.0'])
    actions[before_date + '_2_ratio'] = np.log(1 + actions[before_date + '_4.0']) - np.log(1 + actions[before_date + '_2.0'])
    actions[before_date + '_3_ratio'] = np.log(1 + actions[before_date + '_4.0']) - np.log(1 + actions[before_date + '_3.0'])
    actions[before_date + '_5_ratio'] = np.log(1 + actions[before_date + '_4.0']) - np.log(1 + actions[before_date + '_5.0'])
    actions[before_date + '_6_ratio'] = np.log(1 + actions[before_date + '_4.0']) - np.log(1 + actions[before_date + '_6.0'])

    actions[before_date + '_1_mean'] = actions[before_date + '_1.0'] / day
    actions[before_date + '_2_mean'] = actions[before_date + '_2.0'] / day
    actions[before_date + '_3_mean'] = actions[before_date + '_3.0'] / day
    actions[before_date + '_4_mean'] = actions[before_date + '_4.0'] / day
    actions[before_date + '_5_mean'] = actions[before_date + '_5.0'] / day
    actions[before_date + '_6_mean'] = actions[before_date + '_6.0'] / day    
    return actions

5.6 用戶近期行爲特徵

def get_recent_user_feat(end_date, all_actions):
    actions_3 = get_accumulate_user_feat(end_date, all_actions, 3)
    actions_30 = get_accumulate_user_feat(end_date, all_actions, 30)
    actions = pd.merge(actions_3, actions_30, how = 'left', on = 'user_id')
    del actions_3
    del actions_30
    
    actions['recent_action1'] = np.log(1 + actions['user_action_30_1.0'] - actions['user_action_3_1.0']) - np.log(1 + actions['user_action_30_1.0'])
    actions['recent_action2'] = np.log(1 + actions['user_action_30_2.0'] - actions['user_action_3_2.0']) - np.log(1 + actions['user_action_30_2.0'])
    actions['recent_action3'] = np.log(1 + actions['user_action_30_3.0'] - actions['user_action_3_3.0']) - np.log(1 + actions['user_action_30_3.0'])
    actions['recent_action4'] = np.log(1 + actions['user_action_30_4.0'] - actions['user_action_3_4.0']) - np.log(1 + actions['user_action_30_4.0'])
    actions['recent_action5'] = np.log(1 + actions['user_action_30_5.0'] - actions['user_action_3_5.0']) - np.log(1 + actions['user_action_30_5.0'])
    actions['recent_action6'] = np.log(1 + actions['user_action_30_6.0'] - actions['user_action_3_6.0']) - np.log(1 + actions['user_action_30_6.0'])
    return actions

# 用戶對同類別下各種商品的行爲
# 增加用戶對不同類別的交互特徵
def get_user_cate_feature(start_date, end_date, all_actions):
    actions = get_time_action(start_date, end_date, all_actions)
    actions = actions[['user_id', 'cate', 'type']]
    df = pd.get_dummies(actions['type'], prefix = 'type')
    actions = pd.concat([actions[['user_id', 'cate']], df], axis = 1)
    actions = actions.groupby(['user_id', 'cate']).sum()
    actions = actions.unstack()
    actions.columns = actions.columns.swaplevel(0, 1)
    actions.columns = ['cate_4_type1', 'cate_5_type1', 'cate_6_type1', 'cate_7_type1', 'cate_8_type1',
                       'cate_9_type1', 'cate_10_type1', 'cate_11_type1', 'cate_4_type2', 'cate_5_type2',
                       'cate_6_type2', 'cate_7_type2','cate_8_type2', 'cate_9_type2', 'cate_10_type2',
                       'cate_11_type2', 'cate_4_type3', 'cate_5_type3', 'cate_6_type3', 'cate_7_type3',
                       'cate_8_type3', 'cate_9_type3', 'cate_10_type3', 'cate_11_type3','cate_4_type4',
                       'cate_5_type4', 'cate_6_type4', 'cate_7_type4', 'cate_8_type4', 'cate_9_type4',
                       'cate_10_type4', 'cate_11_type4', 'cate_4_type5', 'cate_5_type5', 'cate_6_type5',
                       'cate_7_type5', 'cate_8_type5', 'cate_9_type5', 'cate_10_type5', 'cate_11_type5',
                       'cate_4_type6', 'cate_5_type6', 'cate_6_type6', 'cate_7_type6',
                       'cate_8_type6', 'cate_9_type6', 'cate_10_type6', 'cate_11_type6']
    actions = actions.fillna(0)
    actions['cate_action_sum'] = actions.sum(axis = 1)
    actions['cate8_percentage'] = (actions['cate_8_type1'] + actions['cate_8_type2'] +
                                  actions['cate_8_type3'] + actions['cate_8_type4'] +
                                  actions['cate_8_type5'] + actions['cate_8_type6']) / actions['cate_action_sum']
    actions['cate4_percentage'] = (actions['cate_4_type1'] + actions['cate_4_type2'] +
                                  actions['cate_4_type3'] + actions['cate_4_type4'] +
                                  actions['cate_4_type5'] + actions['cate_4_type6']) / actions['cate_action_sum']
    actions['cate5_percentage'] = (actions['cate_5_type1'] + actions['cate_5_type2'] +
                                  actions['cate_5_type3'] + actions['cate_5_type4'] +
                                  actions['cate_5_type5'] + actions['cate_5_type6']) / actions['cate_action_sum']
    actions['cate6_percentage'] = (actions['cate_6_type1'] + actions['cate_6_type2'] +
                                  actions['cate_6_type3'] + actions['cate_6_type4'] +
                                  actions['cate_6_type5'] + actions['cate_6_type6']) / actions['cate_action_sum']
    actions['cate7_percentage'] = (actions['cate_7_type1'] + actions['cate_7_type2'] +
                                  actions['cate_7_type3'] + actions['cate_7_type4'] +
                                  actions['cate_7_type5'] + actions['cate_7_type6']) / actions['cate_action_sum']
    actions['cate9_percentage'] = (actions['cate_9_type1'] + actions['cate_9_type2'] +
                                  actions['cate_9_type3'] + actions['cate_9_type4'] +
                                  actions['cate_9_type5'] + actions['cate_9_type6']) / actions['cate_action_sum']
    actions['cate10_percentage'] = (actions['cate_10_type1'] + actions['cate_10_type2'] +
                                  actions['cate_10_type3'] + actions['cate_10_type4'] +
                                  actions['cate_10_type5'] + actions['cate_10_type6']) / actions['cate_action_sum']
    actions['cate11_percentage'] = (actions['cate_11_type1'] + actions['cate_11_type2'] +
                                  actions['cate_11_type3'] + actions['cate_11_type4'] +
                                  actions['cate_11_type5'] + actions['cate_11_type6']) / actions['cate_action_sum']
    actions['cate8_type1_percentage'] = np.log(1 + actions['cate_8_type1']) - np.log(
                    1 + actions['cate_8_type1'] + actions['cate_4_type1'] + actions['cate_5_type1']
                    + actions['cate_6_type1'] + actions['cate_7_type1'] + actions['cate_9_type1']
                    + actions['cate_10_type1'] + actions['cate_11_type1'])
    actions['cate8_type2_percentage'] = np.log(1 + actions['cate_8_type2']) - np.log(
                    1 + actions['cate_8_type2'] + actions['cate_4_type2'] + actions['cate_5_type2']
                    + actions['cate_6_type2'] + actions['cate_7_type2'] + actions['cate_9_type2']
                    + actions['cate_10_type2'] + actions['cate_11_type2'])
    actions['cate8_type3_percentage'] = np.log(1 + actions['cate_8_type3']) - np.log(
                    1 + actions['cate_8_type3'] + actions['cate_4_type3'] + actions['cate_5_type3']
                    + actions['cate_6_type3'] + actions['cate_7_type3'] + actions['cate_9_type3']
                    + actions['cate_10_type3'] + actions['cate_11_type3'])
    actions['cate8_type4_percentage'] = np.log(1 + actions['cate_8_type4']) - np.log(
                    1 + actions['cate_8_type4'] + actions['cate_4_type4'] + actions['cate_5_type4']
                    + actions['cate_6_type4'] + actions['cate_7_type4'] + actions['cate_9_type4']
                    + actions['cate_10_type4'] + actions['cate_11_type4'])
    actions['cate8_type5_percentage'] = np.log(1 + actions['cate_8_type5']) - np.log(
                    1 + actions['cate_8_type5'] + actions['cate_4_type5'] + actions['cate_5_type5']
                    + actions['cate_6_type5'] + actions['cate_7_type5'] + actions['cate_9_type5']
                    + actions['cate_10_type5'] + actions['cate_11_type5'])
    actions['cate8_type6_percentage'] = np.log(1 + actions['cate_8_type6']) - np.log(
                    1 + actions['cate_8_type6'] + actions['cate_4_type6'] + actions['cate_5_type6']
                    + actions['cate_6_type6'] + actions['cate_7_type6'] + actions['cate_9_type6']
                    + actions['cate_10_type6'] + actions['cate_11_type6'])
    actions['user_id'] = actions.index
    actions = actions[['user_id', 'cate8_percentage', 'cate4_percentage', 'cate5_percentage', 'cate6_percentage',
                       'cate7_percentage', 'cate9_percentage', 'cate10_percentage', 'cate11_percentage', 'cate8_type1_percentage',
                       'cate8_type2_percentage', 'cate8_type3_percentage', 'cate8_type4_percentage', 'cate8_type5_percentage', 'cate8_type6_percentage']]
    actions.reset_index(inplace = True, drop = True)
    return actions

5.7 累積商品特徵

def get_accumulate_product_feat(start_date, end_date, all_actions):
    actions = get_time_action(start_date, end_date, all_actions)
    df = pd.get_dummies(actions['type'], prefix = 'product_action')
    # 按照商品-日期分組，計算某個時間段該商品的各項行爲的標準差
    actions['date'] = pd.to_datetime(actions['time']).apply(lambda x: x.date())
    actions = pd.concat([actions[['sku_id', 'date']], df], axis = 1)
    actions = actions.groupby(['sku_id'], as_index = False).sum()
    days_interal = (datetime.strptime(end_date, '%Y-%m-%d') - datetime.strptime(start_date, '%Y-%m-%d')).days
    
    actions['product_action_1_ratio'] = np.log(1+ actions['product_action_4.0']) - np.log(1 + actions['product_action_1.0'])
    actions['product_action_2_ratio'] = np.log(1+ actions['product_action_4.0']) - np.log(1 + actions['product_action_2.0'])
    actions['product_action_3_ratio'] = np.log(1+ actions['product_action_4.0']) - np.log(1 + actions['product_action_3.0'])
    actions['product_action_4_ratio'] = np.log(1+ actions['product_action_4.0']) - np.log(1 + actions['product_action_4.0'])
    actions['product_action_6_ratio'] = np.log(1+ actions['product_action_4.0']) - np.log(1 + actions['product_action_6.0'])
    # 計算各種行爲的均值
    actions['product_action_1_mean']= actions['product_action_1.0'] / days_interal
    actions['product_action_2_mean']= actions['product_action_2.0'] / days_interal
    actions['product_action_3_mean']= actions['product_action_3.0'] / days_interal
    actions['product_action_4_mean']= actions['product_action_4.0'] / days_interal
    actions['product_action_5_mean']= actions['product_action_5.0'] / days_interal
    actions['product_action_6_mean']= actions['product_action_6.0'] / days_interal
    return actions

5.8 類別特徵

def get_accumulate_cate_feat(start_date, end_date, all_actions):
    actions = get_time_action(start_date, end_date, all_actions)
    actions['date'] = pd.to_datetime(actions['time']).apply(lambda x: x.date())
    df = pd.get_dummies(actions['type'], prefix = 'cate_action')
    actions = pd.concat([actions[['cate', 'date']], df], axis = 1)
    # 按照類別分組，統計各個商品類別下行爲的轉化率
    actions = actions.groupby('cate', as_index = False).sum()
    days_interal = (datetime.strptime(end_date, '%Y-%m-%d') - datetime.strptime(start_date, '%Y-%m-%d')).days
    
    actions['cate_action_1_ratio'] = np.log(1 + actions['cate_action_4.0']) - np.log(1 + actions['cate_action_1.0']) 
    actions['cate_action_2_ratio'] = np.log(1 + actions['cate_action_4.0']) - np.log(1 + actions['cate_action_2.0'])
    actions['cate_action_3_ratio'] = np.log(1 + actions['cate_action_4.0']) - np.log(1 + actions['cate_action_3.0'])
    actions['cate_action_5_ratio'] = np.log(1 + actions['cate_action_4.0']) - np.log(1 + actions['cate_action_5.0'])
    actions['cate_action_6_ratio'] = np.log(1 + actions['cate_action_4.0']) - np.log(1 + actions['cate_action_6.0'])
    # 按照類別分組，統計各個商品類別下行爲在一段時間的均值
    actions['cate_action_1_mean'] = actions['cate_action_1.0'] / days_interal
    actions['cate_action_2_mean'] = actions['cate_action_2.0'] / days_interal
    actions['cate_action_3_mean'] = actions['cate_action_3.0'] / days_interal
    actions['cate_action_4_mean'] = actions['cate_action_4.0'] / days_interal
    actions['cate_action_5_mean'] = actions['cate_action_5.0'] / days_interal
    actions['cate_action_6_mean'] = actions['cate_action_6.0'] / days_interal
    return actions

六. 構造訓練集/測試集

6.1 標籤

def get_labels(start_date, end_date, all_actions):
    actions = get_time_action(start_date, end_date, all_actions)
    actions = actions[(actions['type'] == 4) & (actions['cate'] == 8)]
    actions = actions.groupby(['user_id', 'sku_id'], as_index = False).sum()
    actions['label'] = 1
    actions = actions[['user_id', 'sku_id', 'label']]
    return actions

6.2 構造測試集

def make_actions(user, product, all_actions, train_start_date):
    train_end_date = datetime.strptime(train_start_date, '%Y-%m-%d') + timedelta(days = 3)
    train_end_date = train_end_date.strftime('%Y-%m-%d')
    print(train_end_date)
    
    start_days = datetime.strptime(train_end_date, '%Y-%m-%d') - timedelta(days = 30)
    start_days = start_days.strftime('%Y-%m-%d')
    
    user_acc = get_recent_user_feat(train_end_date, all_actions) # 用戶近期行爲特徵   
    user_cate = get_user_cate_feature(train_start_date, train_end_date, all_actions) # 用戶對同類別下各種商品的行爲特徵
    product_acc = get_accumulate_product_feat(start_days, train_end_date, all_actions) # 累積商品特徵
    cate_acc = get_accumulate_cate_feat(start_days, train_end_date, all_actions) # 類別特徵
    comment_acc = get_comments_product_feat(train_end_date) # 評論特徵
    
    # 標記
    test_start_date = train_end_date
    test_end_date = datetime.strptime(test_start_date, '%Y-%m-%d') + timedelta(days = 5)
    test_end_date = test_end_date.strftime('%Y-%m-%d')
    labels = get_labels(test_start_date, test_end_date, all_actions)
    
    actions = None
    for i in (3, 5, 7, 10, 15, 21, 30):
        start_days = datetime.strptime(train_end_date, '%Y-%m-%d') - timedelta(days = i)
        start_days = start_days.strftime('%Y-%m-%d')
        if actions is None:
            actions = get_action_feat(start_days, train_end_date, all_actions, i)
        else:
            actions = pd.merge(actions, get_action_feat(start_days, train_end_date, all_actions, i), how = 'left',
                              on = ['user_id', 'sku_id', 'cate'])
            
    actions = pd.merge(actions, user, how = 'left', on = 'user_id')
    actions = pd.merge(actions, user_acc, how = 'left', on = 'user_id')
    actions = pd.merge(actions, user_cate, how = 'left', on = 'user_id')
    actions = pd.merge(actions, product, how = 'left', on = ['sku_id', 'cate'])
    actions = pd.merge(actions, product_acc, how = 'left', on = 'sku_id')
    actions = pd.merge(actions, cate_acc, how = 'left', on = 'cate')
    actions = pd.merge(actions, comment_acc, how = 'left', on = 'sku_id')
    actions = pd.merge(actions, labels, how = 'left', on = ['user_id', 'sku_id'])
    actions = actions.fillna(0)
    
    action_postive = actions[actions['label'] == 1]
    action_negative = actions[actions['label'] == 0]
    del actions
    
    neg_len = len(action_postive) * 10
    action_negative = action_negative.sample(n = neg_len)
    action_sample = pd.concat([action_postive, action_negative], ignore_index = True)
    return action_sample

def make_train_set(train_start_date, setNums, f_path, all_actions):
    train_actions = None
    user = get_basic_user_feat()
    product = get_basic_product_feat()
    for i in range(setNums):
        if train_actions is None:
            train_actions = make_actions(user, product, all_actions, train_start_date)
        else:
            train_actions = pd.concat([train_actions, make_actions(user, product, all_actions, train_start_date)], ignore_index = True)
        # 接下來每次移動一天
        train_start_date = datetime.strptime(train_start_date, '%Y-%m-%d') + timedelta(days = 1)
        train_start_date = train_start_date.strftime('%Y-%m-%d')
        print('Round {0} / {1} over!'.format(i+1, setNums))
    train_actions.to_csv(f_path, index = False)

train_start_date = '2016-02-01'
make_train_set(train_start_date, 20, 'train_set.csv',all_actions)

6.3 構造驗證集(線下測試集)

def make_val_answer(val_start_date, val_end_date, all_actions, label_val_s1_path):
    actions = get_time_action(val_start_date, val_end_date, all_actions)
    actions = actions[(actions['type'] == 4) & (actions['cate'] == 8)]
    actions = actions[['user_id', 'sku_id']]
    actions = actions.drop_duplicates()
    actions.to_csv(label_val_s1_path, index = False)
    
def make_val_set(train_start_date, train_end_date, val_s1_path):
    start_days = datetime.strptime(train_end_date, '%Y-%m-%d') - timedelta(days = 30)
    start_days = start_days.strftime('%Y-%m-%d')
    all_actions = get_all_action()
    user = get_basic_user_feat() # 用戶基本特徵
    product = get_basic_product_feat() # 商品基本特徵
    user_acc = get_recent_user_feat(train_end_date, all_actions)
    user_cate = get_user_cate_feature(train_start_date, train_end_date, all_actions)
    product_acc = get_accumulate_product_feat(start_days, train_end_date, all_actions)
    cate_acc = get_accumulate_cate_feat(start_days, train_end_date, all_actions)
    comment_acc = get_comments_product_feat(train_end_date)
    
    actions = None
    for i in (3, 5, 7, 10, 15, 21, 30):
        start_days = datetime.strptime(train_end_date, '%Y-%m-%d') - timedelta(days = i)
        start_days = start_days.strftime('%Y-%m-%d')
        if actions is None:
            actions = get_action_feat(start_days, train_end_date, all_actions, i)
        else:
            actions = pd.merge(actions, get_action_feat(start_days, train_end_date, all_actions, i),
                              how = 'left', on = ['user_id', 'sku_id', 'cate'])
    actions = pd.merge(actions, user, how = 'left', on = 'user_id')
    actions = pd.merge(actions, user_acc, how = 'left', on = 'user_id')
    actions = pd.merge(actions, user_cate, how = 'left', on = 'user_id')
    actions = pd.merge(actions, product, how = 'left', on = ['sku_id', 'cate'])
    actions = pd.merge(actions, product_acc, how = 'left', on = 'sku_id')
    actions = pd.merge(actions, cate_acc, how = 'left', on = 'cate')
    actions = pd.merge(actions, comment_acc, how = 'left', on = 'sku_id')
    actions = actions.fillna(0)
    
    val_start_date = train_end_date
    val_end_date = datetime.strptime(val_start_date, '%Y-%m-%d') + timedelta(days = 5)
    val_end_date = val_end_date.strftime('%Y-%m-%d')
    make_val_answer(val_start_date, val_end_date, all_actions, 'label_' + val_s1_path)
    
    actions.to_csv(val_s1_path, index = False)
    
make_val_set('2016-02-21', '2016-02-24', 'val_1.csv')

6.4 構造測試集

def make_test_set(train_start_date, train_end_date):
    start_days = datetime.strptime(train_end_date, '%Y-%m-%d') - timedelta(days = 30)
    start_days = start_days.strftime('%Y-%m-%d')
    all_actions = get_all_action()
    user = get_basic_user_feat()
    product = get_basic_product_feat()
    user_acc = get_recent_user_feat(train_end_date, all_actions)
    user_cate = get_user_cate_feature(train_start_date, train_end_date, all_actions)
    product_acc = get_accumulate_product_feat(start_days, train_end_date, all_actions)
    cate_acc = get_accumulate_cate_feat(start_days, train_end_date, all_actions)
    comment_acc = get_comments_product_feat(train_end_date)
    
    actions = None
    for i in (3, 5, 7, 10, 15, 21, 30):
        start_days = datetime.strptime(train_end_date, '%Y-%m-%d') - timedelta(days = i)
        start_days = start_days.strftime('%Y-%m-%d')
        if actions is None:
            actions = get_action_feat(start_days, train_end_date, all_actions, i)
        else:
            actions = pd.merge(actions, get_action_feat(start_days, train_end_date, all_actions, i),
                              how = 'left', on = ['user_id', 'sku_id', 'cate'])
    actions = pd.merge(actions, user, how = 'left', on = 'user_id')
    actions = pd.merge(actions, user_acc, how = 'left', on = 'user_id')
    actions = pd.merge(actions, user_cate, how = 'left', on = 'user_id')
    actions = pd.merge(actions, product, how = 'left', on = ['sku_id', 'cate'])
    actions = pd.merge(actions, product_acc, how = 'left', on = 'sku_id')
    actions = pd.merge(actions, cate_acc, how = 'left', on = 'cate')
    actions = pd.merge(actions, comment_acc, how = 'left', on = 'sku_id')
    actions = actions.fillna(0)
    
    actions.to_csv('test_set.csv', index = False)
    
make_test_set('2016-04-13', '2016-04-16')

七. Xgboost建模

data = pd.read_csv('train_set.csv')
data_x = data.loc[:, data.columns != 'label']
data_y = data.loc[:, data.columns == 'label']
print(data_x.shape)
data_x.head()

from sklearn.model_selection import train_test_split
x_train, x_test, y_train, y_test = train_test_split(data_x, data_y, test_size = 0.25, random_state = 42)
print(x_test.shape)

x_val = x_test.iloc[:1500, :]
y_val = y_test.iloc[:1500, :]
print(x_val.shape)

x_test = x_test.iloc[1500:, :]
y_test = y_test.iloc[1500:, :]
print(x_test.shape)

del x_train['user_id']
del x_train['sku_id']
del x_val['user_id']
del x_val['sku_id']

x_train.head()

import xgboost as xgb

dtrain = xgb.DMatrix(x_train, label = y_train)
dvalid = xgb.DMatrix(x_val, label = y_val)

param = {'n_estimators': 4000, 'max_depth': 3, 'min_child_weight': 5, 'gamma': 0,
         'subsample': 1.0, 'colsample_bytree': 0.8, 'scale_pos_weight':10,
         'eta': 0.1, 'silent': 1, 'objective': 'binary:logistic','eval_metric':'auc'}
num_round = param['n_estimators']
plst = param.items()
evallist = [(dtrain, 'train'), (dvalid, 'eval')]
bst = xgb.train(plst, dtrain, num_round, evallist, early_stopping_rounds = 10)
bst.save_model('bst.model')

bst.attributes()

{‘best_iteration’: ‘125’, ‘best_msg’: ‘[125]\ttrain-auc:0.984232\teval-auc:0.972103’,
‘best_score’: ‘0.972103’}

def create_feature_map(featrues):
    outfile = open(r'xgb.fmap', 'w')
    i = 0
    for feat in features:
        outfile.write('{0} \t {1} \tq\n'.format(i, feat))
        i += 1
    outfile.close()
    
features = list(x_train.columns[:])
create_feature_map(features)

import operator

def feature_importance(bst_xgb):
    importance = bst_xgb.get_fscore(fmap = r'xgb.fmap')
    importance = sorted(importance.items(), key = operator.itemgetter(1), reverse = True)
    
    df = pd.DataFrame(importance, columns = ['feature', 'fscore'])
    df['fscore'] = df['fscore'] / df['fscore'].sum()
    file_name = 'feature_importance_' + str(datetime.now().date())[5:] + '.csv'
    df.to_csv(file_name)
    
feature_importance(bst)

importances = pd.read_csv('feature_importance_06-21.csv')
importances.sort_values('fscore', inplace = True, ascending = False)
importances.head()

users = x_test[['user_id', 'sku_id', 'cate']].copy()
del x_test['user_id']
del x_test['sku_id']
x_test_DMatrix = xgb.DMatrix(x_test)
y_pred = bst.predict(x_test_DMatrix, ntree_limit = bst.best_ntree_limit)
x_test['pred_label'] = y_pred
x_test.head()

def label(column):
    if column['pred_label'] > 0.5:
        column['pred_label'] = 1
    else:
        column['pred_label'] = 0
    return column
x_test = x_test.apply(label, axis = 1)
x_test.head()

x_test['true_label'] = y_test
x_test.head()

x_test['user_id'] = users['user_id']
x_test['sku_id'] = users['sku_id']

# 所有購買用戶
all_user_set = x_test[x_test['true_label'] == 1]['user_id'].unique()
print(len(all_user_set))
# 所有預測購買的用戶
all_user_test_set = x_test[x_test['pred_label'] == 1]['user_id'].unique()
print(len(all_user_test_set))
all_user_test_item_pair = x_test[x_test['pred_label'] == 1]['user_id'].map(str) + '-' + x_test[x_test['pred_label'] == 1]['sku_id'].map(str)
all_user_test_item_pair = np.array(all_user_test_item_pair)
print(len(all_user_test_item_pair))

188
366
394

pos, neg = 0, 0
for user_id in all_user_test_set:
    if user_id in all_user_set:
        pos += 1
    else:
        neg += 1
all_user_acc = 1.0 * pos / (pos + neg)
all_user_recall = 1.0 * pos / len(all_user_set)
print('所有用戶中預測購買用戶的準確率爲 ' + str(all_user_acc))
print('所有用戶中預測購買用戶的召回率爲 ' + str(all_user_recall))

所有用戶中預測購買用戶的準確率爲 0.48633879781420764
所有用戶中預測購買用戶的召回率爲 0.9468085106382979

#所有實際商品對
all_user_item_pair = x_test[x_test['true_label'] == 1]['user_id'].map(str) + '-' + x_test[x_test['true_label'] == 1]['sku_id'].map(str)
all_user_item_pair = np.array(all_user_item_pair)
# print(len(all_user_item_pair))

pos, neg = 0, 0
for user_item_pair in all_user_test_item_pair:
    print(user_item_pair)
    if user_item_pair in all_user_item_pair:
        pos += 1
    else:
        neg += 1
all_item_acc = 1.0 * pos / (pos + neg)
all_item_recall = 1.0 * pos / len(all_user_item_pair)
print('所有用戶中預測購買用戶的準確率爲 ' + str(all_item_acc))
print('所有用戶中預測購買用戶的召回率爲 ' + str(all_item_recall))

F1 = 6.0 * all_user_recall * all_user_acc / (5.0 * all_user_recall + all_user_acc)
F2 = 5.0 * all_item_acc * all_item_recall / (2.0 * all_item_recall + 3.0 * all_item_acc)
score = 0.4 * F1 + 0.6 * F2
print('F1 = ' + str(F1))
print('F2 = ' + str(F2))
print('score = ' + str(score))

所有用戶中預測購買用戶的準確率爲 0.5177664974619289
所有用戶中預測購買用戶的召回率爲 0.9532710280373832
F1 = 0.5292368681863231
F2 = 0.7132867132867132
score = 0.6396667752465572