RovOpt/rov_train.py

import warnings

warnings.filterwarnings("ignore")
from sklearn.metrics import r2_score
import os
import pandas as pd
import gc
import math
import numpy as np
import time
from sklearn.linear_model import SGDRegressor
from sklearn.linear_model import SGDClassifier
import lightgbm as lgb
from sklearn.model_selection import train_test_split
from sklearn.model_selection import StratifiedKFold
from sklearn import metrics
import pickle
from sklearn.metrics import mean_squared_error
import seaborn as sns
import matplotlib.pylab as plt
from odps import ODPS
from odps.df import DataFrame as odpsdf
from datetime import datetime as dt
import datetime

now_date = datetime.date.today() 
# day = datetime.datetime.strftime(now_date, '%Y%m%d')
diff_1 = datetime.timedelta(days=1)
diff_5 = datetime.timedelta(days=7)
input_dt = now_date - diff_1
input_day = datetime.datetime.strftime(input_dt, '%Y%m%d')
now_day = datetime.datetime.strftime(now_date, '%Y%m%d')
train_dt = now_date - diff_5
train_day = datetime.datetime.strftime(train_dt, '%Y%m%d')

def getRovfeaturetable(dt, table):
    odps = ODPS('LTAI4FtW5ZzxMvdw35aNkmcp', '0VKnydcaHK3ITjylbgUsLubX6rwiwc', 'usercdm',
                endpoint='http://service.cn.maxcompute.aliyun.com/api', connect_timeout=3000, \
                read_timeout=500000, pool_maxsize=1000, pool_connections=1000)

    featureArray = []
    for record in odps.read_table(table, partition='dt=%s' % dt):
        valueFeature = {}
        for i in featurename:
            if i == 'dt':
                valueFeature[i] = dt
            else:
                valueFeature[i] = record[i]
        featureArray.append(valueFeature)
    featureArray = pd.DataFrame(featureArray)
    print(dt, table, 'feature table finish')
    return featureArray

def getdatasample(date, max_range, table):
    new_date = dt.strptime(date, '%Y%m%d')
    datelist = []
    testlist = []
    for i in range(0, max_range):
        delta = datetime.timedelta(days=i)
        tar_dt = new_date - delta
        datelist.append(tar_dt.strftime("%Y%m%d"))
    print(datelist)
    for tm in datelist:
        testlist.append(getRovtestable(tm, table))
    testdata = pd.concat(testlist)
    testdata.reset_index(inplace=True)
    testdata = testdata.drop(axis=1, columns='index')
    return testdata


traindata = getrainingdata(train_day, 30, 'rov_feature_add_v1')
data_test_ori_rk = getestingdata(input_day, 1, 'rov_predict_table_add_v1')


def select_recent_video(df):
    """对每一个视频添加row number，按照日期排序，最后选取最近的那一天"""
    df['dt'] = df['dt'].astype(int)
    df['rk'] = df['dt'].groupby(df['videoid']).rank(ascending=0, method='first')
    df = df[df['rk'] == 1]
    return df


data_test_ori = select_recent_video(data_test_ori_rk)
data_test_ori.loc[data_test_ori['dt'] != int(input_day), 'futre7dayreturn'] = 0
data_test_ori = data_test_ori.drop(axis=1, columns='rk')

traindata.drop_duplicates(subset=['videoid', 'dt'], keep='first', inplace=True)
data_test_ori.drop_duplicates(subset=['videoid', 'dt'], keep='first', inplace=True)


def basic_cal(df):
    df['weighted_retrn'] = df['futre7dayreturn'].astype('int') 
    df['weighted_retrn_log'] = df.apply(lambda x: np.log(x['weighted_retrn'] + 1),axis=1)
    ## 设置回流大于thresh， label就是1， 没有分享或有分享但是回流数是零的标为0
    df['return_back'] = df.apply(lambda x:1 if x['weighted_retrn']> 0 else 0,axis=1)
    return df 


data_train = basic_cal(traindata)
data_test = basic_cal(data_test_ori)

def today_view_category(df):
### 对当天的曝光量分三个级别，未来三天的曝光量分3个级别，添加Category feaure
    data_test1_view1 =   df.loc[data_test['day1viewcount_rank'] > 10000]['day1viewcount'].mean()
    data_test1_view2 =   df.loc[(data_test['day1viewcount_rank'] > 3000)&(data_test['day1viewcount_rank'] <= 10000)]['day1viewcount'].mean()
    data_test1_view3 =   df.loc[(data_test['day1viewcount_rank'] > 1000)&(data_test['day1viewcount_rank'] <= 3000)]['day1viewcount'].mean()
    data_test1_view4 =   df.loc[(data_test['day1viewcount_rank'] > 300)&(data_test['day1viewcount_rank'] <= 1000)]['day1viewcount'].mean()
    data_test1_view5 =   df.loc[(data_test['day1viewcount_rank'] > 100)&(data_test['day1viewcount_rank'] <= 300)]['day1viewcount'].mean()
    data_test1_view6 =   df.loc[(data_test['day1viewcount_rank'] > 30)&(data_test['day1viewcount_rank'] <= 100)]['day1viewcount'].mean()
    data_test1_view7 =   df.loc[(data_test['day1viewcount_rank'] > 0)&(data_test['day1viewcount_rank'] <= 30)]['day1viewcount'].mean()
    
    df.loc[df['day1viewcount_rank'] > 10000, 'todyviewcount'] = data_test1_view1
    df.loc[(data_test['day1viewcount_rank'] > 3000)&(data_test['day1viewcount_rank'] <= 10000), 'todyviewcount'] = data_test1_view2
    df.loc[(data_test['day1viewcount_rank'] > 1000)&(data_test['day1viewcount_rank'] <= 3000), 'todyviewcount'] = data_test1_view3
    df.loc[(data_test['day1viewcount_rank'] > 300)&(data_test['day1viewcount_rank'] <= 1000), 'todyviewcount'] = data_test1_view4
    df.loc[(data_test['day1viewcount_rank'] > 100)&(data_test['day1viewcount_rank'] <= 300), 'todyviewcount'] = data_test1_view5
    df.loc[(data_test['day1viewcount_rank'] > 30)&(data_test['day1viewcount_rank'] <= 100), 'todyviewcount'] = data_test1_view6
    df.loc[(data_test['day1viewcount_rank'] > 0)&(data_test['day1viewcount_rank'] <= 30), 'todyviewcount'] = data_test1_view7
    return df

data_test =  today_view_category(data_test) 


def dataprepare(df_pre):
    #  直接将特征送进去，不加交叉特征。
    # 是否对数据补零
    df_pre = df_pre.fillna(0)
    df_new_feature = df_pre[features]
    df_target = df_pre['weighted_retrn_log']
    df_new_feature = pd.concat([df_new_feature, df_pre[cate_feat],df_pre[one_hot_feature]], axis=1)
    return df_new_feature, df_target

data_test['videoid'] = data_test['videoid'].astype('int')

data_train = data_train[data_train['weighted_retrn'] > 0]
print(data_train.shape, 'train shape')
data_test = pd.merge(data_test, recall_video_stage_one, on=['videoid'], how='inner')
print('score>0.5 video_count:', data_test.shape)

df_new_feature,df_target= dataprepare(data_train)
df_new_feature_test, df_target_test = dataprepare(data_test)


#数值
from scipy import sparse

df_new_feature_part_one = sparse.csr_matrix(np.array(pd.DataFrame(df_new_feature).loc[:,'day1playcount':'videocategory555']))
df_new_feature_test_part_one = sparse.csr_matrix(np.array(pd.DataFrame(df_new_feature_test).loc[:,'day1playcount':'videocategory555']))

print('value feature generate successfully')

train_videoid = pd.DataFrame(df_new_feature).loc[:,'videoid']
test_videoid = pd.DataFrame(df_new_feature_test).loc[:,'videoid']

train_videoid_list = pd.DataFrame(df_new_feature).loc[:,'videoid'].to_numpy().reshape(len(pd.DataFrame(df_new_feature).loc[:,'videoid']),1).tolist()
test_videoid_list = pd.DataFrame(df_new_feature_test).loc[:,'videoid'].to_numpy().reshape(len(pd.DataFrame(df_new_feature_test).loc[:,'videoid']),1).tolist()


allvideo_raw = list(set(np.array(pd.concat([train_videoid,test_videoid])).tolist()))
allvideo = np.array(allvideo_raw).reshape(len(allvideo_raw),1).tolist()
from sklearn.preprocessing import MultiLabelBinarizer

mlb_model_videoid = MultiLabelBinarizer(sparse_output=True).fit(allvideo)
train_videoid = mlb_model_videoid.transform(train_videoid_list)
test_videoid = mlb_model_videoid.transform(test_videoid_list)

print('videoid feature generate successfully')

#获取tag-one-hot
tags ,train_tag,test_tag = tag_preprocessing('tag')
#获取tag tfidf
tag_dict = get_tag_tfidf('20200305','video_tag_tf_idf')
print('lenth tag_dict:',len(tag_dict))
#获取tfidf_tag 稀疏矩阵
tag_corpus = tags.tolist()  #corpus
tag_tfidf_list = ttfidf_list_generation(tag_corpus,tag_dict )
tag_tf_idf_matrix  = sparse.csr_matrix(np.array(tag_tfidf_list))

tag_feature_train = train_tag.multiply(tag_tf_idf_matrix)  
tag_feature_test = test_tag.multiply(tag_tf_idf_matrix)  
print('tag tfidf feature generate successfully')

print('tag dimension:', len(tag_tfidf_list))


# In[28]:


#获取values without tag
words ,train_words,test_words = tag_preprocessing('words_no_tag')
#获取words tfidf
words_dict = get_tag_tfidf('20200305','video_words_without_tags_tfidf')
print('lenth words_dict:',len(words_dict))
#获取tfidf_tag 稀疏矩阵
words_corpus = words.tolist()  #corpus
words_tfidf_list = ttfidf_list_generation(words_corpus,words_dict )
words_tf_idf_matrix  = sparse.csr_matrix(np.array(words_tfidf_list))
words_feature_train = train_words.multiply(words_tf_idf_matrix)  
words_feature_test = test_words.multiply(words_tf_idf_matrix)  
print('tag tfidf feature generate successfully')
print('words dimension:', len(words_tfidf_list))


def featureImportance(fold1_df,fold2_df,fold3_df,fold4_df,values_lenth,video_id_lenth,tag_length,word_length):
    Feature_Data= pd.DataFrame()
    for df in (fold1_df,fold2_df,fold3_df,fold4_df):
        fold1_df1 = df.iloc[0:values_lenth,:]
        videoid_fold1_importance = df.iloc[values_lenth:values_lenth+video_id_lenth,:]['importance'].sum()
        fold1_df2 = pd.DataFrame([{'Feature':'videoid','importance':videoid_fold1_importance,'fold':1}])
        tag_fold1_importance = df.iloc[values_lenth+video_id_lenth:values_lenth+video_id_lenth+tag_length,:]['importance'].sum()
        fold1_df3 = pd.DataFrame([{'Feature':'tags','importance':tag_fold1_importance,'fold':1}])
        words_fold1_importance = df.iloc[values_lenth+video_id_lenth+tag_length:values_lenth+video_id_lenth+tag_length+word_length,:]['importance'].sum()
        fold1_df4 = pd.DataFrame([{'Feature':'words','importance':words_fold1_importance,'fold':1}])
        
        
        Feature_Data = pd.concat([Feature_Data,fold1_df1,fold1_df2,fold1_df3,fold1_df4])
        
    return Feature_Data


def MAPE(true, pred):
    true = np.array(true)
    sum_ = 0
    count = 0
    for i in range(len(true)):
        if true[i] != 0:
            sum_ = sum_ + np.abs(true[i] - pred[i]) / true[i]
            count = count + 1

        else:
            continue

    return sum_ / count



def do_train():
    from scipy.sparse import hstack

    df_new_feature = hstack([df_new_feature_part_one,train_videoid,tag_feature_train, words_feature_train])
    df_new_feature_test = hstack([df_new_feature_test_part_one,test_videoid,tag_feature_test,words_feature_test])

    #target
    df_target_test = sparse.csr_matrix(pd.DataFrame(df_target_test).values).toarray()
    df_target = sparse.csr_matrix(pd.DataFrame(df_target).values).toarray()


    param = {'num_leaves': 18,
         'min_data_in_leaf': 60,
         'objective': 'regression',
         'max_depth': -1,
         'learning_rate': 0.01,
         "min_child_samples": 30,
         "boosting": "gbdt",
         "feature_fraction": 0.8,
         "bagging_freq": 1,
         "bagging_fraction": 0.8,
         "bagging_seed": 11,
         "metric": 'rmse',
         "lambda_l1": 0.1,
         "verbosity": -1,
         "nthread": 4,
         #          'max_bin': 512,
         "random_state": 4590}

    folds = StratifiedKFold(n_splits=4, shuffle=True, random_state=4590)
    oof = np.zeros(len(pd.DataFrame(df_new_feature.toarray())))
    predictions = np.zeros(len(df_target_test))
    feature_importance_df = pd.DataFrame()


    values_lenth = len(features + cate_feat)
    video_id_lenth = len(mlb_model_videoid.classes_)
    tag_length = len(tag_tfidf_list)
    word_length = len(words_tfidf_list)


    change_view = pd.DataFrame(pd.DataFrame(df_new_feature_test.toarray()))
    change_view = change_view.sort_index()


    for fold_, (trn_idx, val_idx) in enumerate(folds.split(df_new_feature, data_train['return_back'].values)):
        print("folds {}".format(fold_))
        trn_data = lgb.Dataset(df_new_feature.tocsr()[trn_idx,:], label=pd.DataFrame(df_target).iloc[trn_idx])
        val_data = lgb.Dataset(df_new_feature.tocsr()[val_idx,:], label=pd.DataFrame(df_target).iloc[val_idx])

        num_round = 10000
        clf = lgb.train(param, trn_data, num_round, valid_sets=[trn_data, val_data], verbose_eval=100,
                early_stopping_rounds=200)
        oof[val_idx] = clf.predict(df_new_feature.tocsr()[val_idx,:], num_iteration=clf.best_iteration)
        predictions += clf.predict(df_new_feature_test, num_iteration=clf.best_iteration) / folds.n_splits

        fold_importance_df = pd.DataFrame()
        
        column = features+cate_feat+mlb_model_videoid.classes_.tolist()+ tag_corpus + words_corpus
        fold_importance_df["Feature"] = np.array(column)
        
        fold_importance_df["importance"] = clf.feature_importance()
        fold_importance_df["fold"] = fold_ + 1
        feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)


    fold1_df = feature_importance_df.loc[feature_importance_df['fold']==1]
    fold2_df = feature_importance_df.loc[feature_importance_df['fold']==2]
    fold3_df = feature_importance_df.loc[feature_importance_df['fold']==3]
    fold4_df = feature_importance_df.loc[feature_importance_df['fold']==4]



    feature_importance_df = featureImportance(fold1_df,fold2_df,fold3_df,fold4_df,values_lenth,video_id_lenth,tag_length,word_length)

    print('oof_rmse:', np.sqrt(mean_squared_error(df_target, oof)))
    print('oof_mse:', mean_squared_error(df_target, oof))

    print('test_rmse:', np.sqrt(mean_squared_error(df_target_test, predictions)))
    print('test_mse:', mean_squared_error(df_target_test, predictions))


    print('oof_mape:', MAPE(df_target, oof))
    print('test_mape:', MAPE(df_target_test, predictions))

    print('verification r2:', r2_score(df_target, oof))
    print('test r2:', r2_score(df_target_test, predictions))

    sub_df_ = pd.DataFrame({"videoid": data_test["videoid"].values})
    sub_df_['score'] = predictions
    print('regre ranking shape', sub_df_.shape)