import warnings warnings.filterwarnings("ignore") from sklearn.metrics import r2_score import os import pandas as pd import gc import math import numpy as np import time from sklearn.linear_model import SGDRegressor from sklearn.linear_model import SGDClassifier import lightgbm as lgb from sklearn.model_selection import train_test_split from sklearn.model_selection import StratifiedKFold from sklearn.preprocessing import MultiLabelBinarizer from sklearn import metrics import pickle from sklearn.metrics import mean_squared_error import seaborn as sns import matplotlib.pylab as plt from odps import ODPS from odps.df import DataFrame as odpsdf from datetime import datetime as dt import datetime from scipy import sparse from scipy.sparse import hstack import process_feature import process_tag def getRovfeaturetable(dt, table): odps = ODPS('LTAI4FtW5ZzxMvdw35aNkmcp', '0VKnydcaHK3ITjylbgUsLubX6rwiwc', 'usercdm', endpoint='http://service.cn.maxcompute.aliyun.com/api', connect_timeout=3000, \ read_timeout=500000, pool_maxsize=1000, pool_connections=1000) featureArray = [] for record in odps.read_table(table, partition='dt=%s' % dt): valueFeature = {} for i in process_feature.featurename: if i == 'dt': valueFeature[i] = dt else: valueFeature[i] = record[i] featureArray.append(valueFeature) featureArray = pd.DataFrame(featureArray) print(dt, table, 'feature table finish') return featureArray def getdatasample(date, max_range, table): new_date = dt.strptime(date, '%Y%m%d') datelist = [] testlist = [] for i in range(0, max_range): delta = datetime.timedelta(days=i) tar_dt = new_date - delta datelist.append(tar_dt.strftime("%Y%m%d")) for tm in datelist: testlist.append(getRovfeaturetable(tm, table)) testdata = pd.concat(testlist) testdata.reset_index(inplace=True) testdata = testdata.drop(axis=1, columns='index') return testdata def select_recent_video(df): """对每一个视频添加row number，按照日期排序，最后选取最近的那一天""" df['dt'] = df['dt'].astype(int) df['rk'] = df['dt'].groupby(df['videoid']).rank(ascending=0, method='first') df = df[df['rk'] == 1] return df def basic_cal(df): df['weighted_retrn'] = df['futre7dayreturn'].astype('int') df['weighted_retrn_log'] = df.apply(lambda x: np.log(x['weighted_retrn'] + 1),axis=1) df['return_back'] = df.apply(lambda x:1 if x['weighted_retrn']> 0 else 0,axis=1) return df def dataprepare(df_pre): # 直接将特征送进去，不加交叉特征。 df_pre = df_pre.fillna(0) #df_new_feature = df_pre[process_feature.features] df_new_feature = df_pre[process_feature.filter_recent_features()] df_target = df_pre['weighted_retrn_log'] df_new_feature = pd.concat([df_new_feature, df_pre[process_feature.cate_feat],df_pre[process_feature.one_hot_feature]], axis=1) return df_new_feature, df_target def featureImportance(fold1_df,fold2_df,fold3_df,fold4_df,values_lenth,video_id_lenth,tag_length,word_length): Feature_Data= pd.DataFrame() for df in (fold1_df,fold2_df,fold3_df,fold4_df): fold1_df1 = df.iloc[0:values_lenth,:] videoid_fold1_importance = df.iloc[values_lenth:values_lenth+video_id_lenth,:]['importance'].sum() fold1_df2 = pd.DataFrame([{'Feature':'videoid','importance':videoid_fold1_importance,'fold':1}]) tag_fold1_importance = df.iloc[values_lenth+video_id_lenth:values_lenth+video_id_lenth+tag_length,:]['importance'].sum() fold1_df3 = pd.DataFrame([{'Feature':'tags','importance':tag_fold1_importance,'fold':1}]) words_fold1_importance = df.iloc[values_lenth+video_id_lenth+tag_length:values_lenth+video_id_lenth+tag_length+word_length,:]['importance'].sum() fold1_df4 = pd.DataFrame([{'Feature':'words','importance':words_fold1_importance,'fold':1}]) Feature_Data = pd.concat([Feature_Data,fold1_df1,fold1_df2,fold1_df3,fold1_df4]) return Feature_Data def MAPE(true, pred): true = np.array(true) sum_ = 0 count = 0 for i in range(len(true)): if true[i] != 0: sum_ = sum_ + np.abs(true[i] - pred[i]) / true[i] count = count + 1 else: continue return sum_ / count def process_train_predict_data(): now_date = datetime.date.today() # day = datetime.datetime.strftime(now_date, '%Y%m%d') diff_1 = datetime.timedelta(days=1) diff_5 = datetime.timedelta(days=7) predict_dt = now_date - diff_1 predict_day = datetime.datetime.strftime(predict_dt, '%Y%m%d') train_dt = now_date - diff_5 train_day = datetime.datetime.strftime(train_dt, '%Y%m%d') #read data from ali train_data = getdatasample(train_day, 30, 'rov_feature_add_v1') predict_data = getdatasample(predict_day, 1, 'rov_predict_table_add_v1') #pickle for test import _pickle as cPickle with open('train_data.pickle','wb') as output_file: cPickle.dump(train_data, output_file) with open('predict_data.pickle','wb') as output_file: cPickle.dump(predict_data, output_file) #with open(r"train_data.pickle", "rb") as input_file: ''' with open(r"train_data.pickle", "rb") as input_file: train_data = cPickle.load(input_file) with open(r"predict_data.pickle", "rb") as input_file: predict_data = cPickle.load(input_file) ''' #end pickle train_data = basic_cal(train_data) predict_data = basic_cal(predict_data) predict_data = select_recent_video(predict_data) #predict_data.loc[predict_data['dt'] != int(predict_day), 'futre7dayreturn'] = 0 predict_data = predict_data.drop(axis=1, columns='rk') train_data.drop_duplicates(subset=['videoid', 'dt'], keep='first', inplace=True) predict_data.drop_duplicates(subset=['videoid', 'dt'], keep='first', inplace=True) train_data = train_data.fillna(0) predict_data = predict_data.fillna(0) train_data = process_feature.cal_feature(train_data) predict_data = process_feature.cal_feature(predict_data) predict_data['videoid'] = predict_data['videoid'].astype('int') df_new_feature,df_target= dataprepare(train_data) df_new_feature_predict, df_target_predict = dataprepare(predict_data) df_new_feature_part_one = sparse.csr_matrix(np.array(pd.DataFrame(df_new_feature).loc[:,'day1playcount':'videocategory555'])) df_new_feature_predict_part_one = sparse.csr_matrix(np.array(pd.DataFrame(df_new_feature_predict).loc[:,'day1playcount':'videocategory555'])) print('value feature generate successfully') train_videoid = pd.DataFrame(df_new_feature).loc[:,'videoid'] predict_videoid = pd.DataFrame(df_new_feature_predict).loc[:,'videoid'] train_videoid_list = pd.DataFrame(df_new_feature).loc[:,'videoid'].to_numpy().reshape(len(pd.DataFrame(df_new_feature).loc[:,'videoid']),1).tolist() predict_videoid_list = pd.DataFrame(df_new_feature_predict).loc[:,'videoid'].to_numpy().reshape(len(pd.DataFrame(df_new_feature_predict).loc[:,'videoid']),1).tolist() allvideo_raw = list(set(np.array(pd.concat([train_videoid,predict_videoid])).tolist())) allvideo = np.array(allvideo_raw).reshape(len(allvideo_raw),1).tolist() mlb_model_videoid = MultiLabelBinarizer(sparse_output=True).fit(allvideo) train_videoid = mlb_model_videoid.transform(train_videoid_list) predict_videoid = mlb_model_videoid.transform(predict_videoid_list) print('videoid feature generate successfully') #获取tag-one-hot tags ,train_tag,predict_tag = process_tag.tag_preprocessing('tag', df_new_feature, df_new_feature_predict) #获取tag tfidf tag_dict = process_tag.get_tag_tfidf('20200305','video_tag_tf_idf') print('lenth tag_dict:',len(tag_dict)) #获取tfidf_tag 稀疏矩阵 tag_corpus = tags.tolist() #corpus tag_tfidf_list = process_tag.ttfidf_list_generation(tag_corpus,tag_dict ) tag_tf_idf_matrix = sparse.csr_matrix(np.array(tag_tfidf_list)) tag_feature_train = train_tag.multiply(tag_tf_idf_matrix) tag_feature_test = predict_tag.multiply(tag_tf_idf_matrix) print('tag tfidf feature generate successfully') print('tag dimension:', len(tag_tfidf_list)) #获取values without tag words ,train_words,test_words = process_tag.tag_preprocessing('words_no_tag', df_new_feature, df_new_feature_predict) #获取words tfidf words_dict = process_tag.get_tag_tfidf('20200305','video_words_without_tags_tfidf') print('lenth words_dict:',len(words_dict)) #获取tfidf_tag 稀疏矩阵 words_corpus = words.tolist() #corpus words_tfidf_list = process_tag.ttfidf_list_generation(words_corpus,words_dict ) words_tf_idf_matrix = sparse.csr_matrix(np.array(words_tfidf_list)) words_feature_train = train_words.multiply(words_tf_idf_matrix) words_feature_test = test_words.multiply(words_tf_idf_matrix) print('tag tfidf feature generate successfully') print('words dimension:', len(words_tfidf_list)) df_new_feature = hstack([df_new_feature_part_one,train_videoid,tag_feature_train, words_feature_train]) df_new_feature_predict = hstack([df_new_feature_predict_part_one,predict_videoid,tag_feature_test,words_feature_test]) return train_data, predict_data, df_target, df_target_predict, df_new_feature, df_new_feature_predict def do_train(train_data, predict_data, df_target, df_target_predict, df_new_feature, df_new_feature_predict): #target df_target_predict = sparse.csr_matrix(pd.DataFrame(df_target_predict).values).toarray() df_target = sparse.csr_matrix(pd.DataFrame(df_target).values).toarray() param = {'num_leaves': 18, 'min_data_in_leaf': 60, 'objective': 'regression', 'max_depth': -1, 'learning_rate': 0.01, "min_child_samples": 30, "boosting": "gbdt", "feature_fraction": 0.8, "bagging_freq": 1, "bagging_fraction": 0.8, "bagging_seed": 11, "metric": 'rmse', "lambda_l1": 0.1, "verbosity": -1, "nthread": 4, # 'max_bin': 512, "random_state": 4590} folds = StratifiedKFold(n_splits=4, shuffle=True, random_state=4590) #oof = np.zeros(len(pd.DataFrame(df_new_feature.toarray()))) oof = np.zeros(len(df_target)) predictions = np.zeros(len(df_target_predict)) feature_importance_df = pd.DataFrame() # values_lenth = len(process_feature.features + process_feature.cate_feat) # video_id_lenth = len(mlb_model_videoid.classes_) # tag_length = len(tag_tfidf_list) # word_length = len(words_tfidf_list) change_view = pd.DataFrame(pd.DataFrame(df_new_feature_predict.toarray())) change_view = change_view.sort_index() for fold_, (trn_idx, val_idx) in enumerate(folds.split(df_new_feature, train_data['return_back'].values)): print("folds {}".format(fold_)) trn_data = lgb.Dataset(df_new_feature.tocsr()[trn_idx,:], label=pd.DataFrame(df_target).iloc[trn_idx]) val_data = lgb.Dataset(df_new_feature.tocsr()[val_idx,:], label=pd.DataFrame(df_target).iloc[val_idx]) num_round = 10000 clf = lgb.train(param, trn_data, num_round, valid_sets=[trn_data, val_data], verbose_eval=100, early_stopping_rounds=200) oof[val_idx] = clf.predict(df_new_feature.tocsr()[val_idx,:], num_iteration=clf.best_iteration) predictions += clf.predict(df_new_feature_predict, num_iteration=clf.best_iteration) / folds.n_splits fold_importance_df = pd.DataFrame() # column = process_feature.features+process_feature.cate_feat+mlb_model_videoid.classes_.tolist()+ tag_corpus + words_corpus # fold_importance_df["Feature"] = np.array(column) # fold_importance_df["importance"] = clf.feature_importance() # fold_importance_df["fold"] = fold_ + 1 # feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0) # fold1_df = feature_importance_df.loc[feature_importance_df['fold']==1] # fold2_df = feature_importance_df.loc[feature_importance_df['fold']==2] # fold3_df = feature_importance_df.loc[feature_importance_df['fold']==3] # fold4_df = feature_importance_df.loc[feature_importance_df['fold']==4] # feature_importance_df = featureImportance(fold1_df,fold2_df,fold3_df,fold4_df,values_lenth,video_id_lenth,tag_length,word_length) print('oof_rmse:', np.sqrt(mean_squared_error(df_target, oof))) print('oof_mse:', mean_squared_error(df_target, oof)) print('test_rmse:', np.sqrt(mean_squared_error(df_target_predict, predictions))) print('test_mse:', mean_squared_error(df_target_predict, predictions)) print('oof_mape:', MAPE(df_target, oof)) print('test_mape:', MAPE(df_target_predict, predictions)) print('verification r2:', r2_score(df_target, oof)) print('test r2:', r2_score(df_target_predict, predictions)) sub_df_ = pd.DataFrame({"videoid": predict_data["videoid"].values}) sub_df_['score'] = predictions print('regre ranking shape', sub_df_.shape) sub_df_.to_csv('result.csv') if __name__ == '__main__': train_data, predict_data, df_target, df_target_predict, df_new_feature, df_new_feature_predict = process_train_predict_data() do_train(train_data, predict_data, df_target, df_target_predict, df_new_feature, df_new_feature_predict)