update

config.py

@@ -22,6 +22,9 @@ class BaseConfig(object):
     # 预测数据文件存放路径
     PREDICT_DATA_FILENAME = 'predict_data.pickle'
 
+    # 模型存放文件
+    MODEL_FILENAME = 'model.pickle'
+
 
 def set_config():
     return BaseConfig()

rov_train.py

@@ -1,9 +1,11 @@
+import os
 import lightgbm as lgb
+import pandas as pd
 
 from sklearn.model_selection import train_test_split
-from sklearn.metrics import mean_absolute_error, r2_score
+from sklearn.metrics import mean_absolute_error, r2_score, mean_absolute_percentage_error
 from config import set_config
-from utils import read_from_pickle
+from utils import read_from_pickle, write_to_pickle
 from log import Log
 
 config_ = set_config()
@@ -46,12 +48,13 @@ def process_data(filename):
     return x, y, video_ids, features
 
 
-def train(x, y):
+def train(x, y, features):
     """
     训练模型
-    :param x:
-    :param y:
-    :return:
+    :param x: X
+    :param y: Y
+    :param features: 特征列表
+    :return: None
     """
     # 训练集、测试集分割
     x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.33)
@@ -85,39 +88,68 @@ def train(x, y):
                       valid_sets=[test_set], early_stopping_rounds=100,
                       verbose_eval=100, evals_result=evals_result)
 
+    # 将模型特征重要度存入csv
+    feature_importance_data = {'feature': features, 'feature_importance': model.feature_importance()}
+    feature_importance_filename = 'model_feature_importance.csv'
+    pack_result_to_csv(filename=feature_importance_filename, sort_columns=['feature_importance'],
+                       ascending=False, **feature_importance_data)
+
     # 测试集预测
-    pre_test_y = model.predict(data=x_test, num_iteration=model.best_iteration)
+    pre_y_test = model.predict(data=x_test, num_iteration=model.best_iteration)
     y_test = y_test.values
 
-    err_mae = mean_absolute_error(y_test, pre_test_y)
-    r2 = r2_score(y_test, pre_test_y)
-
-    print(err_mae, r2)
+    err_mae = mean_absolute_error(y_test, pre_y_test)
+    err_mape = mean_absolute_percentage_error(y_test, pre_y_test)
+    r2 = r2_score(y_test, pre_y_test)
 
+    # 将测试集结果存入csv
+    test_data = {'pre_y_test': pre_y_test, 'y_test': y_test}
+    test_result_filename = 'test_result.csv'
+    pack_result_to_csv(filename=test_result_filename, sort_columns=['pre_y_test'], ascending=False, **test_data)
 
-if __name__ == '__main__':
-    # dt_test = '20211007'
-    # project_test = 'usercdm'
-    # table_test = 'rov_feature_add_v1'
-    # res = get_rov_feature_table(dt_test, table_test)
-    # res = get_data_with_date(date=dt_test, delta_days=2, project=project_test, table=table_test)
-    # print(res.shape)
-    # write_to_pickle(res, 'test.pickle')
-
-    # data = read_from_pickle('test.pickle')
-    # if data is not None:
-    #     print(data.shape, type(data))
-    #     print(list(data))
-    #     print(data[data['futre7dayreturn']<0])
-    # else:
-    #     print(data)
+    print(err_mae, err_mape, r2)
 
-    train_filename = config_.TRAIN_DATA_FILENAME
-    x, y, videos, fea = process_data(filename=train_filename)
-    print(x.shape, y.shape)
-    print(len(fea), fea)
-    train(x, y)
+    # 保存模型
+    write_to_pickle(data=model, filename=config_.MODEL_FILENAME)
 
 
+def pack_result_to_csv(filename, sort_columns=None, filepath=config_.DATA_DIR_PATH, ascending=True, **data):
+    """
+    打包数据并存入csv
+    :param filename: csv文件名
+    :param sort_columns: 指定排序列名列名，type-list, 默认为None
+    :param filepath: csv文件存放路径，默认为config_.DATA_DIR_PATH
+    :param ascending: 是否按指定列的数组升序排列，默认为True，即升序排列
+    :param data: 数据
+    :return: None
+    """
+    if not os.path.exists(filepath):
+        os.makedirs(filepath)
+    file = os.path.join(filepath, filename)
+    df = pd.DataFrame(data=data)
+    if sort_columns:
+        df = df.sort_values(by=sort_columns, ascending=ascending)
+    df.to_csv(file, index=False)
+
+
+def predict():
+    """预测"""
+    # 读取预测数据并进行清洗
+    x, y, video_ids, _ = process_data(config_.PREDICT_DATA_FILENAME)
+    # 获取训练好的模型
+    model = read_from_pickle(filename=config_.MODEL_FILENAME)
+    # 预测
+    y_ = model.predict(x)
+    # 打包预测结果存入csv
+    predict_data = {'y_': y_, 'y': y, 'video_ids': video_ids}
+    predict_result_filename = 'predict.csv'
+    pack_result_to_csv(filename=predict_result_filename, sort_columns=['y_'], ascending=False, **predict_data)
 
 
+if __name__ == '__main__':
+    train_filename = config_.TRAIN_DATA_FILENAME
+    X, Y, videos, fea = process_data(filename=train_filename)
+    print(X.shape, Y.shape)
+    print(len(fea), fea)
+    train(X, Y, features=fea)
+    predict()