cp logs & data to oss

config.py

@@ -9,6 +9,9 @@ class BaseConfig(object):
     # 数据存放路径
     DATA_DIR_PATH = './data'
 
+    # oss 目标Bucket
+    BUCKET_NAME = 'ali-recommend'
+
     # 训练数据截止时间距当前日期间隔天数
     TRAIN_DIFF = 7
     # 训练数据所需数据范围天数
@@ -66,6 +69,9 @@ class DevelopmentConfig(BaseConfig):
     # 计算完ROV通知后端接口地址
     NOTIFY_BACKEND_UPDATE_ROV_SCORE_URL = 'http://videotest-internal.yishihui.com/longvideoapi/openapi/recommend/updateRovScore'
 
+    # logs 上传oss 目标Bucket指定目录
+    OSS_FOLDER = 'rov-offline/dev/'
+
 
 class TestConfig(BaseConfig):
     """测试环境配置"""
@@ -95,6 +101,9 @@ class TestConfig(BaseConfig):
     # 计算完ROV通知后端接口地址
     NOTIFY_BACKEND_UPDATE_ROV_SCORE_URL = 'http://videotest-internal.yishihui.com/longvideoapi/openapi/recommend/updateRovScore'
 
+    # logs 上传oss 目标Bucket指定目录
+    OSS_FOLDER = 'rov-offline/test/'
+
 
 class PreProductionConfig(BaseConfig):
     """预发布环境配置"""
@@ -124,6 +133,9 @@ class PreProductionConfig(BaseConfig):
     # 计算完ROV通知后端接口地址
     NOTIFY_BACKEND_UPDATE_ROV_SCORE_URL = 'http://videopre-internal.piaoquantv.com/longvideoapi/openapi/recommend/updateRovScore'
 
+    # logs 上传oss 目标Bucket指定目录
+    OSS_FOLDER = 'rov-offline/pre/'
+
 
 class ProductionConfig(BaseConfig):
     """生产环境配置"""
@@ -153,6 +165,9 @@ class ProductionConfig(BaseConfig):
     # 计算完ROV通知后端接口地址
     NOTIFY_BACKEND_UPDATE_ROV_SCORE_URL = 'http://longvideoapi-internal.piaoquantv.com/longvideoapi/openapi/recommend/updateRovScore'
 
+    # logs 上传oss 目标Bucket指定目录
+    OSS_FOLDER = 'rov-offline/pro/'
+
 
 def set_config():
     return DevelopmentConfig()

pool_predict.py

@@ -1,4 +1,5 @@
 import time
+import os
 
 from config import set_config
 from utils import request_post, filter_video_status
@@ -115,4 +116,7 @@ if __name__ == '__main__':
         predict(app_type=app_type)
         log_.info('{} predict end...'.format(app_name))
     log_.info('flow pool predict end...')
+    # 将日志上传到oss
+    log_cmd = "ossutil cp -r {} oss://{}/{}".format("/data/rov-offline/logs", config_.BUCKET_NAME, config_.OSS_FOLDER)
+    os.system(log_cmd)
 

rov_data_check.py

@@ -12,6 +12,12 @@ config_ = set_config()
 def rov_train_recall_pool_update():
     # 训练数据和预测数据都准备好时，更新模型，预测
     os.system('sh /data/rov-offline/rov_train_recall_pool_update.sh')
+    # 将日志上传到oss
+    log_cmd = "ossutil cp -r {} oss://{}/{}".format("/data/rov-offline/logs", config_.BUCKET_NAME, config_.OSS_FOLDER)
+    os.system(log_cmd)
+    # 将data上传到oss
+    data_cmd = "ossutil cp -r {} oss://{}/{}".format("/data/rov-offline/data", config_.BUCKET_NAME, config_.OSS_FOLDER)
+    os.system(data_cmd)
 
 
 def data_check(project, table, date):