add log score_df

config.py

@@ -1,7 +1,7 @@
 # coding:utf-8
 import os
-from log import Log
-log_ = Log()
+# from log import Log
+# log_ = Log()
 
 
 class BaseConfig(object):
@@ -241,6 +241,14 @@ class DevelopmentConfig(BaseConfig):
         'charset': 'utf8'
     }
 
+    # 日志服务配置
+    ALIYUN_LOG = {
+        'ENDPOINT': 'cn-hangzhou.log.aliyuncs.com',
+        'ACCESSID': 'LTAIWYUujJAm7CbH',
+        'ACCESSKEY': 'RfSjdiWwED1sGFlsjXv0DlfTnZTG1P',
+        'PROJECT': 'rov-server-test',
+    }
+
     # Hologres视频状态存储表名
     VIDEO_STATUS = 'longvideo_test.dwd_mdm_item_video_stat'
 
@@ -304,6 +312,14 @@ class TestConfig(BaseConfig):
         'charset': 'utf8'
     }
 
+    # 日志服务配置
+    ALIYUN_LOG = {
+        'ENDPOINT': 'cn-hangzhou.log.aliyuncs.com',
+        'ACCESSID': 'LTAIWYUujJAm7CbH',
+        'ACCESSKEY': 'RfSjdiWwED1sGFlsjXv0DlfTnZTG1P',
+        'PROJECT': 'rov-server-test',
+    }
+
     # Hologres视频状态存储表名
     VIDEO_STATUS = 'longvideo_test.dwd_mdm_item_video_stat'
 
@@ -367,6 +383,14 @@ class PreProductionConfig(BaseConfig):
         'charset': 'utf8'
     }
 
+    # 日志服务配置
+    ALIYUN_LOG = {
+        'ENDPOINT': 'cn-hangzhou.log.aliyuncs.com',
+        'ACCESSID': 'LTAIWYUujJAm7CbH',
+        'ACCESSKEY': 'RfSjdiWwED1sGFlsjXv0DlfTnZTG1P',
+        'PROJECT': 'rov-server',
+    }
+
     # Hologres视频状态存储表名
     VIDEO_STATUS = 'longvideo.dwd_mdm_item_video_stat'
 
@@ -430,6 +454,14 @@ class ProductionConfig(BaseConfig):
         'charset': 'utf8'
     }
 
+    # 日志服务配置
+    ALIYUN_LOG = {
+        'ENDPOINT': 'cn-hangzhou.log.aliyuncs.com',
+        'ACCESSID': 'LTAIWYUujJAm7CbH',
+        'ACCESSKEY': 'RfSjdiWwED1sGFlsjXv0DlfTnZTG1P',
+        'PROJECT': 'rov-server',
+    }
+
     # Hologres视频状态存储表名
     VIDEO_STATUS = 'longvideo.dwd_mdm_item_video_stat'
 
@@ -455,7 +487,7 @@ def set_config():
     env = os.environ.get('ROV_OFFLINE_ENV')
     # env = 'dev'
     if env is None:
-        log_.error('ENV ERROR: is None!')
+        # log_.error('ENV ERROR: is None!')
         return
     if env == 'dev':
         return DevelopmentConfig(), env
@@ -466,5 +498,5 @@ def set_config():
     elif env == 'pro':
         return ProductionConfig(), env
     else:
-        log_.error('ENV ERROR: is {}'.format(env))
+        # log_.error('ENV ERROR: is {}'.format(env))
         return

log.py

@@ -2,8 +2,43 @@
 import os
 import logging
 import time
+import logging.config
+from log_conf import conf
 
 
+class Log(object):
+    def __init__(self):
+        # 配置
+        logging.config.dictConfig(conf)
+
+    def __console(self, level, message):
+        if level == 'info':
+            logger = logging.getLogger('sls')
+            logger.info(message)
+        elif level == 'debug':
+            logger = logging.getLogger('root')
+            logger.debug(message)
+        elif level == 'warning':
+            logger = logging.getLogger('root')
+            logger.warning(message)
+        elif level == 'error':
+            logger = logging.getLogger('error')
+            logger.error(message)
+
+    def debug(self, message):
+        self.__console('debug', message)
+
+    def info(self, message):
+        self.__console('info', message)
+
+    def warning(self, message):
+        self.__console('warning', message)
+
+    def error(self, message):
+        self.__console('error', message)
+
+
+"""
 class Log(object):
     # def __init__(self, pag_source, log_path=os.path.join(os.path.dirname(os.path.realpath(__file__)), "logs")):
     def __init__(self, log_path=os.path.join(os.path.dirname(os.path.realpath(__file__)), "logs")):
@@ -57,3 +92,4 @@ class Log(object):
 
     def error(self, message):
         self.__console('error', message)
+"""

log_conf.py

@@ -0,0 +1,85 @@
+# log conf
+import logging
+import aliyun
+import os
+import time
+from config import set_config
+config_, _ = set_config()
+
+# 本地日志存储路径
+log_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "logs")
+if not os.path.exists(log_path):
+    os.makedirs(log_path)
+# 文件的命名
+log_name = os.path.join(log_path, '{}.log'.format(time.strftime('%Y%m%d')))
+
+conf = {
+    'version': 1,
+    'formatters': {
+        'rawFormatter': {
+            'class': 'logging.Formatter',
+            'format': '%(message)s'
+        },
+        'simpleFormatter': {
+            'class': 'logging.Formatter',
+            'format': '%(asctime)s %(levelname)s: %(message)s'
+        }
+    },
+    'handlers': {
+        'consoleHandler': {
+            '()': 'logging.StreamHandler',
+            'level': 'DEBUG',
+            'formatter': 'simpleFormatter',
+        },
+        'slsHandler': {
+            '()': 'aliyun.log.QueuedLogHandler',
+            'level': 'INFO',
+            'formatter': 'rawFormatter',
+            # custom args:
+            'end_point': config_.ALIYUN_LOG.get('ENDPOINT', ''),
+            'access_key_id': config_.ALIYUN_LOG.get('ACCESSID', ''),
+            'access_key': config_.ALIYUN_LOG.get('ACCESSKEY', ''),
+            'project': config_.ALIYUN_LOG.get('PROJECT', ''),
+            'log_store': "info",
+            'extract_kv': True,
+            'extract_json': True
+        },
+        'errorHandler': {
+            '()': 'aliyun.log.QueuedLogHandler',
+            'level': 'ERROR',
+            'formatter': 'rawFormatter',
+            # custom args:
+            'end_point': config_.ALIYUN_LOG.get('ENDPOINT', ''),
+            'access_key_id': config_.ALIYUN_LOG.get('ACCESSID', ''),
+            'access_key': config_.ALIYUN_LOG.get('ACCESSKEY', ''),
+            'project': config_.ALIYUN_LOG.get('PROJECT', ''),
+            'log_store': "error",
+            'extract_kv': True,
+            'extract_json': True
+        },
+        'fileHandler': {
+            '()': 'logging.FileHandler',
+            'level': 'INFO',
+            'formatter': 'simpleFormatter',
+            'filename': log_name,
+            'mode': 'a',
+            'encoding': 'utf-8'
+        }
+    },
+    'loggers': {
+        'root': {
+            'handlers': ['consoleHandler', ],
+            'level': 'DEBUG'
+        },
+        'sls': {
+            'handlers': ['consoleHandler', 'slsHandler', 'fileHandler'],
+            'level': 'INFO',
+            'propagate': False
+        },
+        'error': {
+            'handlers': ['consoleHandler', 'errorHandler', 'fileHandler'],
+            'level': 'ERROR',
+            'propagate': False
+        }
+    }
+}

region_rule_rank_h.py

@@ -216,6 +216,11 @@ def rank_by_h(project, table, now_date, now_h, rule_params, region_code_list):
             # to-csv
             score_filename = f"score_{region}_{key}_{datetime.datetime.strftime(now_date, '%Y%m%d%H')}.csv"
             score_df.to_csv(f'./data/{score_filename}')
+            # to-logs
+            log_.info({"region_code": region,
+                       "redis_key_prefix": config_.RECALL_KEY_NAME_PREFIX_REGION_BY_H,
+                       "rule_key": key,
+                       "score_df": score_df[['videoid', 'score']]})
 
 
 def h_rank_bottom(now_date, now_h, rule_key, region_code_list):

rule_rank_day.py

@@ -161,6 +161,10 @@ def rank_by_day(now_date, rule_params, project, table):
         # to-csv
         score_filename = f"score_{key}_{datetime.strftime(now_date, '%Y%m%d')}.csv"
         score_df.to_csv(f'./data/{score_filename}')
+        # to-logs
+        log_.info({"redis_key_prefix": config_.RECALL_KEY_NAME_PREFIX_BY_DAY,
+                   "rule_key": key,
+                   "score_df": score_df[['videoid', 'score']]})
 
 
 def day_rank_bottom(now_date, rule_key):

rule_rank_h.py

@@ -206,6 +206,10 @@ def rank_by_h(now_date, now_h, rule_params):
         # to-csv
         score_filename = f"score_{key}_{datetime.datetime.strftime(now_date, '%Y%m%d%H')}.csv"
         score_df.to_csv(f'./data/{score_filename}')
+        # to-logs
+        log_.info({"redis_key_prefix": config_.RECALL_KEY_NAME_PREFIX_BY_H,
+                   "rule_key": key,
+                   "score_df": score_df[['videoid', 'score']]})
 
 
 def h_rank_bottom(now_date, now_h, rule_key):