Merge branch 'feature/zhangbo_flow_recall' of algorithm/rov-offline into master

alg_recsys_coldstart_offlinecheck.py

@@ -0,0 +1,66 @@
+# -*- coding: utf-8 -*-
+import time
+import traceback
+from config import set_config
+from log import Log
+from utils import execute_sql_from_odps
+from db_helper import RedisHelper
+from datetime import datetime, timedelta
+
+from alg_recsys_recall_4h_region_trend import records_process_for_list
+config_, _ = set_config()
+log_ = Log()
+redis_helper = RedisHelper()
+
+
+
+def main():
+    date_write = ""
+    while True:
+        date_cur = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        if "2024-05-07 20:3" in date_cur:
+            print("退出")
+            break
+        # if date_write == date_cur:
+        #     time.sleep(10)
+        #     continue
+
+        for level in ["1", "2", "3", "4"]:
+            key = "flow:pool:level:item:4:" + level
+            result = redis_helper.get_data_from_set(key)
+            if not result:
+                result = []
+            size = len(result)
+            log_str = "\t".join([date_cur, "4", level, str(size)])
+            # print(log_str)
+            log_.info(log_str)
+        for level in ["1", "2", "3", "4"]:
+            key = "flow:pool:level:item:0:" + level
+            result = redis_helper.get_data_from_set(key)
+            if not result:
+                result = []
+            size = len(result)
+            log_str = "\t".join([date_cur, "0", level, str(size)])
+            # print(log_str)
+            log_.info(log_str)
+        for level in ["1", "2", "3", "4"]:
+            key = "flow:pool:level:item:21:" + level
+            result = redis_helper.get_data_from_set(key)
+            if not result:
+                result = []
+            size = len(result)
+            log_str = "\t".join([date_cur, "21", level, str(size)])
+            # print(log_str)
+            log_.info(log_str)
+        time.sleep(30)
+        # date_write = datetime.now().strftime("%Y-%m-%d %H:%M")
+
+
+if __name__ == '__main__':
+    main()
+
+
+
+
+# cd /root/zhangbo/rov-offline
+# nohup python alg_recsys_coldstart_offlinecheck.py > p.log 2>&1 &

flowpool_data_update_with_level.py

@@ -29,9 +29,7 @@ def get_videos_from_flow_pool(app_type, size=1000):
     request_data = {'appType': app_type, 'batchFlag': batch_flag, 'size': size}
     videos = []
     retry = 0
-    print("zhangbo1")
     while True:
-        print("zhangbo2")
         print(config_.GET_VIDEOS_FROM_POOL_URL)
         result = request_post(request_url=config_.GET_VIDEOS_FROM_POOL_URL, request_data=request_data)
         if result is None:
@@ -293,6 +291,20 @@ def get_data_from_odps(project, sql):
         data_df = None
     return data_df
 
+def print_check_log(s):
+    log_.info(s)
+    apptype_list = ["0", "4", "5", "21", "3", "6"]
+    level_list = ["1", "2", "3", "4"]
+    date_cur = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+    for app in apptype_list:
+        for level in level_list:
+            key =  "flow:pool:level:item:" + app + ":" + level
+            result = redis_helper.get_data_from_set(key)
+            if not result:
+                result = []
+            size = len(result)
+            log_str = "\t".join([date_cur, app, level, str(size)])
+            log_.info(log_str)
 
 if __name__ == '__main__':
     st_time = time.time()
@@ -322,8 +334,10 @@ if __name__ == '__main__':
 
     # 更新剩余分发数
     log_.info(f"video_info_list count = {len(video_info_list)}")
+    print_check_log("--------------------前序日志--------------------")
     get_videos_remain_view_count(video_info_list)
     log_.info('flow pool predict end...')
     log_.info(f"expend time = {(time.time() - st_time) * 1000}ms")
+    print_check_log("--------------------后续日志--------------------")
 
 # python flowpool_data_update_with_level.py    测试环境必须手动执行python 才能有数据