523同步策略

alg_recsys_recall_24h_noregion.py

@@ -18,8 +18,7 @@ log_ = Log()
 
 RULE_PARAMS = {
     'rule_params': {
-        'rule66': {'cal_score_func': 2, 'return_count': 100, 'platform_return_rate': 0.001,
-                  'view_type': 'preview'},
+        'rule66': {'cal_score_func': 2, 'return_count': 100, 'platform_return_rate': 0.001, 'view_type': 'preview'},
     },
     'data_params': config_.DATA_PARAMS,
     'params_list': [
@@ -186,7 +185,7 @@ def video_rank_h(df, now_date, now_h, rule_key, param, data_key, notify_backend)
     :return:
     """
     redis_helper = RedisHelper()
-    log_.info(f"videos_count = {len(df)}")
+    log_.info(f"一共有多少个视频 = {len(df)}")
 
     # videoid重复时，保留分值高
     df = df.sort_values(by=['score'], ascending=False)
@@ -199,49 +198,46 @@ def video_rank_h(df, now_date, now_h, rule_key, param, data_key, notify_backend)
         day_recall_df = df[df['回流人数'] > return_count]
     else:
         day_recall_df = df
+    log_.info(f"回流量-过滤后，一共有多少个视频 = {len(day_recall_df)}")
     platform_return_rate = param.get('platform_return_rate', 0)
     day_recall_df = day_recall_df[day_recall_df['platform_return_rate'] > platform_return_rate]
     day_recall_videos = day_recall_df['videoid'].to_list()
-    log_.info(f'h_by24h_recall videos count = {len(day_recall_videos)}')
+    log_.info(f"回流率-过滤后，一共有多少个视频 = {len(day_recall_videos)}")
     # 视频状态过滤
     if data_key in ['data7', ]:
         filtered_videos = filter_video_status_app(day_recall_videos)
     else:
         filtered_videos = filter_video_status(day_recall_videos)
-    log_.info('filtered_videos count = {}'.format(len(filtered_videos)))
+    log_.info(f"视频状态-过滤后，一共有多少个视频 = {len(filtered_videos)}")
 
     # 写入对应的redis
     now_dt = datetime.strftime(now_date, '%Y%m%d')
     day_video_ids = []
     day_recall_result = {}
-    # json_data = []
     for video_id in filtered_videos:
         score = day_recall_df[day_recall_df['videoid'] == video_id]['score']
         day_recall_result[int(video_id)] = float(score)
         day_video_ids.append(int(video_id))
-        # json_data.append({'videoId': video_id, 'rovScore': float(score)})
 
+    # recall:item:score:24h:
     h_24h_recall_key_name = \
         f"{config_.RECALL_KEY_NAME_PREFIX_BY_24H}{data_key}:{rule_key}:{now_dt}:{now_h}"
-    log_.info("h_24h_recall_key_name:redis:{}".format(h_24h_recall_key_name))
+    log_.info("打印非地域24小时redis key:{}".format(h_24h_recall_key_name))
     if len(day_recall_result) > 0:
-        log_.info(f"count = {len(day_recall_result)}, key = {h_24h_recall_key_name}")
+        log_.info(f"开始写入头部数据：count = {len(day_recall_result)}, key = {h_24h_recall_key_name}")
         redis_helper.add_data_with_zset(key_name=h_24h_recall_key_name, data=day_recall_result, expire_time=2 * 3600)
-        # 清空线上过滤应用列表
-        # redis_helper.del_keys(key_name=f"{config_.H_VIDEO_FILER_24H}{app_type}.{data_key}.{rule_key}")
+    else:
+        log_.info(f"无数据，不写入。")
 
-    # 去重筛选结果，保留剩余数据并写入Redis
+    # ---------------处理剩余结果---------------
+    log_.info('开始处理剩余结果other')
     all_videos = df['videoid'].to_list()
-    log_.info(f'h_by24h_recall all videos count = {len(all_videos)}')
-    # 视频状态过滤
     if data_key in ['data7', ]:
         all_filtered_videos = filter_video_status_app(all_videos)
     else:
         all_filtered_videos = filter_video_status(all_videos)
-    log_.info(f'all_filtered_videos count = {len(all_filtered_videos)}')
-    # 与筛选结果去重
     other_videos = [video for video in all_filtered_videos if video not in day_video_ids]
-    log_.info(f'other_videos count = {len(other_videos)}')
+    log_.info(f'过滤后剩余视频数量 count = {len(other_videos)}')
     # 写入对应的redis
     other_24h_recall_result = {}
     json_data = []
@@ -249,18 +245,20 @@ def video_rank_h(df, now_date, now_h, rule_key, param, data_key, notify_backend)
         score = df[df['videoid'] == video_id]['score']
         other_24h_recall_result[int(video_id)] = float(score)
         json_data.append({'videoId': video_id, 'rovScore': float(score)})
-    # other_h_24h_recall_key_name = \
-    #     f"{config_.RECALL_KEY_NAME_PREFIX_BY_24H_OTHER}{app_type}:{data_key}:{rule_key}:{now_dt}:{now_h}"
+    # recall:item:score:24h:other:
     other_h_24h_recall_key_name = \
         f"{config_.RECALL_KEY_NAME_PREFIX_BY_24H_OTHER}{data_key}:{rule_key}:{now_dt}:{now_h}"
+    log_.info("打印非地域24小时（剩余）redis key:{}".format(other_h_24h_recall_key_name))
     if len(other_24h_recall_result) > 0:
-        log_.info(f"count = {len(other_24h_recall_result)}")
+        log_.info(f"开始写入尾部数据：count = {len(other_24h_recall_result)}, key = {other_h_24h_recall_key_name}")
         redis_helper.add_data_with_zset(key_name=other_h_24h_recall_key_name, data=other_24h_recall_result,
                                         expire_time=2 * 3600)
+    else:
+        log_.info(f"无尾部数据，不写入。")
+
     # 通知后端更新兜底视频数据
     if notify_backend is True:
         log_.info('json_data count = {}'.format(len(json_data[:5000])))
-        # log_.info(f"json_data = {json_data}")
         result = request_post(request_url=config_.NOTIFY_BACKEND_updateFallBackVideoList_URL,
                               request_data={'videos': json_data[:5000]})
         if result is None:
@@ -322,38 +320,17 @@ def rank_by_h(now_date, now_h, rule_params, project, table):
     # rank
     data_params_item = rule_params.get('data_params')
     rule_params_item = rule_params.get('rule_params')
-    """
-    for param in rule_params.get('params_list'):
-        data_key = param.get('data')
-        data_param = data_params_item.get(data_key)
-        log_.info(f"data_key = {data_key}, data_param = {data_param}")
-        df_list = [feature_df[feature_df['apptype'] == apptype] for apptype in data_param]
-        df_merged = reduce(merge_df, df_list)
-
-        rule_key = param.get('rule')
-        rule_param = rule_params_item.get(rule_key)
-        log_.info(f"rule_key = {rule_key}, rule_param = {rule_param}")
-        # 计算score
-        cal_score_func = rule_param.get('cal_score_func', 1)
-        if cal_score_func == 2:
-            score_df = cal_score2(df=df_merged, param=rule_param)
-        else:
-            score_df = cal_score1(df=df_merged)
-        video_rank_h(df=score_df, now_date=now_date, now_h=now_h,
-                     rule_key=rule_key, param=rule_param, data_key=data_key)
-    """
 
     for param in rule_params.get('params_list'):
         score_df_list = []
         notify_backend = param.get('notify_backend', False)
         data_key = param.get('data')
         data_param = data_params_item.get(data_key)
-        log_.info(f"data_key = {data_key}, data_param = {data_param}")
         rule_key = param.get('rule')
         rule_param = rule_params_item.get(rule_key)
-        log_.info(f"rule_key = {rule_key}, rule_param = {rule_param}")
-        # cal_score_func = rule_param.get('cal_score_func', 1)
         merge_func = rule_param.get('merge_func', 1)
+        log_.info("数据采用:{},统计采用{}.".format(data_key, rule_key))
+        log_.info("具体的规则是:{}.".format(rule_param))
 
         if merge_func == 2:
             for apptype, weight in data_param.items():
@@ -417,29 +394,7 @@ def h_rank_bottom(now_date, now_h, rule_params):
             if len(final_data) > 0:
                 redis_helper.add_data_with_zset(key_name=final_key_name, data=final_data, expire_time=2 * 3600)
 
-    """
-    for app_type, params in rule_params.items():
-        log_.info(f"app_type = {app_type}")
-        for param in params.get('params_list'):
-            data_key = param.get('data')
-            rule_key = param.get('rule')
-            log_.info(f"data_key = {data_key}, rule_key = {rule_key}")
-            for key_prefix in key_prefix_list:
-                key_name = f"{key_prefix}{app_type}:{data_key}:{rule_key}:{redis_dt}:{redis_h}"
-                initial_data = redis_helper.get_all_data_from_zset(key_name=key_name, with_scores=True)
-                if initial_data is None:
-                    initial_data = []
-                final_data = dict()
-                for video_id, score in initial_data:
-                    final_data[video_id] = score
-                # 存入对应的redis
-                final_key_name = \
-                    f"{key_prefix}{app_type}:{data_key}:{rule_key}:{datetime.strftime(now_date, '%Y%m%d')}:{now_h}"
-                if len(final_data) > 0:
-                    redis_helper.add_data_with_zset(key_name=final_key_name, data=final_data, expire_time=2 * 3600)
-                # 清空线上过滤应用列表
-                # redis_helper.del_keys(key_name=f"{config_.H_VIDEO_FILER_24H}{app_type}.{data_key}.{rule_key}")
-    """
+
 
 def h_timer_check():
     try:
@@ -447,26 +402,26 @@ def h_timer_check():
         table = config_.TABLE_24H_APP_TYPE
         rule_params = RULE_PARAMS
         now_date = datetime.today()
-        log_.info(f"now_date: {datetime.strftime(now_date, '%Y%m%d%H')}")
+        log_.info(f"开始执行: {datetime.strftime(now_date, '%Y%m%d%H')}")
         now_min = datetime.now().minute
         now_h = datetime.now().hour
         # 查看当前天级更新的数据是否已准备好
         h_data_count = h_data_check(project=project, table=table, now_date=now_date, now_h=now_h)
         if now_h == 23 or now_h < 8:
-            log_.info(f'now_h = {now_h} use bottom data!')
+            log_.info("当前时间{}小时，使用bottom的data，开始。".format(now_h))
             h_rank_bottom(now_date=now_date, now_h=now_h, rule_params=rule_params)
-            log_.info(f"24h_data end!")
+            log_.info("当前时间{}小时，使用bottom的data，完成。".format(now_h))
         elif h_data_count > 0:
-            log_.info(f'h_by24h_data_count = {h_data_count}')
-            # 数据准备好，进行更新
+            log_.info('上游数据表查询数据条数 h_data_count = {}，开始计算。'.format(h_data_count))
             rank_by_h(now_date=now_date, now_h=now_h, rule_params=rule_params, project=project, table=table)
-            log_.info(f"24h_data end!")
+            log_.info("----------正常完成----------")
         elif now_min > 40:
-            log_.info('h_by24h_recall data is None, use bottom data!')
+            log_.info('当前分钟超过40，预计执行无法完成，使用 bottom data!')
             h_rank_bottom(now_date=now_date, now_h=now_h, rule_params=rule_params)
-            log_.info(f"24h_data end!")
+            log_.info('当前分钟超过40，完成。')
         else:
             # 数据没准备好，1分钟后重新检查
+            log_.info("上游数据未就绪，等待...")
             Timer(60, h_timer_check).start()
 
     except Exception as e:
@@ -481,5 +436,6 @@ def h_timer_check():
 
 
 if __name__ == '__main__':
-    log_.info(f"24h_data start...")
+    log_.info("文件alg_recsys_recall_24h_noregion.py:「24小时无地域」 开始执行")
     h_timer_check()
+    log_.info("文件alg_recsys_recall_24h_noregion.py:「24小时无地域」 执行完毕")

config.py

@@ -277,7 +277,25 @@ class BaseConfig(object):
         },
         'data_params': DATA_PARAMS,
         'params_list': [
-
+            {'data': 'data1', 'rule': 'rule3', 'notify_backend': True},
+            # {'data': 'data2', 'rule': 'rule3'},
+            {'data': 'data2', 'rule': 'rule4'},
+            # {'data': 'data3', 'rule': 'rule4'},
+            # {'data': 'data4', 'rule': 'rule4'},
+            # {'data': 'data6', 'rule': 'rule4'},
+            {'data': 'data7', 'rule': 'rule4'},
+            # {'data': 'data1', 'rule': 'rule5'},
+            # {'data': 'data1', 'rule': 'rule6'},
+            # {'data': 'data8', 'rule': 'rule4'},
+            # {'data': 'data9', 'rule': 'rule4'},
+            {'data': 'data10', 'rule': 'rule4'},
+            # {'data': 'data11', 'rule': 'rule4'},
+            # {'data': 'data12', 'rule': 'rule4'},
+            # {'data': 'data13', 'rule': 'rule4'},
+            # # {'data': 'data14', 'rule': 'rule4'},
+            # {'data': 'data1', 'rule': 'rule7'},
+            # {'data': 'data1', 'rule': 'rule8'},
+            {'data': 'videos5', 'rule': 'rule4'},  # [内容精选]
             {'data': 'data66', 'rule': 'rule66'},
         ]
     }
@@ -312,7 +330,25 @@ class BaseConfig(object):
         },
         'data_params': DATA_PARAMS,
         'params_list': [
-
+            {'data': 'data1', 'rule': 'rule2'},
+            # {'data': 'data2', 'rule': 'rule2'},
+            {'data': 'data2', 'rule': 'rule4'},
+            # {'data': 'data3', 'rule': 'rule4'},
+            # {'data': 'data4', 'rule': 'rule4'},
+            # {'data': 'data6', 'rule': 'rule4'},
+            {'data': 'data7', 'rule': 'rule5'},
+            # {'data': 'data1', 'rule': 'rule6'},
+            # {'data': 'data1', 'rule': 'rule7'},
+            # {'data': 'data8', 'rule': 'rule4'},
+            # {'data': 'data9', 'rule': 'rule4'},
+            {'data': 'data10', 'rule': 'rule4'},
+            # {'data': 'data11', 'rule': 'rule4'},
+            # {'data': 'data12', 'rule': 'rule4'},
+            # {'data': 'data13', 'rule': 'rule4'},
+            # {'data': 'data14', 'rule': 'rule4'},
+            # {'data': 'data1', 'rule': 'rule8'},
+            # {'data': 'data1', 'rule': 'rule9'},
+            {'data': 'videos5', 'rule': 'rule4'},  # [内容精选]
             {'data': 'data66', 'rule': 'rule66'},
         ]
     }
@@ -328,12 +364,10 @@ class BaseConfig(object):
             # backrate = lastonehour_return / (lastonehour_share + 10)
             # ctr = lastonehour_play / (lastonehour_show + 1000), 对ctr限最大值：K2 = 0.6 if ctr > 0.6 else ctr
             'rule1': {'view_type': 'video-show', 'platform_return_rate': 0.001, 'merge_func': 2},
-            'rule66': {'view_type': 'video-show', 'platform_return_rate': 0.001},
         },
         'data_params': DATA_PARAMS,
         'params_list': [
-
-            {'data': 'data66', 'rule': 'rule66'},
+            {'data': 'data10', 'rule': 'rule1'},
         ],
     }
 
@@ -459,27 +493,48 @@ class BaseConfig(object):
                 'view_type': 'video-show-region', 'platform_return_rate': 0.001,
                 'region_24h_rule_key': 'rule66', '24h_rule_key': 'rule66'
             },
-            'rule67': {
-                'view_type': 'video-show-region', 'platform_return_rate': 0.001,
-                'region_24h_rule_key': 'rule66', '24h_rule_key': 'rule66', 'h_rule_key': 'rule66'
-            },
-            'rule68': {
-                'view_type': 'video-show-region', 'platform_return_rate': 0.001,
-                'region_24h_rule_key': 'rule66', '24h_rule_key': 'rule66',
-                'score_func': 'back_rate_exponential_weighting1'
-            },
-            'rule69': {
-                'view_type': 'video-show-region', 'platform_return_rate': 0.001,
-                'region_24h_rule_key': 'rule66', '24h_rule_key': 'rule66',
-            },
         },
         'data_params': DATA_PARAMS,
         'params_list': [
-
-            {'data': 'data66', 'rule': 'rule66'}, # 523
-            {'data': 'data66', 'rule': 'rule67'},  # 523->510
-            {'data': 'data66', 'rule': 'rule68'},  # 523->514
-            {'data': 'data66', 'rule': 'rule69'},  # 523->518
+            {'data': 'data1', 'rule': 'rule4'},  # 095 vlog
+            {'data': 'data1', 'rule': 'rule4-1'},  # 095-1
+            # {'data': 'data1', 'rule': 'rule4-2'},  # 262 特殊地域屏蔽危险视频
+            # {'data': 'data2', 'rule': 'rule4'},
+            {'data': 'data2', 'rule': 'rule7-1'},  # 121 内容精选
+            # {'data': 'data3', 'rule': 'rule7'},
+            # {'data': 'data4', 'rule': 'rule7'},
+            # {'data': 'data6', 'rule': 'rule7'},
+            {'data': 'data7', 'rule': 'rule8'},  # 票圈视频APP 10003.110156
+            # {'data': 'data1', 'rule': 'rule9'},
+            # {'data': 'data1', 'rule': 'rule10'},
+            # {'data': 'data1', 'rule': 'rule11'},
+            # {'data': 'data8', 'rule': 'rule7'},
+            # {'data': 'data9', 'rule': 'rule7'},
+            {'data': 'data10', 'rule': 'rule7'},  # 144 票圈视频
+            # {'data': 'data11', 'rule': 'rule7'},
+            # {'data': 'data12', 'rule': 'rule7'},
+            # {'data': 'data13', 'rule': 'rule7'},
+            # {'data': 'data1', 'rule': 'rule12'},
+            # {'data': 'data14', 'rule': 'rule7'},  # 159
+            # {'data': 'data1', 'rule': 'rule13'},  # 161
+            # {'data': 'data1', 'rule': 'rule14'},  # 162
+            # {'data': 'data1', 'rule': 'rule15'},  # 200 vlog
+            # {'data': 'data1', 'rule': 'rule16'},  # 214 vlog
+            # {'data': 'data1', 'rule': 'rule17'},  # 215 vlog
+            # {'data': 'data1', 'rule': 'rule18'},  # 224 vlog
+            {'data': 'videos5', 'rule': 'rule7-1'},  # 428 [内容精选]
+            # {'data': 'data1', 'rule': 'rule20'},  # 461 vlog 分值计算公式 增加h-2分享当前小时回流数据、h-3分享当前小时回流数据特征
+            # {'data': 'data1', 'rule': 'rule21'},  # 462 vlog 分值计算公式 增加[h-3,h-2]之间的回流留存特征
+            # {'data': 'data1', 'rule': 'rule22'},  # 463 vlog 分值计算公式 增加h-2分享当前小时回流/h-2分享、h-3分享当前小时回流/h-3分享 特征
+            # {'data': 'data1', 'rule': 'rule23'},  # 465 vlog 回流数据使用 分享限制地域，回流不限制地域 统计数据
+            # {'data': 'data1', 'rule': 'rule24'},  # 466 vlog 分值计算公式 增加[h-3,h-2]之间的回流留存特征 + 回流数据使用 分享限制地域，回流不限制地域 统计数据
+            # {'data': 'data10', 'rule': 'rule25'},  # 500
+            # {'data': 'data10', 'rule': 'rule26'},  # 501
+            {'data': 'data10', 'rule': 'rule27'},  # 502
+            {'data': 'data10', 'rule': 'rule28'},  # 503
+            # {'data': 'data10', 'rule': 'rule29'},  # 509
+            {'data': 'data10', 'rule': 'rule30'},  # 510
+            {'data': 'data66', 'rule': 'rule66'}, # 520
         ],
         'params_list_new': [
             # {'data': 'data10', 'rule': 'rule19'},  # 316 票圈视频 + 召回在线去重
@@ -806,10 +861,24 @@ class BaseConfig(object):
         'value': 1
     }
 
+    # 各召回池数据更新状态
+    RULE_24H_DATA_STATUS = 'rule:24h:data:status'
+    REGION_24H_DATA_STATUS = 'region:24h:data:status'
+    RULE_H_DATA_STATUS = 'rule:h:data:status'
+
+
     # ##################################### 广告模型配置 #####################################
 
     # 广告模型数据
     ad_model_data = {
+        'ad_out_v1': {
+            'project': 'loghubods',
+            'table': 'user_video_features_data_final'
+        },
+        'ad_out_v1_user': {
+            'project': 'loghubods',
+            'table': 'alg_recsys_user_info'
+        },
         'user_group': {
             'project': 'loghubods',
             'table': 'user_share_return_admodel'
@@ -1253,7 +1322,14 @@ class BaseConfig(object):
                   'user': {'data': 'user0out', 'rule': 'rule2'},
                   'care_model': True, 'threshold_rate': 0.7,
                   },  # 所有广告类型数据 + 优化阈值计算方式 + else未开启关怀模式人群多出广告 + 使用以是否直接跳出为目标的数据 + return25_nmids用户不出广告 + 所有用户组top3不出广告
-
+        '173-u': {'video': {'data': 'videos0out'},
+                  'user': {'data': 'user0out', 'rule': 'rule2'},
+                  'care_model': True, 'threshold_rate': 0.7,
+                  },  # else非关怀模式人群多出广告 + 所有广告类型数据 + 优化阈值计算方式 + 使用以是否直接跳出为目标的数据
+        '173-v': {'video': {'data': 'videos0out'},
+                  'user': {'data': 'user0out', 'rule': 'rule2'},
+                  'care_model': True, 'threshold_rate': 0.7,
+                  },  # else非关怀模式人群多出广告 + 所有广告类型数据 + 优化阈值计算方式 + 使用以是否直接跳出为目标的数据
         # 票圈视频+
         # '190-a': {'video': {'data': 'data1'},
         #           'user': {'data': 'user0', 'rule': 'rule1'}},
@@ -2411,7 +2487,15 @@ class DevelopmentConfig(BaseConfig):
 
     # 测试环境 过滤用mysql地址
     FILTER_MYSQL_INFO = {
-        'host': 'am-bp1g3ys9u00u483uc131930.ads.aliyuncs.com',
+        # 'host': 'am-bp1g3ys9u00u483uc131930.ads.aliyuncs.com',
+        # 'port': 3306,
+        # 'user': 'lv_manager',
+        # 'password': 'lv_manager@2020',
+        # 'db': 'longvideo',
+        # 'charset': 'utf8'
+
+        ##### test环境的filter mysql会过滤掉所有数据，测试时先使用pro的filter mysql。 注意测试结束后切换注释。
+        'host': 'am-bp15tqt957i3b3sgi131950.ads.aliyuncs.com',
         'port': 3306,
         'user': 'lv_manager',
         'password': 'lv_manager@2020',
@@ -2495,21 +2579,18 @@ class TestConfig(BaseConfig):
 
     # 测试环境 过滤用mysql地址
     FILTER_MYSQL_INFO = {
-        # 'host': 'am-bp1g3ys9u00u483uc131930.ads.aliyuncs.com',
-        # 'port': 3306,
-        # 'user': 'lv_manager',
-        # 'password': 'lv_manager@2020',
-        # 'db': 'longvideo',
-        # 'charset': 'utf8'
-
-        ##### test环境的filter mysql会过滤掉所有数据，测试时先使用pro的filter mysql。 注意测试结束后切换注释。
-
-        'host': 'am-bp15tqt957i3b3sgi131950.ads.aliyuncs.com',
+        'host': 'am-bp1g3ys9u00u483uc131930.ads.aliyuncs.com',
         'port': 3306,
         'user': 'lv_manager',
         'password': 'lv_manager@2020',
         'db': 'longvideo',
         'charset': 'utf8'
+        # 'host': 'am-bp15tqt957i3b3sgi131950.ads.aliyuncs.com',
+        # 'port': 3306,
+        # 'user': 'lv_manager',
+        # 'password': 'lv_manager@2020',
+        # 'db': 'longvideo',
+        # 'charset': 'utf8'
     }
 
     # 日志服务配置