add benshanzhufu

benshanzhufu_videos.py

@@ -0,0 +1,35 @@
+import traceback
+from config import set_config
+from log import Log
+from utils import execute_sql_from_odps
+from db_helper import RedisHelper
+
+config_, _ = set_config()
+log_ = Log()
+redis_helper = RedisHelper()
+
+
+def get_benshanzhufu_videos():
+    """获取本山祝福视频并存入redis"""
+    try:
+        # 获取本山祝福视频
+        sql = f"SELECT videoid FROM {config_.PROJECT_BENSHAN_ZHUFU}.{config_.TABLE_BENSHAN_ZHUFU};"
+        records = execute_sql_from_odps(project=config_.PROJECT_BENSHAN_ZHUFU, sql=sql)
+        video_id_list = []
+        with records.open_reader() as reader:
+            for record in reader:
+                video_id = int(record['videoid'])
+                video_id_list.append(video_id)
+        log_.info(f"count = {len(video_id_list)}")
+        # 存入redis
+        if len(video_id_list) > 0:
+            redis_helper.del_keys(key_name=config_.BENSHAN_ZHUFU_KEY_NAME)
+            redis_helper.add_data_with_set(key_name=config_.BENSHAN_ZHUFU_KEY_NAME, values=video_id_list,
+                                           expire_time=25 * 3600)
+
+    except Exception as e:
+        log_.error(traceback.format_exc())
+
+
+if __name__ == '__main__':
+    get_benshanzhufu_videos()

config.py

@@ -556,6 +556,10 @@ class BaseConfig(object):
     # 本山祝福视频列表，在北京+重庆+无地域划分表中屏蔽
     PROJECT_BENSHAN_ZHUFU = 'loghubods'
     TABLE_BENSHAN_ZHUFU = 'benshanzhufu_videolist'
+    # 本山祝福视频redis存储key
+    BENSHAN_ZHUFU_KEY_NAME = 'com.weiqu.video.benshanzf'
+    # 本山祝福视频 过滤 地域
+    BENSHAN_ZHUFU_REGION_CODE = ['110000', '500000', '-1']
 
 
 class DevelopmentConfig(BaseConfig):

region_rule_rank_h.py

@@ -11,7 +11,7 @@ import math
 from functools import reduce
 from odps import ODPS
 from threading import Timer
-from utils import MysqlHelper, RedisHelper, get_data_from_odps, filter_video_status
+from utils import MysqlHelper, RedisHelper, get_data_from_odps, filter_video_status, filter_benshanzf_video
 from config import set_config
 from log import Log
 from check_video_limit_distribute import update_limit_video_score
@@ -157,6 +157,11 @@ def video_rank(df, now_date, now_h, rule_key, param, region, app_type, data_key)
     filtered_videos = filter_video_status(h_recall_videos)
     log_.info('filtered_videos count = {}'.format(len(filtered_videos)))
 
+    # 本山祝福视频过滤
+    if region in config_.BENSHAN_ZHUFU_REGION_CODE:
+        filtered_videos = filter_benshanzf_video(video_ids=filtered_videos)
+        log_.info(f"benshanzhufu filtered_videos count = {len(filtered_videos)}")
+
     # 写入对应的redis
     h_video_ids = []
     h_recall_result = {}
@@ -211,9 +216,16 @@ def dup_to_redis(h_video_ids, now_date, now_h, rule_key, region_24h_rule_key, re
     if redis_helper.key_exists(key_name=region_24h_key_name):
         region_24h_data = redis_helper.get_all_data_from_zset(key_name=region_24h_key_name, with_scores=True)
         log_.info(f'region 24h data count = {len(region_24h_data)}')
+
+        # 本山祝福视频过滤
+        region_24h_video_ids = [int(video_id) for video_id, _ in region_24h_data]
+        if region in config_.BENSHAN_ZHUFU_REGION_CODE:
+            region_24h_video_ids = filter_benshanzf_video(video_ids=region_24h_video_ids)
+            log_.info(f"benshanzhufu filtered_videos count = {len(region_24h_video_ids)}")
+
         region_24h_dup = {}
         for video_id, score in region_24h_data:
-            if int(video_id) not in h_video_ids:
+            if int(video_id) not in h_video_ids and int(video_id) in region_24h_video_ids:
                 region_24h_dup[int(video_id)] = score
                 h_video_ids.append(int(video_id))
         log_.info(f"region 24h data dup count = {len(region_24h_dup)}")
@@ -251,9 +263,16 @@ def dup_to_redis(h_video_ids, now_date, now_h, rule_key, region_24h_rule_key, re
     if redis_helper.key_exists(key_name=day_key_name):
         day_data = redis_helper.get_all_data_from_zset(key_name=day_key_name, with_scores=True)
         log_.info(f'24h data count = {len(day_data)}')
+
+        day_video_ids = [int(video_id) for video_id, _ in day_data]
+        # 本山祝福视频过滤
+        if region in config_.BENSHAN_ZHUFU_REGION_CODE:
+            day_video_ids = filter_benshanzf_video(video_ids=day_video_ids)
+            log_.info(f"benshanzhufu filtered_videos count = {len(day_video_ids)}")
+
         day_dup = {}
         for video_id, score in day_data:
-            if int(video_id) not in h_video_ids:
+            if int(video_id) not in h_video_ids and int(video_id) in day_video_ids:
                 day_dup[int(video_id)] = score
                 h_video_ids.append(int(video_id))
         log_.info(f"24h data dup count = {len(day_dup)}")
@@ -271,9 +290,14 @@ def dup_to_redis(h_video_ids, now_date, now_h, rule_key, region_24h_rule_key, re
     model_key_name = get_rov_redis_key(now_date=now_date)
     model_data = redis_helper.get_all_data_from_zset(key_name=model_key_name, with_scores=True)
     log_.info(f'model data count = {len(model_data)}')
+    model_video_ids = [int(video_id) for video_id, _ in model_data]
+    # 本山祝福视频过滤
+    if region in config_.BENSHAN_ZHUFU_REGION_CODE:
+        model_video_ids = filter_benshanzf_video(video_ids=model_video_ids)
+        log_.info(f"benshanzhufu filtered_videos count = {len(model_video_ids)}")
     model_data_dup = {}
     for video_id, score in model_data:
-        if int(video_id) not in h_video_ids:
+        if int(video_id) not in h_video_ids and int(video_id) in model_video_ids:
             model_data_dup[int(video_id)] = score
             h_video_ids.append(int(video_id))
     log_.info(f"model data dup count = {len(model_data_dup)}")

utils.py

@@ -290,6 +290,25 @@ def filter_video_status_app(video_ids):
     return filtered_videos
 
 
+def filter_benshanzf_video(video_ids):
+    """
+    过滤本山祝福视频
+    :param video_ids: 需过滤的视频列表 type-list
+    :return: filtered_videos  过滤后的列表  type-list
+    """
+    if len(video_ids) == 0:
+        return video_ids
+    # 根据Redis缓存中的数据过滤
+    redis_helper = RedisHelper()
+    # key拼接
+    benshanzf_videos_list = redis_helper.get_data_from_set(key_name=config_.BENSHAN_ZHUFU_KEY_NAME)
+    if not benshanzf_videos_list:
+        return video_ids
+    benshanzf_videos = [int(video) for video in benshanzf_videos_list]
+    filtered_videos = [video_id for video_id in video_ids if video_id not in benshanzf_videos]
+    return filtered_videos
+
+
 def update_video_w_h_rate(video_ids, key_name):
     """
     获取横屏视频的宽高比，并存入redis中 （width/height>1)