风险视频注入redis

alg_recsys_recall_hour_region_task.sh

@@ -22,8 +22,5 @@ elif [[ $ROV_OFFLINE_ENV == 'pro' ]]; then
   /root/anaconda3/bin/python alg_recsys_recall_24h_noregion.py > "logs_dir/alg_recsys_recall_24h_noregion_{$cur_time}_{$cur_h}.log" &
   /root/anaconda3/bin/python alg_recsys_recall_24h_region.py > "logs_dir/alg_recsys_recall_24h_region_{$cur_time}_{$cur_h}.log"
   wait
-  echo "并行执行时间:{$(date "+%Y-%m-%d %H:%M:%S")}"
   /root/anaconda3/bin/python alg_recsys_recall_1h_region.py '24h' > "logs_dir/alg_recsys_recall_1h_region_{$cur_time}_{$cur_h}.log"
-  echo "结束执行时间:{$(date "+%Y-%m-%d %H:%M:%S")}"
-  echo "all done"
 fi

alg_recsys_recall_shield_videos.py

@@ -0,0 +1,42 @@
+import traceback
+from config import set_config
+from log import Log
+from utils import execute_sql_from_odps
+from db_helper import RedisHelper
+
+config_, _ = set_config()
+log_ = Log()
+redis_helper = RedisHelper()
+
+table = "loghubods.special_area_recommend_limit"
+
+RISK_SHIELD_FILTER_VIDEO_V1_STR = "RISK_SHIELD_FILTER_VIDEO_V1_STR"
+
+def get_special_area_limit_videos():
+    """获取特殊地域屏蔽视频并存入redis"""
+    try:
+        # 获取特殊地域屏蔽视频
+        sql = f"SELECT videoid FROM {config_.PROJECT_SPECIAL_AREA_LIMIT}.{config_.TABLE_SPECIAL_AREA_LIMIT};"
+        records = execute_sql_from_odps(project=config_.PROJECT_SPECIAL_AREA_LIMIT, sql=sql)
+        video_id_list = []
+        with records.open_reader() as reader:
+            for record in reader:
+                video_id = int(record['videoid'])
+                video_id_list.append(video_id)
+        log_.info(f"special area limit videos count = {len(video_id_list)}")
+        # 存入redis
+        if len(video_id_list) > 0:
+            redis_helper.del_keys(key_name=RISK_SHIELD_FILTER_VIDEO_V1_STR)
+            redis_helper.add_data_with_set(key_name=RISK_SHIELD_FILTER_VIDEO_V1_STR, values=video_id_list,
+                                           expire_time=25 * 3600)
+            # redis_helper.del_keys(key_name=config_.SPECIAL_AREA_LIMIT_KEY_NAME)
+            # redis_helper.add_data_with_set(key_name=config_.SPECIAL_AREA_LIMIT_KEY_NAME, values=video_id_list,
+            #                                expire_time=25 * 3600)
+    except Exception as e:
+        log_.error(str(e) + str(traceback.format_exc()))
+
+def main():
+    get_special_area_limit_videos()
+
+if __name__ == '__main__':
+    main()

config.py

@@ -2780,7 +2780,7 @@ def set_config():
     # 获取环境变量 ROV_OFFLINE_ENV
     env = os.environ.get('ROV_OFFLINE_ENV')
     print("ROV_OFFLINE_ENV:{}".format(str(env)))
-    # env = 'dev'
+    env = 'dev'
     if env is None:
         # log_.error('ENV ERROR: is None!')
         return

zhangbo/redis_test.py

@@ -0,0 +1,65 @@
+from utils import RedisHelper
+from config import set_config
+from log import Log
+config_, _ = set_config()
+log_ = Log()
+redis_helper = RedisHelper()
+
+if __name__ == '__main__':
+
+    # string
+    # RISK_SHIELD_FILTER_RULE_V1_JSON
+    # RISK_SHIELD_FILTER_VIDEO_V1_STR
+    key = "RISK_SHIELD_FILTER_RULE_V1_JSON"
+    value = "{\"0\": [\"110000\"]}"
+    redis_helper.set_data_to_redis(key, value, 7200)
+    value_get = redis_helper.get_data_from_redis(key_name=key)
+    print('key:', key, type(key))
+    print('value_get:', value_get, type(value_get))
+
+    key = "RISK_SHIELD_FILTER_VIDEO_V1_STR"
+    value = "7536230,1,2,3,4,5,6,7,8,9,10"
+    redis_helper.set_data_to_redis(key, value, 7200)
+    value_get = redis_helper.get_data_from_redis(key_name=key)
+    print('key:', key, type(key))
+    print('value_get:', value_get, type(value_get))
+
+    # redis_helper.del_keys(key_name=key)
+
+    # python zhangbo/redis_test.py
+
+
+    # dt = '20231203'
+    # data_key = 'test_lr_v1'
+    # key_name = f"{config_.KEY_NAME_PREFIX_AD_GROUP}{data_key}:{dt}"
+    # redis_data = {
+    #     'yxh': 0.3,
+    #     'zb': 0.13213231231,
+    # }
+    # redis_data = {
+    #     'mz': 0.2,
+    #     'zb': 0.6666,
+    # }
+    # print(key_name)
+    # #redis_helper.add_data_with_zset(key_name=key_name, data=redis_data, expire_time=2 * 24 * 3600)
+    #
+    # print(redis_helper.get_score_with_value(key_name=key_name, value='zb'))
+    # print(redis_helper.get_score_with_value(key_name=key_name, value='mz'))
+    # print(redis_helper.get_score_with_value(key_name='ad:users:group:predict:share:rate:test_lr_v1:20231129', value='16537775'))
+    #
+    # model_key = 'ad_out_v1'
+    # KEY_NAME_PREFIX_AD_OUT_MODEL_CONFIG = 'ad:out:model:config:'
+    # abtest_id = '173'
+    # abtest_config_tag = 'u'
+    # config_key_prefix = f"{KEY_NAME_PREFIX_AD_OUT_MODEL_CONFIG}{model_key}:{abtest_id}:{abtest_config_tag}"
+    # threshold_key = f"{config_key_prefix}:threshold"
+    # use_mean_key = f"{config_key_prefix}:use_mean"
+    # threshold = redis_helper.get_data_from_redis(key_name=threshold_key)
+    # use_mean = redis_helper.get_data_from_redis(key_name=use_mean_key)
+    # print('threshold:', threshold, type(threshold))
+    # print('use_mean:', use_mean, type(use_mean))
+    # expire_time = 30 * 24 * 3600
+    # redis_helper.set_data_to_redis(threshold_key, 0.4, expire_time)
+    # redis_helper.set_data_to_redis(use_mean_key, 1, expire_time)
+    # print(threshold_key)
+    # print(use_mean_key)