rov-offline/alg_recsys_recall_shield_videos.py

# -*- coding: utf-8 -*-
import traceback
from config import set_config
from log import Log
from utils import execute_sql_from_odps
from db_helper import RedisHelper
import datetime
import json
config_, _ = set_config()
log_ = Log()
redis_helper = RedisHelper()

table = "loghubods.special_area_recommend_limit"

RISK_SHIELD_FILTER_VIDEO_V1_STR = "RISK_SHIELD_FILTER_VIDEO_V1_STR"

def get_special_area_limit_videos():
    """获取特殊地域屏蔽视频并存入redis"""
    try:
        # 获取特殊地域屏蔽视频
        sql = "SELECT videoid FROM {}.{};".format(config_.PROJECT_SPECIAL_AREA_LIMIT,
                                                  config_.TABLE_SPECIAL_AREA_LIMIT)
        print("sql:"+sql)
        records = execute_sql_from_odps(project=config_.PROJECT_SPECIAL_AREA_LIMIT, sql=sql)
        video_id_list = []
        with records.open_reader() as reader:
            for record in reader:
                video_id = int(record['videoid'])
                video_id_list.append(video_id)
        log_.info(f"special area limit videos count = {len(video_id_list)}")
        log_.info("videos = {}".format(",".join([str(i) for i in video_id_list])))
        # 存入redis
        if len(video_id_list) > 0:
            value = ",".join([str(i) for i in video_id_list])
            redis_helper.set_data_to_redis(key_name=RISK_SHIELD_FILTER_VIDEO_V1_STR, value=value,
                                           expire_time=3600*24 * 7)
            # redis_helper.del_keys(key_name=config_.SPECIAL_AREA_LIMIT_KEY_NAME)
            # redis_helper.add_data_with_set(key_name=config_.SPECIAL_AREA_LIMIT_KEY_NAME, values=video_id_list,
            #                                expire_time=25 * 3600)
    except Exception as e:
        log_.error(str(e) + str(traceback.format_exc()))

def main():
    log_.info("开始执行:" + datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))

    # -----------------风险过滤需求------------------------------------------------
    get_special_area_limit_videos()
    expire_time = 3600*24 * 30

    key = "RISK_SHIELD_FILTER_RULE_V1_JSON"
    value = "{\"2\": []}"
    redis_helper.set_data_to_redis(key, value, expire_time)
    value_get = redis_helper.get_data_from_redis(key_name=key)
    print('key:', key, type(key))
    print('value_get:', value_get, type(value_get))

    key = "RISK_SHIELD_FILTER_EXPANSION_FACTOR_INT"
    value = "10"
    redis_helper.set_data_to_redis(key, value, expire_time)
    value_get = redis_helper.get_data_from_redis(key_name=key)
    print('key:', key, type(key))
    print('value_get:', value_get, type(value_get))

    key = "RISK_SHIELD_FILTER_FLAG_BOOL"
    value = "True"
    redis_helper.set_data_to_redis(key, value, expire_time)
    value_get = redis_helper.get_data_from_redis(key_name=key)
    print('key:', key, type(key))
    print('value_get:', value_get, type(value_get))



    # -----------------多样性需求的过滤------------------------------------------------
    expire_time = 3600 * 24 * 30

    key = "TAGS_FILTER_FLAG_BOOL"
    value = "True"
    redis_helper.set_data_to_redis(key, value, expire_time)
    value_get = redis_helper.get_data_from_redis(key_name=key)
    print('key:', key, type(key))
    print('value_get:', value_get, type(value_get))

    key = "TAGS_FILTER_RULE_V1_JSON"
    with open('alg_recsys_recall_tags_videos.json', 'r') as f:
        json_read = json.load(f)
        value = json.dumps(json_read, ensure_ascii=False)
    redis_helper.set_data_to_redis(key, value, expire_time)
    value_get = redis_helper.get_data_from_redis(key_name=key)
    print('key:', key, type(key))
    print('value_get:', value_get, type(value_get))

    key = "TAGS_FILTER_RULE_V2_JSON"
    with open('alg_recsys_recall_tags_videos_v2.json', 'r') as f:
        json_read = json.load(f)
        value = json.dumps(json_read, ensure_ascii=False)
    redis_helper.set_data_to_redis(key, value, expire_time)
    value_get = redis_helper.get_data_from_redis(key_name=key)
    print('key:', key, type(key))
    print('value_get:', value_get, type(value_get))

    key = "TAGS_FILTER_RULE_V3_JSON"
    with open('alg_recsys_recall_tags_videos_v3.json', 'r') as f:
        json_read = json.load(f)
        value = json.dumps(json_read, ensure_ascii=False)
    redis_helper.set_data_to_redis(key, value, expire_time)
    value_get = redis_helper.get_data_from_redis(key_name=key)
    print('key:', key, type(key))
    print('value_get:', value_get, type(value_get))

    log_.info("完成执行:" + datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))

if __name__ == '__main__':
    main()



# cd /root/zhangbo/rov-offline
# python alg_recsys_recall_shield_videos.py