rov-offline/special_mids_update.py

import time
import datetime
import pandas as pd
import math
import random
from odps import ODPS
from threading import Timer
from get_data import get_data_from_odps
from db_helper import RedisHelper, MysqlHelper
from my_config import set_config
from log import Log
from my_utils import filter_video_status_with_applet_rec

config_, env = set_config()
log_ = Log()


def get_special_mid(now_date, project, table):
    """获取特殊mid数据"""
    dt = datetime.datetime.strftime(now_date, '%Y%m%d%H')
    odps = ODPS(
        access_id=config_.ODPS_CONFIG['ACCESSID'],
        secret_access_key=config_.ODPS_CONFIG['ACCESSKEY'],
        project=project,
        endpoint=config_.ODPS_CONFIG['ENDPOINT'],
        onnect_timeout=3000,
        read_timeout=500000,
        pool_maxsize=1000,
        pool_connections=100
    )
    records = odps.read_table(name=table)
    mid_list = []
    for record in records:
        if record['sharemid']:
            mid_list.append(record['sharemid'])
        if record['clickmid']:
            mid_list.append(record['clickmid'])
    mid_list2 = list(set(mid_list))
    redis_helper = RedisHelper()
    key_name = config_.KEY_NAME_SPECIAL_MID
    if redis_helper.key_exists(key_name=key_name):
        redis_helper.del_keys(key_name=key_name)
    redis_helper.add_data_with_set(key_name=key_name, values=mid_list2, expire_time=48 * 3600)
    log_.info(f"dt = {dt}, special_mid_count = {len(mid_list2)}")


if __name__ == '__main__':
    project = config_.SPECIAL_MID_VIDEOS_PROJECT.get('mid')
    table = config_.SPECIAL_MID_VIDEOS_TABLE.get('mid')
    now_date = datetime.datetime.today()
    log_.info(f"now_date: {datetime.datetime.strftime(now_date, '%Y%m%d%H')}")
    get_special_mid(now_date=now_date, project=project, table=table)