# -*- coding: utf-8 -*- import pandas as pd import traceback import odps from odps import ODPS from threading import Timer from datetime import datetime, timedelta from db_helper import MysqlHelper from my_utils import check_table_partition_exits_v2, get_dataframe_from_odps, \ get_odps_df_of_max_partition, get_odps_instance, get_odps_df_of_recent_partitions from my_utils import request_post, send_msg_to_feishu from my_config import set_config import numpy as np from log import Log import os from argparse import ArgumentParser from constants import AutoReplyAccountType from alg_growth_common import check_unsafe_video, filter_unsafe_video, filter_audit_failed_video CONFIG, _ = set_config() LOGGER = Log() BASE_GROUP_NAME = 'stg0909-base' EXPLORE1_GROUP_NAME = 'stg0909-explore1' EXPLORE2_GROUP_NAME = 'stg0909-explore2' GH_IDS = ('gh_ac43e43b253b', 'gh_93e00e187787', 'gh_77f36c109fb1', 'gh_68e7fdc09fe4', 'gh_b181786a6c8c') CDN_IMG_OPERATOR = "?x-oss-process=image/resize,m_fill,w_600,h_480,limit_0/format,jpg/watermark,image_eXNoL3BpYy93YXRlcm1hcmtlci9pY29uX3BsYXlfd2hpdGUucG5nP3gtb3NzLXByb2Nlc3M9aW1hZ2UvcmVzaXplLHdfMTQ0,g_center" ODS_PROJECT = "loghubods" EXPLORE_POOL_TABLE = 'alg_growth_video_return_stats_history' GH_REPLY_STATS_TABLE = 'alg_growth_gh_reply_video_stats' GH_REPLY_STATS_HOUR_TABLE = 'alg_growth_gh_reply_video_stats_hour' ODPS_RANK_RESULT_TABLE = 'alg_gh_autoreply_video_rank_data' RDS_RANK_RESULT_TABLE = 'alg_gh_autoreply_video_rank_data' GH_DETAIL = 'gh_detail' STATS_PERIOD_DAYS = 5 STATS_PERIOD_DAYS_FOR_QUIT = 30 SEND_N = 2 pd.set_option('display.max_rows', None) def get_and_update_gh_ids(run_dt): gh = get_odps_df_of_max_partition(ODS_PROJECT, GH_DETAIL, {'dt': run_dt}) gh = gh.to_pandas() gh_type = AutoReplyAccountType.SELF_OWNED_GZH.value gh = gh.query(f'type == {gh_type} and is_delete == 0') # default单独处理 if 'default' not in gh['gh_id'].values: new_row = pd.DataFrame({'gh_id': ['default'], 'gh_name': ['默认'], 'type': [2], 'category1': ['泛生活']}, index=[0]) gh = pd.concat([gh, new_row], ignore_index=True) gh = gh.drop_duplicates(subset=['gh_id']) global GH_IDS GH_IDS = tuple(gh['gh_id']) return gh def check_data_partition(project, table, data_dt, data_hr=None): """检查数据是否准备好""" try: partition_spec = {'dt': data_dt} if data_hr: partition_spec['hour'] = data_hr part_exist, data_count = check_table_partition_exits_v2( project, table, partition_spec) except Exception as e: data_count = 0 return data_count def get_last_strategy_result(project, rank_table, dt_version, key): strategy_df = get_odps_df_of_max_partition( project, rank_table, {'ctime': dt_version} ).to_pandas() sub_df = strategy_df.query(f'strategy_key == "{key}"') sub_df = sub_df[['gh_id', 'video_id', 'strategy_key', 'sort']].drop_duplicates() return sub_df def process_reply_stats(project, daily_table, hourly_table, period, run_dt, run_hour): # 获取多天+当天即转统计数据用于聚合 df = get_odps_df_of_recent_partitions( project, daily_table, period, {'dt': run_dt}).to_pandas() hour_data_version = f'{run_dt}{run_hour}' hourly_df = get_odps_df_of_recent_partitions( project, hourly_table, 1, {'dt': hour_data_version}).to_pandas() df = pd.concat([df, hourly_df]).reset_index(drop=True) df['video_id'] = df['video_id'].astype('int64') df = df[['gh_id', 'video_id', 'send_count', 'first_visit_uv', 'day0_return']] # 获取统计数据时统一去除不安全视频 df = filter_unsafe_video(df) # 账号内聚合 df = df.groupby(['video_id', 'gh_id']).agg({ 'send_count': 'sum', 'first_visit_uv': 'sum', 'day0_return': 'sum' }).reset_index() # 聚合所有数据作为default default_stats_df = df.groupby('video_id').agg({ 'send_count': 'sum', 'first_visit_uv': 'sum', 'day0_return': 'sum' }).reset_index() default_stats_df['gh_id'] = 'default' merged_df = pd.concat([df, default_stats_df]).reset_index(drop=True) merged_df['score'] = merged_df['day0_return'] / (merged_df['send_count'] + 500) return merged_df def rank_for_layer1(run_dt, run_hour, project, table, gh_df): # TODO: 加审核 df = get_odps_df_of_max_partition(project, table, {'dt': run_dt}) df = df.to_pandas() # use statistic data to quit some low-efficiency video stats_df = get_odps_df_of_recent_partitions( ODS_PROJECT, GH_REPLY_STATS_TABLE, 30, {'dt': run_dt}).to_pandas() stats_df['video_id'] = stats_df['video_id'].astype('int64') stats_df = stats_df[['video_id', 'send_count', 'first_visit_uv', 'day0_return']] stats_df = stats_df.groupby(['video_id']).agg({ 'send_count': 'sum', 'first_visit_uv': 'sum', 'day0_return': 'sum' }) # do not add to denominator stats_df['return_by_send'] = stats_df['day0_return'] / (stats_df['send_count']) stats_df['open_rate'] = stats_df['first_visit_uv'] / (stats_df['send_count']) # do not filter video that does not have enough data stats_df = stats_df.query('send_count > 1000') df = df.merge(stats_df, on='video_id', how='left') open_rate_threshold = df.open_rate.quantile(q=0.2) return_by_send_threshold = df.return_by_send.quantile(q=0.2) filter_condition = 'open_rate < {} and return_by_send < {}' \ .format(open_rate_threshold, return_by_send_threshold) filter_rows = df.query(filter_condition) df = df.drop(filter_rows.index) print("low-efficient video to quit:") print(filter_rows[['video_id', 'title', 'send_count', 'open_rate', 'return_by_send']]) df = filter_unsafe_video(df) # 确保重跑时可获得一致结果 dt_version = f'{run_dt}{run_hour}' np.random.seed(int(dt_version) + 1) # TODO: 修改权重计算策略 df['score'] = 1.0 # 按照 category1 分类后进行加权随机抽样 sampled_df = df.groupby('category1').apply( lambda x: x.sample(n=SEND_N, weights=x['score'], replace=False)).reset_index(drop=True) sampled_df['sort'] = sampled_df.groupby('category1')['score'].rank(method='first', ascending=False).astype(int) sampled_df['strategy_key'] = EXPLORE1_GROUP_NAME sampled_df['dt_version'] = dt_version extend_df = sampled_df.merge(gh_df, on='category1') result_df = extend_df[['strategy_key', 'dt_version', 'gh_id', 'sort', 'video_id', 'score']] return result_df def rank_for_layer2(run_dt, run_hour, rank_table): stats_df = process_reply_stats( ODS_PROJECT, GH_REPLY_STATS_TABLE, GH_REPLY_STATS_HOUR_TABLE, STATS_PERIOD_DAYS, run_dt, run_hour) # 确保重跑时可获得一致结果 dt_version = f'{run_dt}{run_hour}' np.random.seed(int(dt_version) + 1) # TODO: 计算账号间相关性 ## 账号两两组合，取有RoVn数值视频的交集，单个账号内的RoVn（平滑后）组成向量 ## 求向量相关系数或cosine相似度 ## 单个视频的RoVn加权求和 # 当前实现基础版本：只在账号内求二级探索排序分 sampled_dfs = [] # 处理default逻辑（default-explore2） default_stats_df = stats_df.query('gh_id == "default"') sampled_df = default_stats_df.sample(n=SEND_N, weights=default_stats_df['score']) sampled_df['sort'] = range(1, len(sampled_df) + 1) sampled_dfs.append(sampled_df) # 基础过滤for账号 df = stats_df.query('day0_return > 100') # 目标账号失效视频过滤 df = filter_audit_failed_video(df) # fallback to base if necessary base_strategy_df = get_last_strategy_result( ODS_PROJECT, rank_table, dt_version, BASE_GROUP_NAME) base_strategy_df = filter_audit_failed_video(base_strategy_df) for gh_id in GH_IDS: if gh_id == 'default': continue sub_df = df.query(f'gh_id == "{gh_id}"') if len(sub_df) < SEND_N: LOGGER.warning( "gh_id[{}] rows[{}] not enough for layer2, fallback to base" .format(gh_id, len(sub_df))) sub_df = base_strategy_df.query(f'gh_id == "{gh_id}"') sub_df['score'] = sub_df['sort'] sampled_df = sub_df.sample(n=SEND_N, weights=sub_df['score']) sampled_df['sort'] = range(1, len(sampled_df) + 1) sampled_dfs.append(sampled_df) extend_df = pd.concat(sampled_dfs) extend_df['strategy_key'] = EXPLORE2_GROUP_NAME extend_df['dt_version'] = dt_version result_df = extend_df[['strategy_key', 'dt_version', 'gh_id', 'sort', 'video_id', 'score']] return result_df def rank_for_base(run_dt, run_hour, rank_table): stats_df = process_reply_stats( ODS_PROJECT, GH_REPLY_STATS_TABLE, GH_REPLY_STATS_HOUR_TABLE, STATS_PERIOD_DAYS, run_dt, run_hour) # TODO: support to set base manually dt_version = f'{run_dt}{run_hour}' # 获取当前base信息, 策略表dt_version(ctime partition)采用当前时间 base_strategy_df = get_last_strategy_result( ODS_PROJECT, rank_table, dt_version, BASE_GROUP_NAME) default_stats_df = stats_df.query('gh_id == "default"') # 在账号内排序，决定该账号（包括default）的base利用内容 # 排序过程中，确保当前base策略参与排序，因此先关联再过滤 non_default_ids = list(filter(lambda x: x != 'default', GH_IDS)) gh_ids_str = ','.join(f'"{x}"' for x in non_default_ids) stats_df = stats_df.query(f'gh_id in ({gh_ids_str})') stats_with_strategy_df = stats_df \ .merge( base_strategy_df, on=['gh_id', 'video_id'], how='outer') \ .query('strategy_key.notna() or score > 0.1') # 合并default和分账号数据 grouped_stats_df = pd.concat([default_stats_df, stats_with_strategy_df]).reset_index() grouped_stats_df = filter_audit_failed_video(grouped_stats_df) def set_top_n(group, n=2): group_sorted = group.sort_values(by='score', ascending=False) top_n = group_sorted.head(n) top_n['sort'] = range(1, n + 1) return top_n ranked_df = grouped_stats_df.groupby('gh_id').apply(set_top_n, SEND_N) ranked_df = ranked_df.reset_index(drop=True) # ranked_df['sort'] = grouped_stats_df.groupby('gh_id')['score'].rank(ascending=False) ranked_df['strategy_key'] = BASE_GROUP_NAME ranked_df['dt_version'] = dt_version ranked_df = ranked_df[['strategy_key', 'dt_version', 'gh_id', 'sort', 'video_id', 'score']] return ranked_df def check_result_data(df): check_unsafe_video(df, False) for gh_id in GH_IDS: for key in (EXPLORE1_GROUP_NAME, EXPLORE2_GROUP_NAME, BASE_GROUP_NAME): sub_df = df.query(f'gh_id == "{gh_id}" and strategy_key == "{key}"') if len(sub_df) != SEND_N: raise Exception(f"Result not enough for gh_id[{gh_id}] group[{key}]") def build_and_transfer_data(run_dt, run_hour, project, **kwargs): dt_version = f'{run_dt}{run_hour}' dry_run = kwargs.get('dry_run', False) next_dt = (datetime.strptime(run_dt, "%Y%m%d") + timedelta(1)).strftime("%Y%m%d") gh_df = get_and_update_gh_ids(next_dt) layer1_rank = rank_for_layer1(run_dt, run_hour, ODS_PROJECT, EXPLORE_POOL_TABLE, gh_df) layer2_rank = rank_for_layer2(run_dt, run_hour, ODPS_RANK_RESULT_TABLE) base_rank = rank_for_base(run_dt, run_hour, ODPS_RANK_RESULT_TABLE) final_rank_df = pd.concat([layer1_rank, layer2_rank, base_rank]).reset_index(drop=True) check_result_data(final_rank_df) odps_instance = get_odps_instance(project) odps_ranked_df = odps.DataFrame(final_rank_df) video_df = get_dataframe_from_odps('videoods', 'wx_video') video_df['cover_url'] = video_df['cover_img_path'] + CDN_IMG_OPERATOR video_df = video_df['id', 'title', 'cover_url'] final_df = odps_ranked_df.join(video_df, on=('video_id', 'id')) final_df = final_df.to_pandas() final_df = final_df[['strategy_key', 'dt_version', 'gh_id', 'sort', 'video_id', 'title', 'cover_url', 'score']] # reverse sending order final_df['sort'] = SEND_N + 1 - final_df['sort'] # clean NAN value final_df = final_df.fillna({'score': 0.0}) if dry_run: print(final_df[['strategy_key', 'gh_id', 'sort', 'video_id', 'score', 'title']] .sort_values(by=['strategy_key', 'gh_id', 'sort'])) return # save to ODPS t = odps_instance.get_table(ODPS_RANK_RESULT_TABLE) part_spec_dict = {'dt': run_dt, 'hour': run_hour, 'ctime': dt_version} part_spec =','.join(['{}={}'.format(k, part_spec_dict[k]) for k in part_spec_dict.keys()]) with t.open_writer(partition=part_spec, create_partition=True, overwrite=True) as writer: writer.write(list(final_df.itertuples(index=False))) # sync to MySQL data_to_insert = [tuple(row) for row in final_df.itertuples(index=False)] data_columns = list(final_df.columns) mysql = MysqlHelper(CONFIG.MYSQL_GROWTH_INFO) mysql.batch_insert(RDS_RANK_RESULT_TABLE, data_to_insert, data_columns) def main_loop(): argparser = ArgumentParser() argparser.add_argument('-n', '--dry-run', action='store_true') argparser.add_argument('--run-at', help='assume to run at date and hour, yyyyMMddHH') args = argparser.parse_args() run_date = datetime.today() if args.run_at: run_date = datetime.strptime(args.run_at, "%Y%m%d%H") LOGGER.info(f"Assume to run at {run_date.strftime('%Y-%m-%d %H:00')}") try: now_date = datetime.today() LOGGER.info(f"开始执行: {datetime.strftime(now_date, '%Y-%m-%d %H:%M')}") last_date = run_date - timedelta(1) last_dt = last_date.strftime("%Y%m%d") # 查看当前天级更新的数据是否已准备好 # 当前上游统计表为天级更新，但字段设计为兼容小时级 h_data_count = check_data_partition(ODS_PROJECT, GH_REPLY_STATS_TABLE, last_dt, '00') if h_data_count > 0: LOGGER.info('上游数据表查询数据条数={}，开始计算'.format(h_data_count)) run_dt = run_date.strftime("%Y%m%d") run_hour = run_date.strftime("%H") LOGGER.info(f'run_dt: {run_dt}, run_hour: {run_hour}') build_and_transfer_data(run_dt, run_hour, ODS_PROJECT, dry_run=args.dry_run) LOGGER.info('数据更新完成') else: LOGGER.info("上游数据未就绪，等待60s") Timer(60, main_loop).start() return except Exception as e: LOGGER.error(f"数据更新失败, exception: {e}, traceback: {traceback.format_exc()}") if CONFIG.ENV_TEXT == '开发环境' or args.dry_run: return send_msg_to_feishu( webhook=CONFIG.FEISHU_ROBOT['growth_task_robot'].get('webhook'), key_word=CONFIG.FEISHU_ROBOT['growth_task_robot'].get('key_word'), msg_text=f"rov-offline{CONFIG.ENV_TEXT} - 数据更新失败\n" f"exception: {e}\n" f"traceback: {traceback.format_exc()}" ) if __name__ == '__main__': LOGGER.info("%s 开始执行" % os.path.basename(__file__)) LOGGER.info(f"environment: {CONFIG.ENV_TEXT}") main_loop()