add ad abtest config: 194-e & update

ad_user_video_predict.py

@@ -1,9 +1,10 @@
 import datetime
 import sys
+import traceback
 import numpy as np
 import pandas as pd
 from odps import ODPS
-from utils import data_check, get_feature_data, send_msg_to_feishu, RedisHelper
+from utils import data_check, get_feature_data, send_msg_to_feishu_new, RedisHelper
 from config import set_config
 from log import Log
 config_, _ = set_config()
@@ -135,26 +136,52 @@ def predict_ad_group_video(dt, config_key, config_param, threshold_record):
 
 
 def predict():
-    now_date = datetime.datetime.today()
-    dt = datetime.datetime.strftime(now_date, '%Y%m%d')
-    log_.info(f"dt = {dt}")
-    # 获取阈值参数记录
-    threshold_record = redis_helper.get_data_from_redis(key_name=config_.KEY_NAME_PREFIX_AD_THRESHOLD_RECORD)
-    threshold_record = eval(threshold_record)
-    log_.info(f"threshold_record = {threshold_record}")
-    params = config_.AD_ABTEST_CONFIG
-    for config_key, config_param in params.items():
-        predict_ad_group_video(dt=dt,
-                               config_key=config_key,
-                               config_param=config_param,
-                               threshold_record=threshold_record)
-    # 阈值参数记录
-    # redis_helper.set_data_to_redis(key_name=config_.KEY_NAME_PREFIX_AD_THRESHOLD_RECORD,
-    #                                value=str(config_.AD_ABTEST_THRESHOLD_CONFIG),
-    #                                expire_time=24*3600)
-    redis_helper.set_data_to_redis(key_name=config_.KEY_NAME_PREFIX_AD_THRESHOLD_RECORD,
-                                   value=str(threshold_record),
-                                   expire_time=2 * 24 * 3600)
+    try:
+        now_date = datetime.datetime.today()
+        dt = datetime.datetime.strftime(now_date, '%Y%m%d')
+        log_.info(f"dt = {dt}")
+        # 获取阈值参数记录
+        threshold_record = redis_helper.get_data_from_redis(key_name=config_.KEY_NAME_PREFIX_AD_THRESHOLD_RECORD)
+        # print(threshold_record)
+        threshold_record = eval(threshold_record)
+        log_.info(f"threshold_record = {threshold_record}")
+        params = config_.AD_ABTEST_CONFIG
+        for config_key, config_param in params.items():
+            predict_ad_group_video(dt=dt,
+                                   config_key=config_key,
+                                   config_param=config_param,
+                                   threshold_record=threshold_record)
+        # 阈值参数记录
+        # redis_helper.set_data_to_redis(key_name=config_.KEY_NAME_PREFIX_AD_THRESHOLD_RECORD,
+        #                                value=str(config_.AD_ABTEST_THRESHOLD_CONFIG),
+        #                                expire_time=24*3600)
+        redis_helper.set_data_to_redis(key_name=config_.KEY_NAME_PREFIX_AD_THRESHOLD_RECORD,
+                                       value=str(threshold_record),
+                                       expire_time=2 * 24 * 3600)
+        msg_list = [
+            f"env: rov-offline {config_.ENV_TEXT}",
+            f"finished time: {datetime.datetime.strftime(datetime.datetime.now(), '%Y%m%d %H:%M:%S')}",
+        ]
+        send_msg_to_feishu_new(
+            webhook=config_.FEISHU_ROBOT['ad_threshold_update_robot'].get('webhook'),
+            key_word=config_.FEISHU_ROBOT['ad_threshold_update_robot'].get('key_word'),
+            title='广告模型阈值更新完成',
+            msg_list=msg_list
+        )
+    except Exception as e:
+        log_.error(f"广告模型阈值更新失败, exception: {e}, traceback: {traceback.format_exc()}")
+        msg_list = [
+            f"env: rov-offline {config_.ENV_TEXT}",
+            f"now time: {datetime.datetime.strftime(datetime.datetime.now(), '%Y%m%d %H:%M:%S')}",
+            f"exception: {e}",
+            f"traceback: {traceback.format_exc()}",
+        ]
+        send_msg_to_feishu_new(
+            webhook=config_.FEISHU_ROBOT['ad_threshold_update_robot'].get('webhook'),
+            key_word=config_.FEISHU_ROBOT['ad_threshold_update_robot'].get('key_word'),
+            title='广告模型阈值更新失败',
+            msg_list=msg_list
+        )
 
 
 if __name__ == '__main__':

ad_video_data_update.py

@@ -1,7 +1,8 @@
 import datetime
 import traceback
+import multiprocessing
 from threading import Timer
-from utils import RedisHelper, data_check, get_feature_data, send_msg_to_feishu
+from utils import RedisHelper, data_check, get_feature_data, send_msg_to_feishu, send_msg_to_feishu_new
 from config import set_config
 from log import Log
 config_, _ = set_config()
@@ -122,15 +123,20 @@ def timer_check(dt, video_key, video_params, top10_abnormal_videos):
         # 数据准备好，进行更新
         update_videos_data(project=project, table=table, dt=dt, update_params=video_params,
                            top10_abnormal_videos=top10_abnormal_videos)
-        log_.info(f"ad video data update end!")
-        send_msg_to_feishu(
-            webhook=config_.FEISHU_ROBOT['server_robot'].get('webhook'),
-            key_word=config_.FEISHU_ROBOT['server_robot'].get('key_word'),
-            msg_text=f"rov-offline{config_.ENV_TEXT} - 视频数据更新完成\n"
-                     f"video_key: {video_key}\n"
-                     f"now_date: {dt}\n"
-                     f"finished time: {datetime.datetime.strftime(datetime.datetime.now(), '%Y%m%d %H:%M:%S')}"
+        log_.info(f"video_key = {video_key} ad video data update end!")
+        msg_list = [
+            f"env: rov-offline {config_.ENV_TEXT}",
+            f"video_key: {video_key}",
+            f"now_date: {dt}",
+            f"finished time: {datetime.datetime.strftime(datetime.datetime.now(), '%Y%m%d %H:%M:%S')}",
+        ]
+        send_msg_to_feishu_new(
+            webhook=config_.FEISHU_ROBOT['ad_video_update_robot'].get('webhook'),
+            key_word=config_.FEISHU_ROBOT['ad_video_update_robot'].get('key_word'),
+            title='广告模型视频分享率预测数据更新完成',
+            msg_list=msg_list
         )
+
     else:
         # 数据没准备好，1分钟后重新检查
         Timer(60, timer_check, args=[dt, video_key, video_params, top10_abnormal_videos]).start()
@@ -146,17 +152,30 @@ def main():
             dt=dt, filter_param=config_.ad_model_data['top10_videos'].get('abnormal_filter_param')
         )
         update_params = config_.AD_VIDEO_DATA_PARAMS
+        pool = multiprocessing.Pool(processes=len(update_params))
         for video_key, video_params in update_params.items():
-            timer_check(dt, video_key, video_params, top10_abnormal_videos)
+            pool.apply_async(
+                func=timer_check,
+                args=(dt, video_key, video_params, top10_abnormal_videos)
+            )
+        pool.close()
+        pool.join()
+        # for video_key, video_params in update_params.items():
+        #     timer_check(dt, video_key, video_params, top10_abnormal_videos)
 
     except Exception as e:
         log_.error(f"视频分享率预测数据更新失败, exception: {e}, traceback: {traceback.format_exc()}")
-        send_msg_to_feishu(
-            webhook=config_.FEISHU_ROBOT['server_robot'].get('webhook'),
-            key_word=config_.FEISHU_ROBOT['server_robot'].get('key_word'),
-            msg_text=f"rov-offline{config_.ENV_TEXT} - 视频分享率预测数据更新失败\n"
-                     f"exception: {e}\n"
-                     f"traceback: {traceback.format_exc()}"
+        msg_list = [
+            f"env: rov-offline {config_.ENV_TEXT}",
+            f"now time: {datetime.datetime.strftime(datetime.datetime.now(), '%Y%m%d %H:%M:%S')}",
+            f"exception: {e}",
+            f"traceback: {traceback.format_exc()}",
+        ]
+        send_msg_to_feishu_new(
+            webhook=config_.FEISHU_ROBOT['ad_video_update_robot'].get('webhook'),
+            key_word=config_.FEISHU_ROBOT['ad_video_update_robot'].get('key_word'),
+            title='广告模型视频分享率预测数据更新失败',
+            msg_list=msg_list
         )
 
 

config.py

@@ -51,6 +51,14 @@ class BaseConfig(object):
         'ad_user_group_update_robot': {
             'webhook': 'https://open.feishu.cn/open-apis/bot/v2/hook/d7b29139-0656-4ec6-988e-ef593556795e',
             'key_word': '用户分组数据更新'
+        },
+        'ad_video_update_robot': {
+            'webhook': 'https://open.feishu.cn/open-apis/bot/v2/hook/f9f6c242-c378-4dc9-8ec1-9c55dbd7fac9',
+            'key_word': '广告视频数据更新'
+        },
+        'ad_threshold_update_robot': {
+            'webhook': 'https://open.feishu.cn/open-apis/bot/v2/hook/85861132-939e-4ba7-802b-b1ba32602173',
+            'key_word': '广告模型阈值更新'
         }
     }
 
@@ -741,7 +749,11 @@ class BaseConfig(object):
             'project': 'loghubods',
             'table': 'abnormal_videoid_admodel',
             'abnormal_filter_param': 1.5
-        }
+        },
+        'videos_share_rate_alladtype': {
+            'project': 'loghubods',
+            'table': 'video_sharerate_admodel_alladtype'
+        },
     }
 
     # 自动调整广告模型阈值数据
@@ -768,19 +780,30 @@ class BaseConfig(object):
         'videos_share_rate': {
             'data1': APP_TYPE['VLOG'],  # vlog
             'data2': APP_TYPE['LOVE_LIVE'],  # 票圈视频
-            'data3': APP_TYPE['LONG_VIDEO'],  # 内容精选
-            'data4': APP_TYPE['SHORT_VIDEO'],  # 票圈短视频
+            # 'data3': APP_TYPE['LONG_VIDEO'],  # 内容精选
+            # 'data4': APP_TYPE['SHORT_VIDEO'],  # 票圈短视频
             'data5': APP_TYPE['LAO_HAO_KAN_VIDEO'],  # 老好看视频
             'data6': APP_TYPE['ZUI_JING_QI'],  # 票圈最惊奇
-            'data21': APP_TYPE['PIAO_QUAN_VIDEO_PLUS'],  # 票圈视频+
+            # 'data21': APP_TYPE['PIAO_QUAN_VIDEO_PLUS'],  # 票圈视频+
         },
         'videos_share_rate_7days': {
             'data1:7days': APP_TYPE['VLOG'],  # vlog 优化4
-            'data4:7days': APP_TYPE['SHORT_VIDEO'],  # 票圈短视频 优化4
+            # 'data4:7days': APP_TYPE['SHORT_VIDEO'],  # 票圈短视频 优化4
             'data5:7days': APP_TYPE['LAO_HAO_KAN_VIDEO'],  # 老好看视频 优化4
-        }
+        },
+        # ##### 广告模型视频数据 - 修复上报后的数据
+        'videos_share_rate_alladtype': {
+            'videos4': APP_TYPE['LOVE_LIVE'],  # 票圈视频
+        },
     }
 
+    # # 广告模型视频数据-修复上报后的数据
+    # AD_VIDEO_DATA_PARAMS_NEW = {
+    #     'videos_share_rate_new': {
+    #         'videos4': APP_TYPE['LOVE_LIVE'],  # 票圈视频
+    #     },
+    # }
+
     # 广告模型异常视频数据处理参数
     AD_ABNORMAL_VIDEOS_PARAM = {
         'data1': 17/48,  # vlog
@@ -902,10 +925,10 @@ class BaseConfig(object):
                   'user': {'data': 'data1', 'rule': 'rule1'}},
         '190-b': {'video': {'data': 'data1'},
                   'user': {'data': 'data1', 'rule': 'rule2'}},
-        '190-c': {'video': {'data': 'data21'},
-                  'user': {'data': 'data21', 'rule': 'rule1'}},
-        '190-d': {'video': {'data': 'data21'},
-                  'user': {'data': 'data21', 'rule': 'rule2'}},
+        # '190-c': {'video': {'data': 'data21'},
+        #           'user': {'data': 'data21', 'rule': 'rule1'}},
+        # '190-d': {'video': {'data': 'data21'},
+        #           'user': {'data': 'data21', 'rule': 'rule2'}},
         # 票圈视频
         '194-a': {'video': {'data': 'data1'},
                   'user': {'data': 'data1', 'rule': 'rule1'}},
@@ -915,24 +938,26 @@ class BaseConfig(object):
                   'user': {'data': 'data2', 'rule': 'rule1'}},  # 本端数据
         '194-d': {'video': {'data': 'data2'},
                   'user': {'data': 'data2', 'rule': 'rule2'}},  # 本端数据 + 优化1
+        '194-e': {'video': {'data': 'videos4'},
+                  'user': {'data': 'data2', 'rule': 'rule2'}},  # 修复后本端数据 + 优化1
         # 内容精选
         '195-a': {'video': {'data': 'data1'},
                   'user': {'data': 'data1', 'rule': 'rule1'}},
         '195-b': {'video': {'data': 'data1'},
                   'user': {'data': 'data1', 'rule': 'rule2'}},
-        '195-c': {'video': {'data': 'data3'},
-                  'user': {'data': 'data3', 'rule': 'rule1'}},  # 本端数据
-        '195-d': {'video': {'data': 'data3'},
-                  'user': {'data': 'data3', 'rule': 'rule2'}},  # 本端数据 + 优化1
+        # '195-c': {'video': {'data': 'data3'},
+        #           'user': {'data': 'data3', 'rule': 'rule1'}},  # 本端数据
+        # '195-d': {'video': {'data': 'data3'},
+        #           'user': {'data': 'data3', 'rule': 'rule2'}},  # 本端数据 + 优化1
         # 票圈短视频
         '196-a': {'video': {'data': 'data1'},
                   'user': {'data': 'data1', 'rule': 'rule1'}},
-        '196-b': {'video': {'data': 'data4'},
-                  'user': {'data': 'data4', 'rule': 'rule1'}},
-        '196-c': {'video': {'data': 'data4'},
-                  'user': {'data': 'data4', 'rule': 'rule3'}},
-        '196-d': {'video': {'data': 'data4:7days'},
-                  'user': {'data': 'data4', 'rule': 'rule3'}},  # 优化4
+        # '196-b': {'video': {'data': 'data4'},
+        #           'user': {'data': 'data4', 'rule': 'rule1'}},
+        # '196-c': {'video': {'data': 'data4'},
+        #           'user': {'data': 'data4', 'rule': 'rule3'}},
+        # '196-d': {'video': {'data': 'data4:7days'},
+        #           'user': {'data': 'data4', 'rule': 'rule3'}},  # 优化4
         # 老好看视频
         '197-a': {'video': {'data': 'data1'},
                   'user': {'data': 'data1', 'rule': 'rule1'}},

utils.py

@@ -128,6 +128,33 @@ def send_msg_to_feishu(webhook, key_word, msg_text):
     print(response.text)
 
 
+def send_msg_to_feishu_new(webhook, key_word, title, msg_list):
+    """发送消息到飞书"""
+    headers = {'Content-Type': 'application/json'}
+    content_list = [
+        [
+            {
+                "tag": "text",
+                "text": msg
+            }
+        ]
+        for msg in msg_list
+    ]
+    payload_message = {
+        "msg_type": "post",
+        "content": {
+            "post": {
+                "zh_cn": {
+                    "title": f"{key_word}: {title}",
+                    "content": content_list,
+                }
+            }
+        }
+    }
+    response = requests.request('POST', url=webhook, headers=headers, data=json.dumps(payload_message))
+    print(response.text)
+
+
 def request_post(request_url, request_data=None, **kwargs):
     """
     post 请求 HTTP接口