add & update ad new strategy tasks

ad_arpu_update.py

@@ -37,11 +37,11 @@ def update_ad_arpu(project, table, dt):
         return
     ecpm = json.loads(ecpm)
     own_ecpm = ecpm.get('own', 0)
-    vx_ecpm = ecpm.get('vx', 0)
+    vx_ecpm = ecpm.get('weixin', 0)
     if own_ecpm == 0 and vx_ecpm == 0:
         return
     # 计算上一周期广告收入
-    income = ad_own_view * float(own_ecpm) + ad_vx_view * float(vx_ecpm)
+    income = ad_own_view * float(own_ecpm) / 1000 + ad_vx_view * float(vx_ecpm) / 1000
     # 计算上一周期arpu
     arpu = income / dau
     arpu = round(arpu, 4)

ad_arpu_update_task.sh

@@ -0,0 +1,8 @@
+source /etc/profile
+echo $ROV_OFFLINE_ENV
+if [[ $ROV_OFFLINE_ENV == 'test' ]]; then
+    cd /data2/rov-offline &&
+    /root/anaconda3/bin/python /data2/rov-offline/ad_arpu_update.py
+elif [[ $ROV_OFFLINE_ENV == 'pro' ]]; then
+    cd /data/rov-offline &&
+    /root/anaconda3/bin/python /data/rov-offline/ad_arpu_update.py

ad_ecpm_update.py

@@ -0,0 +1,71 @@
+import datetime
+import json
+import traceback
+
+from utils import request_get, send_msg_to_feishu
+from db_helper import RedisHelper
+from config import set_config
+from log import Log
+
+config_, _ = set_config()
+log_ = Log()
+
+
+def get_ad_ecpm():
+    """获取广告ecpm值"""
+    ad_ecpm = {}
+    result = request_get(request_url=config_.GET_AD_ECPM_URL)
+    print(result)
+    if result is None:
+        log_.info('获取广告ecpm值失败！')
+        return ad_ecpm
+    if result['code'] != 0:
+        log_.info('获取广告ecpm值失败！')
+        return ad_ecpm
+    if not result['data']:
+        return ad_ecpm
+    ad_ecpm = result['data']
+    print(ad_ecpm)
+    return ad_ecpm
+
+
+def update_ad_ecpm():
+    try:
+        now_date = datetime.datetime.today()
+        log_.info(f"now_date: {datetime.datetime.strftime(now_date, '%Y%m%d%H')}")
+
+        # 获取广告ecpm值
+        ad_ecpm = get_ad_ecpm()
+        log_.info(f"ad_ecpm: {ad_ecpm}")
+        if len(ad_ecpm) == 0:
+            send_msg_to_feishu(
+                webhook=config_.FEISHU_ROBOT['server_robot'].get('webhook'),
+                key_word=config_.FEISHU_ROBOT['server_robot'].get('key_word'),
+                msg_text=f"rov-offline{config_.ENV_TEXT} - 广告ecpm获取失败！"
+            )
+            return
+        redis_helper = RedisHelper()
+        redis_helper.set_data_to_redis(key_name=config_.KEY_NAME_AD_ECPM, value=json.dumps(ad_ecpm))
+        redis_helper.persist_key(key_name=config_.KEY_NAME_AD_ECPM)
+
+        send_msg_to_feishu(
+            webhook=config_.FEISHU_ROBOT['server_robot'].get('webhook'),
+            key_word=config_.FEISHU_ROBOT['server_robot'].get('key_word'),
+            msg_text=f"rov-offline{config_.ENV_TEXT} - 广告ecpm更新完成！\nad_ecpm = {ad_ecpm}"
+
+        )
+        log_.info(f"ad ecpm update end!")
+
+    except Exception as e:
+        log_.error(f"广告ecpm更新失败, exception: {e}, traceback: {traceback.format_exc()}")
+        send_msg_to_feishu(
+            webhook=config_.FEISHU_ROBOT['server_robot'].get('webhook'),
+            key_word=config_.FEISHU_ROBOT['server_robot'].get('key_word'),
+            msg_text=f"rov-offline{config_.ENV_TEXT} - 广告ecpm更新失败\n"
+                     f"exception: {e}\n"
+                     f"traceback: {traceback.format_exc()}"
+        )
+
+
+if __name__ == '__main__':
+    update_ad_ecpm()

ad_ecpm_update_task.sh

@@ -0,0 +1,8 @@
+source /etc/profile
+echo $ROV_OFFLINE_ENV
+if [[ $ROV_OFFLINE_ENV == 'test' ]]; then
+    cd /data2/rov-offline &&
+    /root/anaconda3/bin/python /data2/rov-offline/ad_ecpm_update.py
+elif [[ $ROV_OFFLINE_ENV == 'pro' ]]; then
+    cd /data/rov-offline &&
+    /root/anaconda3/bin/python /data/rov-offline/ad_ecpm_update.py

ad_roi_param_update.py

@@ -0,0 +1,22 @@
+import datetime
+from db_helper import RedisHelper
+from config import set_config
+from log import Log
+
+config_, _ = set_config()
+log_ = Log()
+
+
+def update_ad_roi_param(param):
+    redis_helper = RedisHelper()
+    redis_helper.set_data_to_redis(key_name=config_.KEY_NAME_AD_ROI_PARAM, value=param)
+    redis_helper.persist_key(key_name=config_.KEY_NAME_AD_ROI_PARAM)
+
+
+if __name__ == '__main__':
+    now_date = datetime.datetime.today()
+    log_.info(f"now_date: {datetime.datetime.strftime(now_date, '%Y%m%d %H:%M:%S')}")
+    param = 5
+    log_.info(f"param = {param}")
+    update_ad_roi_param(param=param)
+    log_.info(f"ad roi param update finished!")

ad_user_data_update_with_new_strategy.py

@@ -30,6 +30,7 @@ def predict_user_group_share_rate_with_ad(user_group_initial_df, dt, data_params
 
     # 获取对应的用户组特征
     user_group_df = user_group_initial_df.copy()
+    print(user_group_df)
     # 获取所有广告类型对应的数据
     user_group_df['ad_type'] = user_group_df['ad_type'].astype(int)
     user_group_df = user_group_df[user_group_df['ad_type'] == 0]
@@ -92,7 +93,7 @@ def predict_user_group_share_rate_no_ad(user_group_initial_df, dt, data_params,
     user_group_df = user_group_df[user_group_df['group'].isin(user_group_list)]
 
     # 计算用户组有广告时分享率
-    user_group_df = user_group_df[user_group_df['ad_rate'] != 0]
+    user_group_df = user_group_df[user_group_df['no_ad_rate'] != 0]
     user_group_df['group_no_ad_share_rate'] = \
         user_group_df['no_adrate_share'] * user_group_df['sharerate'] / user_group_df['no_ad_rate']
     user_group_df['group_no_ad_share_rate'].fillna(0, inplace=True)

ad_user_data_update_with_new_strategy_task.sh

@@ -0,0 +1,8 @@
+source /etc/profile
+echo $ROV_OFFLINE_ENV
+if [[ $ROV_OFFLINE_ENV == 'test' ]]; then
+    cd /data2/rov-offline &&
+    /root/anaconda3/bin/python /data2/rov-offline/ad_user_data_update_with_new_strategy.py
+elif [[ $ROV_OFFLINE_ENV == 'pro' ]]; then
+    cd /data/rov-offline &&
+    /root/anaconda3/bin/python /data/rov-offline/ad_user_data_update_with_new_strategy.py

ad_video_data_update_with_new_strategy.py

@@ -70,6 +70,7 @@ def predict_video_share_rate_with_ad(video_initial_df, dt, data_key, data_param,
     video_df['apptype'] = video_df['apptype'].astype(int)
     video_df = video_df[video_df['apptype'] == int(data_param)]
     log_.info(f"video_df length: {len(video_df)}")
+    # print(video_df)
     video_df['ad_rate'].fillna(0, inplace=True)
     video_df['sharerate'].fillna(0, inplace=True)
     video_df['adrate_share'].fillna(0, inplace=True)
@@ -78,9 +79,10 @@ def predict_video_share_rate_with_ad(video_initial_df, dt, data_key, data_param,
     video_df['adrate_share'] = video_df['adrate_share'].astype(float)
 
     # 计算视频有广告时被分享率
-    video_df = video_df[video_df['adrate'] != 0]
+    video_df = video_df[video_df['ad_rate'] != 0]
+    # print(video_df)
     video_df['video_ad_share_rate'] = \
-        video_df['adrate_share'] * video_df['sharerate'] / video_df['adrate']
+        video_df['adrate_share'] * video_df['sharerate'] / video_df['ad_rate']
     video_df['video_ad_share_rate'].fillna(0, inplace=True)
     # log_.info(f"video_df: {video_df}")
     video_df = video_df[video_df['video_ad_share_rate'] != 0]
@@ -122,8 +124,8 @@ def predict_video_share_rate_no_ad(video_initial_df, dt, data_key, data_param, t
     video_df['sharerate'] = video_df['sharerate'].astype(float)
     video_df['no_adrate_share'] = video_df['adrate_share'].astype(float)
 
-    # 计算视频有广告时被分享率
-    video_df = video_df[video_df['adrate'] != 0]
+    # 计算视频无广告时被分享率
+    video_df = video_df[video_df['no_ad_rate'] != 0]
     video_df['video_no_ad_share_rate'] = \
         video_df['no_adrate_share'] * video_df['sharerate'] / video_df['no_ad_rate']
     video_df['video_no_ad_share_rate'].fillna(0, inplace=True)
@@ -156,10 +158,16 @@ def update_videos_data(project, table, dt, update_params, top10_abnormal_videos)
     video_initial_df = get_feature_data(project=project, table=table, features=features, dt=dt)
     for data_key, data_param in update_params.items():
         log_.info(f"data_key = {data_key} update start...")
+        log_.info(f"predict_video_share_rate_with_ad start...")
         predict_video_share_rate_with_ad(video_initial_df=video_initial_df, dt=dt, data_key=data_key,
                                          data_param=data_param, top10_abnormal_videos=top10_abnormal_videos)
+        log_.info(f"predict_video_share_rate_with_ad end!")
+
+        log_.info(f"predict_video_share_rate_no_ad start...")
         predict_video_share_rate_no_ad(video_initial_df=video_initial_df, dt=dt, data_key=data_key,
                                        data_param=data_param, top10_abnormal_videos=top10_abnormal_videos)
+        log_.info(f"predict_video_share_rate_no_ad end!")
+
         log_.info(f"data_key = {data_key} update end!")
 
 
@@ -223,8 +231,8 @@ def main():
             f"traceback: {traceback.format_exc()}",
         ]
         send_msg_to_feishu_new(
-            webhook=config_.FEISHU_ROBOT['ad_video_update_robot'].get('webhook'),
-            key_word=config_.FEISHU_ROBOT['ad_video_update_robot'].get('key_word'),
+            webhook=config_.FEISHU_ROBOT['server_robot'].get('webhook'),
+            key_word=config_.FEISHU_ROBOT['server_robot'].get('key_word'),
             title='新策略 -- 广告模型视频分享率预测数据更新失败',
             msg_list=msg_list
         )

ad_video_data_update_with_new_strategy_task.sh

@@ -0,0 +1,8 @@
+source /etc/profile
+echo $ROV_OFFLINE_ENV
+if [[ $ROV_OFFLINE_ENV == 'test' ]]; then
+    cd /data2/rov-offline &&
+    /root/anaconda3/bin/python /data2/rov-offline/ad_video_data_update_with_new_strategy.py
+elif [[ $ROV_OFFLINE_ENV == 'pro' ]]; then
+    cd /data/rov-offline &&
+    /root/anaconda3/bin/python /data/rov-offline/ad_video_data_update_with_new_strategy.py

config.py

@@ -828,7 +828,7 @@ class BaseConfig(object):
         },  # 新策略使用视频侧数据：所有广告类型数据，按照videoId统计（各视频被分享的概率，各视频出广告的概率，各视频被分享的情况下出广告的概率，各视频不出广告的概率，各视频被分享的情况下不出广告的概率）
         'users_share_rate_new_strategy': {
             'project': 'loghubods',
-            'table': 'usergroup_sharerate_admodel'
+            'table': 'usergroup_data_with_ad_sharerate_adtype'
         },  # 新策略使用用户侧数据：按照用户分组统计（各用户组的分享率，各用户组出广告的概率，各用户组有分享的情况下出广告的概率，各用户组不出广告的概率，各用户组有分享的情况下不出广告的概率）
     }
 
@@ -2366,6 +2366,8 @@ class DevelopmentConfig(BaseConfig):
     GET_VIDEO_LIMIT_LIST_URL = 'http://videotest-internal.yishihui.com/longvideoapi/openapi/recommend/getVideoLimitList'
     # 获取管理后台设置的广告目标uv值接口地址
     GET_AD_TARGET_UV_URL = 'https://testadmin.piaoquantv.com/manager/ad/algo/threshold/productUvTargetList'
+    # 获取广告ecpm值接口地址
+    GET_AD_ECPM_URL = 'https://testapi.piaoquantv.com/ad/getAdEcpmInfo'
 
     # # logs 上传oss 目标Bucket指定目录
     # OSS_FOLDER_LOGS = 'rov-offline/dev/logs/'
@@ -2448,6 +2450,8 @@ class TestConfig(BaseConfig):
     GET_VIDEO_LIMIT_LIST_URL = 'http://videotest-internal.yishihui.com/longvideoapi/openapi/recommend/getVideoLimitList'
     # 获取管理后台设置的广告目标uv值接口地址
     GET_AD_TARGET_UV_URL = 'https://testadmin.piaoquantv.com/manager/ad/algo/threshold/productUvTargetList'
+    # 获取广告ecpm值接口地址
+    GET_AD_ECPM_URL = 'https://testapi.piaoquantv.com/ad/getAdEcpmInfo'
 
     # # logs 上传oss 目标Bucket指定目录
     # OSS_FOLDER_LOGS = 'rov-offline/test/logs/'
@@ -2530,6 +2534,8 @@ class PreProductionConfig(BaseConfig):
     GET_VIDEO_LIMIT_LIST_URL = 'http://prespeed-internal.piaoquantv.com/longvideoapi/openapi/recommend/getVideoLimitList'
     # 获取管理后台设置的广告目标uv值接口地址
     GET_AD_TARGET_UV_URL = 'https://preadmin.piaoquantv.com/manager/ad/algo/threshold/productUvTargetList'
+    # 获取广告ecpm值接口地址
+    GET_AD_ECPM_URL = 'https://preapi.piaoquantv.com/ad/getAdEcpmInfo'
 
     # # logs 上传oss 目标Bucket指定目录
     # OSS_FOLDER_LOGS = 'rov-offline/pre/logs/'
@@ -2612,6 +2618,8 @@ class ProductionConfig(BaseConfig):
     GET_VIDEO_LIMIT_LIST_URL = 'http://recommend-common-internal.piaoquantv.com/longvideoapi/openapi/recommend/getVideoLimitList'
     # 获取管理后台设置的广告目标uv值接口地址
     GET_AD_TARGET_UV_URL = 'https://admin.piaoquantv.com/manager/ad/algo/threshold/productUvTargetList'
+    # 获取广告ecpm值接口地址
+    GET_AD_ECPM_URL = 'https://api.piaoquantv.com/ad/getAdEcpmInfo'
 
     # # logs 上传oss 目标Bucket指定目录
     # OSS_FOLDER_LOGS = 'rov-offline/pro/logs/'