update

ad_threshold_update_task.sh

@@ -2,12 +2,12 @@ source /etc/profile
 echo $ROV_OFFLINE_ENV
 if [[ $ROV_OFFLINE_ENV == 'test' ]]; then
     cd /data2/rov-offline &&
-    /root/anaconda3/bin/python /data2/rov-offline/ad_user_data_update.py &&
+    /root/anaconda3/bin/python /data2/rov-offline/ad_users_data_update.py &&
     /root/anaconda3/bin/python /data2/rov-offline/ad_video_data_update.py &&
     /root/anaconda3/bin/python /data2/rov-offline/ad_user_video_predict.py
 elif [[ $ROV_OFFLINE_ENV == 'pro' ]]; then
     cd /data/rov-offline &&
-    /root/anaconda3/bin/python /data/rov-offline/ad_user_data_update.py &&
+    /root/anaconda3/bin/python /data/rov-offline/ad_users_data_update.py &&
     /root/anaconda3/bin/python /data/rov-offline/ad_video_data_update.py &&
     /root/anaconda3/bin/python /data/rov-offline/ad_user_video_predict.py
 fi

config.py

@@ -614,8 +614,13 @@ class DevelopmentConfig(BaseConfig):
     PROJECT_PATH = '/data2/rov-offline'
 
     # 测试环境redis地址
+    # REDIS_INFO = {
+    #     'host': 'r-bp1ps6my7lzg8rdhwx682.redis.rds.aliyuncs.com',
+    #     'port': 6379,
+    #     'password': 'Wqsd@2019',
+    # }
     REDIS_INFO = {
-        'host': 'r-bp1ps6my7lzg8rdhwx682.redis.rds.aliyuncs.com',
+        'host': 'r-bp16jexstuzih671tz.redis.rds.aliyuncs.com',
         'port': 6379,
         'password': 'Wqsd@2019',
     }

user_group_update.py

@@ -31,17 +31,20 @@ def update_user_group_to_redis(project, table, dt, app_type):
     for group in group_list:
         log_.info(f"group = {group} update redis start ...")
         mid_list = feature_df[group].tolist()
+        mid_list = list(set(mid_list))
         mid_list = [mid for mid in mid_list if mid is not None]
         log_.info(f"mid count = {len(mid_list)}")
-        pool = multiprocessing.Pool(processes=2)
+        # pool = multiprocessing.Pool(processes=2)
         for mid in mid_list:
+            # print(mid)
             key_name = f"{config_.KEY_NAME_PREFIX_MID_GROUP}{mid}"
-            pool.apply_async(
-                func=redis_helper.set_data_to_redis,
-                args=(key_name, group, 25 * 3600)
-            )
-        pool.close()
-        pool.join()
+            redis_helper.set_data_to_redis(key_name=key_name, value=group, expire_time=25 * 3600)
+        #     pool.apply_async(
+        #         func=redis_helper.set_data_to_redis,
+        #         args=(key_name, group, 25 * 3600)
+        #     )
+        # pool.close()
+        # pool.join()
         log_.info(f"group = {group}, mid count = {len(mid_list)}, update redis finished!")
 
 
@@ -50,7 +53,7 @@ def timer_check():
         app_type = config_.APP_TYPE['VLOG']
         project = config_.ad_model_data['user_group'].get('project')
         table = config_.ad_model_data['user_group'].get('table')
-        now_date = datetime.datetime.today()
+        now_date = datetime.datetime.today() - datetime.timedelta(days=1)
         dt = datetime.datetime.strftime(now_date, '%Y%m%d')
         log_.info(f"now_date: {dt}")
         now_min = datetime.datetime.now().minute