Merge branch 'redis-opt' into pre-master

config.py

@@ -234,6 +234,11 @@ class BaseConfig(object):
     # 记录 mid-小时级key 中数据所属(date，h)，完整格式：com.weiqu.video.h.region.24h.record.mid.{appType}.{mid}
     H_WITH_MID_RECORD_KEY_NAME_PREFIX_REGION_24H = 'com.weiqu.video.h.region.24h.record.mid.'
 
+    # 用户上一次在dup1 分地域24h更新列表中对应的位置 redis key前缀，完整key格式：com.weiqu.video.region.dup1.24h.last.item.{appType}.{mid}
+    LAST_VIDEO_FROM_REGION_DUP1_24H_PREFIX = 'com.weiqu.video.region.dup1.24h.last.item.'
+    # 记录 mid-上一次在dup1 分地域24h更新列表中对应的位置key 中数据所属(date，h)，完整格式：com.weiqu.video.h.region.dup1.24h.last.record.mid.{appType}.{mid}
+    RECORD_KEY_NAME_PREFIX_LAST_VIDEO_REGION_DUP1_24H = 'com.weiqu.video.h.region.dup1.24h.last.record.mid.'
+
     # 用户上一次在dup2 24h更新列表中对应的位置 redis key前缀，完整key格式：com.weiqu.video.region.dup2.24h.last.item.{appType}.{mid}
     LAST_VIDEO_FROM_REGION_DUP2_24H_PREFIX = 'com.weiqu.video.region.dup2.24h.last.item.'
     # 记录 mid-上一次在dup2 24h更新列表中对应的位置key 中数据所属(date，h)，完整格式：com.weiqu.video.h.region.dup2.24h.last.record.mid.{appType}.{mid}

recommend.py

@@ -391,6 +391,14 @@ def update_redis_data(result, app_type, mid, last_rov_recall_key, top_K, expire_
                                                    expire_time=expire_time)
                 # log_.info('last video redis update success!')
 
+            # 将此次获取的 地域分组相对24h数据列表 中的视频id同步刷新到redis中，方便下次快速定位到召回位置
+            rov_recall_24h_dup1_video = [item['videoId'] for item in result[:top_K]
+                                         if item['pushFrom'] == config_.PUSH_FROM['rov_recall_region_24h']]
+            if len(rov_recall_24h_dup1_video) > 0:
+                last_video_key = f'{config_.LAST_VIDEO_FROM_REGION_DUP1_24H_PREFIX}{app_type}.{mid}'
+                redis_helper.set_data_to_redis(key_name=last_video_key, value=rov_recall_24h_dup1_video[-1],
+                                               expire_time=expire_time)
+
             # 将此次获取的 相对24h筛选数据列表 中的视频id同步刷新到redis中，方便下次快速定位到召回位置
             rov_recall_24h_dup2_video = [item['videoId'] for item in result[:top_K]
                                          if item['pushFrom'] == config_.PUSH_FROM['rov_recall_24h']]

video_recall.py

@@ -1194,24 +1194,26 @@ class PoolRecall(object):
             if province_code == '-1':
                 t = [
                     # gevent.spawn(self.rov_pool_recall_with_region_by_h, province_code, size, 'day_24h'),
-                     gevent.spawn(self.recall_region_dup_24h, province_code, size, '24h_dup2'),
-                     gevent.spawn(self.recall_region_dup_24h, province_code, size, '24h_dup3')]
+                     gevent.spawn(self.recall_region_dup_24h, province_code, size, '24h_dup2', expire_time),
+                     gevent.spawn(self.recall_region_dup_24h, province_code, size, '24h_dup3', expire_time)]
             else:
                 t = [gevent.spawn(self.rov_pool_recall_with_region_by_h, province_code, size),
-                     gevent.spawn(self.rov_pool_recall_with_region_by_h, province_code, size, 'region_24h'),
+                     # gevent.spawn(self.rov_pool_recall_with_region_by_h, province_code, size, 'region_24h'),
                      # gevent.spawn(self.rov_pool_recall_with_region_by_h, province_code, size, 'day_24h'),
-                     gevent.spawn(self.recall_region_dup_24h, province_code, size, '24h_dup2'),
-                     gevent.spawn(self.recall_region_dup_24h, province_code, size, '24h_dup3')]
+                     gevent.spawn(self.recall_region_dup_24h, province_code, size, 'region_24h', expire_time),
+                     gevent.spawn(self.recall_region_dup_24h, province_code, size, '24h_dup2', expire_time),
+                     gevent.spawn(self.recall_region_dup_24h, province_code, size, '24h_dup3', expire_time)]
         else:
             if province_code == '-1':
                 # t = [gevent.spawn(self.rov_pool_recall_with_region_by_h, province_code, size, 'day_24h')]
-                t = [gevent.spawn(self.recall_region_dup_24h, province_code, size, '24h_dup2')]
+                t = [gevent.spawn(self.recall_region_dup_24h, province_code, size, '24h_dup2', expire_time)]
 
             else:
                 t = [gevent.spawn(self.rov_pool_recall_with_region_by_h, province_code, size),
-                     gevent.spawn(self.rov_pool_recall_with_region_by_h, province_code, size, 'region_24h'),
+                     # gevent.spawn(self.rov_pool_recall_with_region_by_h, province_code, size, 'region_24h'),
                      # gevent.spawn(self.rov_pool_recall_with_region_by_h, province_code, size, 'day_24h'),
-                     gevent.spawn(self.recall_region_dup_24h, province_code, size, '24h_dup2'),
+                     gevent.spawn(self.recall_region_dup_24h, province_code, size, 'region_24h', expire_time),
+                     gevent.spawn(self.recall_region_dup_24h, province_code, size, '24h_dup2', expire_time),
 
                      # gevent.spawn(self.region_dup_recall, province_code, 1, size, expire_time),
                      # gevent.spawn(self.region_dup_recall, province_code, 2, size, expire_time)
@@ -1648,7 +1650,6 @@ class PoolRecall(object):
         record_key = f"{record_key_prefix}{self.app_type}.{self.mid}"
         last_video_key = f'{last_video_key_prefix}{self.app_type}.{self.mid}'
 
-
         if not self.redis_helper.key_exists(key_name=record_key):
             # ###### 记录key不存在
             self.redis_helper.del_keys(key_name=last_video_key)
@@ -1697,7 +1698,16 @@ class PoolRecall(object):
         :return:
         """
         start_time = time.time()
-        if key_flag == '24h_dup2':
+        if key_flag == 'region_24h':
+            # 分地域相对24h的筛选结果
+            # mid对应上一次视频位置 时间记录
+            record_key_prefix = config_.RECORD_KEY_NAME_PREFIX_LAST_VIDEO_REGION_DUP1_24H
+            # 视频列表
+            pool_key_prefix = config_.RECALL_KEY_NAME_PREFIX_DUP1_REGION_24H_H
+            # mid对应上一次视频记录
+            last_video_key_prefix = config_.LAST_VIDEO_FROM_REGION_DUP1_24H_PREFIX
+            push_from = config_.PUSH_FROM['rov_recall_region_24h']
+        elif key_flag == '24h_dup2':
             # 不分地域相对24h的筛选结果
             # mid对应上一次视频位置 时间记录
             record_key_prefix = config_.RECORD_KEY_NAME_PREFIX_LAST_VIDEO_REGION_DUP2_24H