update add_data_with_set

db_helper.py

@@ -232,8 +232,25 @@ class RedisHelper(object):
         :param expire_time: 过期时间，单位：s，默认0.5小时
         :return: None
         """
+        # conn = self.connect()
+        # conn.sadd(key_name, *values)
+        # # 设置过期时间
+        # conn.expire(key_name, expire_time)
+
+        if not values:
+            return
         conn = self.connect()
-        conn.sadd(key_name, *values)
+        # 数据量大时一次性写入耗时长，分批次写入
+        add_data = []
+        for i, val in enumerate(values):
+            if i % 100 == 0:
+                if len(add_data) > 0:
+                    conn.sadd(key_name, *tuple(add_data))
+                add_data = [val]
+            else:
+                add_data.append(val)
+        if len(add_data) > 0:
+            conn.sadd(key_name, *tuple(add_data))
         # 设置过期时间
         conn.expire(key_name, expire_time)
 

flowpool_data_update.py

@@ -364,6 +364,7 @@ def predict_19(app_type):
 
 
 if __name__ == '__main__':
+    st_time = time.time()
     app_type_list = [config_.APP_TYPE['LAO_HAO_KAN_VIDEO'], config_.APP_TYPE['ZUI_JING_QI']]
     log_.info('flow pool predict start...')
     video_info_list = []
@@ -381,8 +382,7 @@ if __name__ == '__main__':
 
     # 更新剩余分发数
     log_.info(f"video_info_list count = {len(video_info_list)}")
-    st_time = time.time()
     get_videos_remain_view_count(video_info_list)
-    log_.info(f"expend time = {(time.time() - st_time) * 1000}")
     log_.info('flow pool predict end...')
+    log_.info(f"expend time = {(time.time() - st_time) * 1000}ms")