debug

db_helper.py

@@ -11,6 +11,7 @@ config_ = set_config()
 log_ = Log()
 
 conn_redis = None
+conn_filter_redis = None
 
 
 class RedisHelper(object):
@@ -19,6 +20,7 @@ class RedisHelper(object):
         初始化redis连接信息
         redis_info: redis连接信息, 格式：dict, {'host': '', 'port': '', 'password': ''}
         """
+        self.redis_info = redis_info
         self.host = redis_info['host']
         self.port = redis_info['port']
         self.password = redis_info['password']
@@ -29,15 +31,26 @@ class RedisHelper(object):
         连接redis
         :return: conn
         """
-        global conn_redis
-        if conn_redis is None:
-            pool = redis.ConnectionPool(host=self.host,
-                                        port=self.port,
-                                        password=self.password,
-                                        decode_responses=True)
-            conn = redis.Redis(connection_pool=pool)
-            conn_redis = conn
-        return conn_redis
+        if self.redis_info == config_.REDIS_INFO_FILTER:
+            global conn_filter_redis
+            if conn_filter_redis is None:
+                pool = redis.ConnectionPool(host=self.host,
+                                            port=self.port,
+                                            password=self.password,
+                                            decode_responses=True)
+                conn = redis.Redis(connection_pool=pool)
+                conn_filter_redis = conn
+            return conn_filter_redis
+        else:
+            global conn_redis
+            if conn_redis is None:
+                pool = redis.ConnectionPool(host=self.host,
+                                            port=self.port,
+                                            password=self.password,
+                                            decode_responses=True)
+                conn = redis.Redis(connection_pool=pool)
+                conn_redis = conn
+            return conn_redis
 
     def key_exists(self, key_name):
         """

utils.py

@@ -539,11 +539,13 @@ class FilterVideos(object):
             return video_ids
         redis_helper = RedisHelper(redis_info=config_.REDIS_INFO_FILTER)
         filter_videos = []
+        print(f"keys: {keys}")
         for i in range(len(keys)//1000 + 1):
             video_ids_temp = video_ids[i*1000:(i+1)*1000]
             if len(video_ids_temp) == 0:
                 break
             mget_res = redis_helper.mget(keys=keys[i*1000:(i+1)*1000])
+            print(f"mget_res: {mget_res}")
             filter_videos.extend([int(data) for data in mget_res if data is not None])
         if len(filter_videos) > 0:
             filtered_videos = set(video_ids) - set(filter_videos)
@@ -568,7 +570,10 @@ class FilterVideos(object):
             gevent.spawn(self.filter_movie_religion_video, filtered_pre_result)]
         gevent.joinall(t)
         filtered_result_list = [i.get() for i in t]
-        filtered_viewed_result = list(set(filtered_result_list[0]) & set(filtered_result_list[1]))
+        filtered_viewed_result_list = [int(item) for item in filtered_result_list[0]]
+        filtered_movie_religion_result_list = [int(item) for item in filtered_result_list[1]]
+        print(f"filtered_viewed_result_list: {filtered_viewed_result_list}\nfiltered_movie_religion_result_list:{filtered_movie_religion_result_list}")
+        filtered_viewed_result = list(set(filtered_viewed_result_list) & set(filtered_movie_religion_result_list))
         print(f"view&m_r res: {filtered_viewed_result}\nexecute_time: {(time.time() - st_viewed) * 1000}")
         if not filtered_viewed_result:
             return None
@@ -615,10 +620,11 @@ if __name__ == '__main__':
         ('weixin_openid_o0w175eRpvbmV6nOhM1VTyyLICWA', '30488803'),
         ('weixin_openid_o0w175TZYvG47pQkOjyJFoxQuqsw', '')
     ]
-    video_df = pd.read_csv('./data/videoids.csv')
+    video_df = pd.read_csv('/Users/liqian/Documents/liqian/work/PythonProject/test/data/videoids.csv')
     videoid_list = video_df['videoid'].tolist()
     for mid, uid in user:
-        video_ids = random.sample(videoid_list, 1000)
+        video_ids = random.sample(videoid_list, 2)
+        video_ids.extend([10002478, 13004472])
         start_time = time.time()
         filter_ = FilterVideos(request_id=f'{mid} - {uid}', app_type=0, mid=mid, uid=uid, video_ids=video_ids)
         res = filter_.filter_videos_new()