log-opt-2022090710

app.py

@@ -24,22 +24,7 @@ from manager_op import get_video_list, search_video
 # from werkzeug.middleware.profiler import ProfilerMiddleware
 # from geventwebsocket.handler import WebSocketHandler
 
-
-# logging.basicConfig(level=logging.DEBUG,  # 控制台打印的日志级别
-#                     filename=os.path.join(os.path.dirname(os.path.realpath(__file__)), "logs", 'app.log'),  # 将日志写入log_new.log文件中
-#                     filemode='w',  # 模式，有w和a，w就是写模式，每次都会重新写日志，覆盖之前的日志 a是追加模式，默认如果不写的话，就是追加模式
-#                     format="%(asctime)s:%(levelname)s:%(name)s -- %(message)s", datefmt="%Y/%m/%d %H:%M:%S"  # 日志格式
-#                     )
-
 app = Flask(__name__)
-
-# handler = logging.FileHandler('app.log', encoding='utf-8')
-# logging_format = logging.Formatter('%(asctime)s - %(levelname)s - %(filename)s- %(funcName)s - %(lineno): %(message)s')
-# handler.setFormatter(logging_format)
-# app.logger.setLevel(logging.DEBUG)
-# app.logger.addHandler(handler)
-
-#app.wsgi_app = ProfilerMiddleware(app.wsgi_app)
 log_ = Log()
 config_ = set_config()
 
@@ -53,20 +38,12 @@ def health_check():
 @app.route('/applet/video/homepage/recommend', methods=['GET', 'POST'])
 def homepage_recommend():
     start_time = time.time()
-    in_homepage = start_time * 1000 + random.randint(0, 100)
+    # in_homepage = start_time * 1000 + random.randint(0, 100)
     # log_.info({'type': 'homepage', 'in_homepage': in_homepage})
     try:
-        log_.info({'request_headers': request.headers})
+        # log_.info({'request_headers': request.headers})
         request_data = json.loads(request.get_data())
         request_id = request_data.get('requestId')
-        # log_.info({
-        #     'logTimestamp': int(time.time() * 1000),
-        #     'request_id': request_id,
-        #     'text': 'in homepage_recommend',
-        #     'type': 'homepage_recommend',
-        #     'in_homepage': in_homepage,
-        #     'executeTime': (time.time() - start_time) * 1000
-        # })
         mid = request_data.get('mid')
         uid = request_data.get('uid')
         category_id = request_data.get('categoryId')
@@ -78,20 +55,20 @@ def homepage_recommend():
         ab_info_data = request_data.get('abInfoData', None)
         version_audit_status = request_data.get('versionAuditStatus', 2)  # 小程序版本审核参数：1-审核中，2-审核通过，默认：2
 
-        # log_.info({'requestUri': '/applet/video/homepage/recommend', 'requestData': request_data})
-        # log_.info('homepage_recommend request data: {}'.format(request_data))
         params = Params(request_id=request_id)
         # size默认为10
         if not size:
             size = 10
         if category_id in config_.CATEGORY['recommend']:
             # 推荐
-            videos = video_homepage_recommend(request_id=request_id,
+            recommend_result = video_homepage_recommend(request_id=request_id,
+            # videos = video_homepage_recommend(request_id=request_id,
                                               mid=mid, uid=uid, size=size, app_type=app_type,
                                               algo_type=algo_type, client_info=client_info,
                                               ab_exp_info=ab_exp_info, params=params,
                                               ab_info_data=ab_info_data, version_audit_status=version_audit_status)
-            result = {'code': 200, 'message': 'success', 'data': {'videos': videos}}
+
+            result = {'code': 200, 'message': 'success', 'data': {'videos': recommend_result['videos']}}
             log_.info({
                 'requestUri': '/applet/video/homepage/recommend',
                 'logTimestamp': int(time.time() * 1000),
@@ -104,6 +81,7 @@ def homepage_recommend():
                 'category_id': category_id,
                 'mid': mid,
                 'uid': uid,
+                'recommend_result': recommend_result,
                 'result': result,
                 'executeTime': (time.time() - start_time) * 1000
             })
@@ -165,14 +143,14 @@ def relevant_recommend():
         # log_.info('relevant_recommend request data: {}'.format(request_data))
 
         params = Params(request_id=request_id)
-
-        videos = video_relevant_recommend(request_id=request_id,
+        recommend_result = video_relevant_recommend(request_id=request_id,
+        # videos = video_relevant_recommend(request_id=request_id,
                                           video_id=video_id, mid=mid, uid=uid, size=page_size, app_type=app_type,
                                           ab_exp_info=ab_exp_info, client_info=client_info, page_type=page_type,
                                           params=params, ab_info_data=ab_info_data,
                                           version_audit_status=version_audit_status)
 
-        result = {'code': 200, 'message': 'success', 'data': {'videos': videos}}
+        result = {'code': 200, 'message': 'success', 'data': {'videos': recommend_result['videos']}}
         log_.info({
             'requestUri': '/applet/video/relevant/recommend',
             'logTimestamp': int(time.time() * 1000),
@@ -184,6 +162,7 @@ def relevant_recommend():
             'version_audit_status': version_audit_status,
             'mid': mid,
             'uid': uid,
+            'recommend_result': recommend_result,
             'result': result,
             'executeTime': (time.time() - start_time) * 1000
         })

recommend.py

@@ -156,6 +156,7 @@ def video_recommend(request_id, mid, uid, size, top_K, flow_pool_P, app_type, al
     :param params:
     :return:
     """
+    result = {}
     # ####### 多进程召回
     start_recall = time.time()
     # log_.info('====== recall')
@@ -230,16 +231,17 @@ def video_recommend(request_id, mid, uid, size, top_K, flow_pool_P, app_type, al
     recall_result_list = [i.get() for i in t]
 
     # end_recall = time.time()
-    log_.info({
-        'logTimestamp': int(time.time() * 1000),
-        'request_id': request_id,
-        'mid': mid,
-        'uid': uid,
-        'operation': 'recall',
-        'recall_result': recall_result_list,
-        'executeTime': (time.time() - start_recall) * 1000
-    })
-
+    # log_.info({
+    #     'logTimestamp': int(time.time() * 1000),
+    #     'request_id': request_id,
+    #     'mid': mid,
+    #     'uid': uid,
+    #     'operation': 'recall',
+    #     'recall_result': recall_result_list,
+    #     'executeTime': (time.time() - start_recall) * 1000
+    # })
+    result['recallResult'] = recall_result_list
+    result['recallTime'] = (time.time() - start_recall) * 1000
 
     # ####### 排序
     start_rank = time.time()
@@ -284,15 +286,17 @@ def video_recommend(request_id, mid, uid, size, top_K, flow_pool_P, app_type, al
     #                                             size=size, top_K=top_K, old_video_index=old_video_index)
 
     # end_rank = time.time()
-    log_.info({
-        'logTimestamp': int(time.time() * 1000),
-        'request_id': request_id,
-        'mid': mid,
-        'uid': uid,
-        'operation': 'rank',
-        'rank_result': rank_result,
-        'executeTime': (time.time() - start_rank) * 1000
-    })
+    # log_.info({
+    #     'logTimestamp': int(time.time() * 1000),
+    #     'request_id': request_id,
+    #     'mid': mid,
+    #     'uid': uid,
+    #     'operation': 'rank',
+    #     'rank_result': rank_result,
+    #     'executeTime': (time.time() - start_rank) * 1000
+    # })
+    result['rankResult'] = rank_result
+    result['rankTime'] = (time.time() - start_rank) * 1000
 
     if not rank_result:
         # 兜底策略
@@ -307,17 +311,22 @@ def video_recommend(request_id, mid, uid, size, top_K, flow_pool_P, app_type, al
                 request_id=request_id, size=size, app_type=app_type, ab_code=ab_code, params=params
             )
 
-        log_.info({
-            'logTimestamp': int(time.time() * 1000),
-            'request_id': request_id,
-            'mid': mid,
-            'uid': uid,
-            'operation': 'bottom',
-            'bottom_result': rank_result,
-            'executeTime': (time.time() - start_bottom) * 1000
-        })
+        # log_.info({
+        #     'logTimestamp': int(time.time() * 1000),
+        #     'request_id': request_id,
+        #     'mid': mid,
+        #     'uid': uid,
+        #     'operation': 'bottom',
+        #     'bottom_result': rank_result,
+        #     'executeTime': (time.time() - start_bottom) * 1000
+        # })
+        result['bottomResult'] = rank_result
+        result['bottomTime'] = (time.time() - start_bottom) * 1000
+
+    result['rankResult'] = rank_result
 
-    return rank_result, last_rov_recall_key
+    return result, last_rov_recall_key
+    # return rank_result, last_rov_recall_key
 
 
 def ab_test_op(rank_result, ab_code_list, app_type, mid, uid, **kwargs):
@@ -899,28 +908,32 @@ def video_homepage_recommend(request_id, mid, uid, size, app_type, algo_type,
     #                       top_K=top_K, expire_time=12 * 3600)
     #
     # else:
+    recommend_result = {}
     param_st = time.time()
     # 特殊mid 和 小程序审核版本推荐处理
     if mid in get_special_mid_list() or version_audit_status == 1:
         rank_result = special_mid_recommend(request_id=request_id, mid=mid, uid=uid, app_type=app_type, size=size)
-        return rank_result
+        recommend_result['videos'] = rank_result
+        return recommend_result
 
     # 普通mid推荐处理
     top_K, flow_pool_P, ab_code, rule_key, data_key, expire_time, no_op_flag, old_video_index = \
         get_recommend_params(recommend_type=0, ab_exp_info=ab_exp_info, ab_info_data=ab_info_data)
-    log_.info({
-        'logTimestamp': int(time.time() * 1000),
-        'request_id': request_id,
-        'app_type': app_type,
-        'mid': mid,
-        'uid': uid,
-        'operation': 'get_recommend_params',
-        'executeTime': (time.time() - param_st) * 1000
-    })
+    # log_.info({
+    #     'logTimestamp': int(time.time() * 1000),
+    #     'request_id': request_id,
+    #     'app_type': app_type,
+    #     'mid': mid,
+    #     'uid': uid,
+    #     'operation': 'get_recommend_params',
+    #     'executeTime': (time.time() - param_st) * 1000
+    # })
+    recommend_result['getRecommendParamsTime'] = (time.time() - param_st) * 1000
 
     # 简单召回 - 排序 - 兜底
     get_result_st = time.time()
-    rank_result, last_rov_recall_key = video_recommend(request_id=request_id,
+    result, last_rov_recall_key = video_recommend(request_id=request_id,
+    # rank_result, last_rov_recall_key = video_recommend(request_id=request_id,
                                                        mid=mid, uid=uid, app_type=app_type,
                                                        size=size, top_K=top_K, flow_pool_P=flow_pool_P,
                                                        algo_type=algo_type, client_info=client_info,
@@ -928,15 +941,19 @@ def video_homepage_recommend(request_id, mid, uid, size, app_type, algo_type,
                                                        rule_key=rule_key, data_key=data_key,
                                                        no_op_flag=no_op_flag, old_video_index=old_video_index,
                                                        params=params)
-    log_.info({
-        'logTimestamp': int(time.time() * 1000),
-        'request_id': request_id,
-        'app_type': app_type,
-        'mid': mid,
-        'uid': uid,
-        'operation': 'get_recommend_result',
-        'executeTime': (time.time() - get_result_st) * 1000
-    })
+    # log_.info({
+    #     'logTimestamp': int(time.time() * 1000),
+    #     'request_id': request_id,
+    #     'app_type': app_type,
+    #     'mid': mid,
+    #     'uid': uid,
+    #     'operation': 'get_recommend_result',
+    #     'executeTime': (time.time() - get_result_st) * 1000
+    # })
+    recommend_result['recommendOperation'] = result
+    rank_result = result.get('rankResult')
+    recommend_result['videos'] = rank_result
+    recommend_result['getRecommendResultTime'] = (time.time() - get_result_st) * 1000
 
     # ab-test
     # result = ab_test_op(rank_result=rank_result,
@@ -946,17 +963,19 @@ def video_homepage_recommend(request_id, mid, uid, size, app_type, algo_type,
     update_redis_st = time.time()
     update_redis_data(result=rank_result, app_type=app_type, mid=mid, last_rov_recall_key=last_rov_recall_key,
                       top_K=top_K)
-    log_.info({
-        'logTimestamp': int(time.time() * 1000),
-        'request_id': request_id,
-        'app_type': app_type,
-        'mid': mid,
-        'uid': uid,
-        'operation': 'update_redis_data',
-        'executeTime': (time.time() - update_redis_st) * 1000
-    })
+    # log_.info({
+    #     'logTimestamp': int(time.time() * 1000),
+    #     'request_id': request_id,
+    #     'app_type': app_type,
+    #     'mid': mid,
+    #     'uid': uid,
+    #     'operation': 'update_redis_data',
+    #     'executeTime': (time.time() - update_redis_st) * 1000
+    # })
+    recommend_result['updateRedisDataTime'] = (time.time() - update_redis_st) * 1000
 
-    return rank_result
+    return recommend_result
+    # return rank_result
 
 
 def video_relevant_recommend(request_id, video_id, mid, uid, size, app_type, ab_exp_info, client_info,
@@ -977,28 +996,34 @@ def video_relevant_recommend(request_id, video_id, mid, uid, size, app_type, ab_
     :param version_audit_status: 小程序版本审核参数：1-审核中，2-审核通过
     :return: videos type-list
     """
+    recommend_result = {}
     param_st = time.time()
     # 特殊mid 和 小程序审核版本推荐处理
     if mid in get_special_mid_list() or version_audit_status == 1:
         rank_result = special_mid_recommend(request_id=request_id, mid=mid, uid=uid, app_type=app_type, size=size)
-        return rank_result
+        recommend_result['videos'] = rank_result
+        return recommend_result
+        # return rank_result
 
     # 普通mid推荐处理
     top_K, flow_pool_P, ab_code, rule_key, data_key, expire_time, no_op_flag, old_video_index = \
         get_recommend_params(recommend_type=1, ab_exp_info=ab_exp_info, ab_info_data=ab_info_data, page_type=page_type)
-    log_.info({
-        'logTimestamp': int(time.time() * 1000),
-        'request_id': request_id,
-        'app_type': app_type,
-        'mid': mid,
-        'uid': uid,
-        'operation': 'get_recommend_params',
-        'executeTime': (time.time() - param_st) * 1000
-    })
+    # log_.info({
+    #     'logTimestamp': int(time.time() * 1000),
+    #     'request_id': request_id,
+    #     'app_type': app_type,
+    #     'mid': mid,
+    #     'uid': uid,
+    #     'operation': 'get_recommend_params',
+    #     'executeTime': (time.time() - param_st) * 1000
+    # })
+    recommend_result['getRecommendParamsTime'] = (time.time() - param_st) * 1000
+
 
     # 简单召回 - 排序 - 兜底
     get_result_st = time.time()
-    rank_result, last_rov_recall_key = video_recommend(request_id=request_id,
+    result, last_rov_recall_key = video_recommend(request_id=request_id,
+    # rank_result, last_rov_recall_key = video_recommend(request_id=request_id,
                                                        mid=mid, uid=uid, app_type=app_type,
                                                        size=size, top_K=top_K, flow_pool_P=flow_pool_P,
                                                        algo_type='', client_info=client_info,
@@ -1006,15 +1031,20 @@ def video_relevant_recommend(request_id, video_id, mid, uid, size, app_type, ab_
                                                        rule_key=rule_key, data_key=data_key, no_op_flag=no_op_flag,
                                                        old_video_index=old_video_index, video_id=video_id,
                                                        params=params)
-    log_.info({
-        'logTimestamp': int(time.time() * 1000),
-        'request_id': request_id,
-        'app_type': app_type,
-        'mid': mid,
-        'uid': uid,
-        'operation': 'get_recommend_result',
-        'executeTime': (time.time() - get_result_st) * 1000
-    })
+    # log_.info({
+    #     'logTimestamp': int(time.time() * 1000),
+    #     'request_id': request_id,
+    #     'app_type': app_type,
+    #     'mid': mid,
+    #     'uid': uid,
+    #     'operation': 'get_recommend_result',
+    #     'executeTime': (time.time() - get_result_st) * 1000
+    # })
+    recommend_result['recommendOperation'] = result
+    rank_result = result.get('rankResult')
+    recommend_result['videos'] = rank_result
+    recommend_result['getRecommendResultTime'] = (time.time() - get_result_st) * 1000
+
     # ab-test
     # result = ab_test_op(rank_result=rank_result,
     #                     ab_code_list=[config_.AB_CODE['position_insert'], config_.AB_CODE['relevant_video_op']],
@@ -1024,17 +1054,19 @@ def video_relevant_recommend(request_id, video_id, mid, uid, size, app_type, ab_
     update_redis_st = time.time()
     update_redis_data(result=rank_result, app_type=app_type, mid=mid, last_rov_recall_key=last_rov_recall_key,
                       top_K=top_K)
-    log_.info({
-        'logTimestamp': int(time.time() * 1000),
-        'request_id': request_id,
-        'app_type': app_type,
-        'mid': mid,
-        'uid': uid,
-        'operation': 'update_redis_data',
-        'executeTime': (time.time() - update_redis_st) * 1000
-    })
-
-    return rank_result
+    # log_.info({
+    #     'logTimestamp': int(time.time() * 1000),
+    #     'request_id': request_id,
+    #     'app_type': app_type,
+    #     'mid': mid,
+    #     'uid': uid,
+    #     'operation': 'update_redis_data',
+    #     'executeTime': (time.time() - update_redis_st) * 1000
+    # })
+    recommend_result['updateRedisDataTime'] = (time.time() - update_redis_st) * 1000
+
+    return recommend_result
+    # return rank_result
 
 
 def special_mid_recommend(request_id, mid, uid, app_type, size,

requirements.txt

@@ -1,9 +1,7 @@
+gevent==20.9.0
+numpy==1.19.2
 Flask==1.1.2
 redis==3.5.3
-numpy==1.19.2
 PyMySQL==1.0.2
 requests==2.24.0
-psycopg2_binary==2.9.1
-gevent==20.9.0
 aliyun_python_sdk==2.2.0
-psycopg2==2.9.3

utils.py

@@ -272,18 +272,18 @@ class FilterVideos(object):
         st_viewed = time.time()
         filtered_viewed_result = self.filter_video_viewed(video_ids=filtered_pre_result)
         # et_viewed = time.time()
-        log_.info({
-            'logTimestamp': int(time.time() * 1000),
-            'pool_type': pool_type,
-            'request_id': self.request_id,
-            'app_type': self.app_type,
-            'mid': self.mid,
-            'uid': self.uid,
-            'operation': 'view_filter',
-            'request_videos': filtered_pre_result,
-            'view_filter_result': filtered_viewed_result,
-            'executeTime': (time.time() - st_viewed) * 1000
-        })
+        # log_.info({
+        #     'logTimestamp': int(time.time() * 1000),
+        #     'pool_type': pool_type,
+        #     'request_id': self.request_id,
+        #     'app_type': self.app_type,
+        #     'mid': self.mid,
+        #     'uid': self.uid,
+        #     'operation': 'view_filter',
+        #     'request_videos': filtered_pre_result,
+        #     'view_filter_result': filtered_viewed_result,
+        #     'executeTime': (time.time() - st_viewed) * 1000
+        # })
         if not filtered_viewed_result:
             return None
         filtered_viewed_videos = [int(video_id) for video_id in filtered_viewed_result]
@@ -297,18 +297,18 @@ class FilterVideos(object):
                     filtered_shield_video_ids = self.filter_shield_video(
                         video_ids=filtered_viewed_videos, shield_key_name_list=shield_key_name_list
                     )
-                    log_.info({
-                        'logTimestamp': int(time.time() * 1000),
-                        'pool_type': pool_type,
-                        'request_id': self.request_id,
-                        'app_type': self.app_type,
-                        'mid': self.mid,
-                        'uid': self.uid,
-                        'operation': 'shield_filter',
-                        'request_videos': filtered_viewed_videos,
-                        'shield_filter_result': filtered_shield_video_ids,
-                        'executeTime': (time.time() - st_viewed) * 1000
-                    })
+                    # log_.info({
+                    #     'logTimestamp': int(time.time() * 1000),
+                    #     'pool_type': pool_type,
+                    #     'request_id': self.request_id,
+                    #     'app_type': self.app_type,
+                    #     'mid': self.mid,
+                    #     'uid': self.uid,
+                    #     'operation': 'shield_filter',
+                    #     'request_videos': filtered_viewed_videos,
+                    #     'shield_filter_result': filtered_shield_video_ids,
+                    #     'executeTime': (time.time() - st_viewed) * 1000
+                    # })
                     return filtered_shield_video_ids
                 else:
                     return filtered_viewed_videos

video_recall.py

@@ -423,12 +423,12 @@ class PoolRecall(object):
             rov_pool_recall_result.extend(update_rov_dup_result)
         rov_pool_recall_result.sort(key=lambda x: x.get('rovScore', 0), reverse=True)
 
-        log_.info({
-            'logTimestamp': int(time.time() * 1000),
-            'request_id': self.request_id,
-            'operation': 'rov_pool_recall',
-            'executeTime': (time.time() - start_time) * 1000
-        })
+        # log_.info({
+        #     'logTimestamp': int(time.time() * 1000),
+        #     'request_id': self.request_id,
+        #     'operation': 'rov_pool_recall',
+        #     'executeTime': (time.time() - start_time) * 1000
+        # })
         return rov_pool_recall_result[:size]
 
     def flow_pool_recall(self, size=10, flow_pool_id=None):
@@ -507,15 +507,15 @@ class PoolRecall(object):
                 ge = gevent.spawn(self.check_video_counts, video_ids=filtered_result, flow_pool_mapping=video_mapping)
                 ge.join()
                 check_result = ge.get()
-                log_.info({
-                    'logTimestamp': int(time.time() * 1000),
-                    'request_id': self.request_id,
-                    'app_type': self.app_type,
-                    'mid': self.mid,
-                    'uid': self.uid,
-                    'operation': 'check_video_counts',
-                    'executeTime': (time.time() - st_check) * 1000
-                })
+                # log_.info({
+                #     'logTimestamp': int(time.time() * 1000),
+                #     'request_id': self.request_id,
+                #     'app_type': self.app_type,
+                #     'mid': self.mid,
+                #     'uid': self.uid,
+                #     'operation': 'check_video_counts',
+                #     'executeTime': (time.time() - st_check) * 1000
+                # })
 
                 for item in check_result:
                     video_id = int(item[0])
@@ -541,12 +541,12 @@ class PoolRecall(object):
 
             idx += get_size
 
-        log_.info({
-            'logTimestamp': int(time.time() * 1000),
-            'request_id': self.request_id,
-            'operation': 'flow_pool_recall',
-            'executeTime': (time.time() - start_time) * 1000
-        })
+        # log_.info({
+        #     'logTimestamp': int(time.time() * 1000),
+        #     'request_id': self.request_id,
+        #     'operation': 'flow_pool_recall',
+        #     'executeTime': (time.time() - start_time) * 1000
+        # })
 
         return flow_pool_recall_result[:size]
 
@@ -1290,12 +1290,12 @@ class PoolRecall(object):
                         else:
                             continue
 
-        log_.info({
-            'logTimestamp': int(time.time() * 1000),
-            'request_id': self.request_id,
-            'operation': 'rov_pool_recall_with_region',
-            'executeTime': (time.time() - start_time) * 1000
-        })
+        # log_.info({
+        #     'logTimestamp': int(time.time() * 1000),
+        #     'request_id': self.request_id,
+        #     'operation': 'rov_pool_recall_with_region',
+        #     'executeTime': (time.time() - start_time) * 1000
+        # })
         return recall_result[:size]
 
     def rov_pool_recall_with_region_by_h(self, province_code, size=4, key_flag=''):
@@ -1852,11 +1852,11 @@ class PoolRecall(object):
             self.redis_helper.set_data_to_redis(key_name=last_video_key, value=recall_data[-1][0],
                                                 expire_time=expire_time)
 
-        log_.info({
-            'logTimestamp': int(time.time() * 1000),
-            'request_id': self.request_id,
-            'operation': push_from,
-            'pool_recall_result': pool_recall_result,
-            'executeTime': (time.time() - start_time) * 1000
-        })
+        # log_.info({
+        #     'logTimestamp': int(time.time() * 1000),
+        #     'request_id': self.request_id,
+        #     'operation': push_from,
+        #     'pool_recall_result': pool_recall_result,
+        #     'executeTime': (time.time() - start_time) * 1000
+        # })
         return pool_recall_result[:size]