Merge branch 'search-data-list' into test

app.py

@@ -20,6 +20,7 @@ from multiprocessing import cpu_count, Process
 from utils import update_video_w_h_rate
 from user2new import user2new
 from params_helper import Params
+from manager_op import get_video_list, search_video
 # from werkzeug.middleware.profiler import ProfilerMiddleware
 # from geventwebsocket.handler import WebSocketHandler
 
@@ -238,6 +239,49 @@ def user_to_new():
         return json.dumps(result)
 
 
+# 管理后台算法视频列表可视化 - 视频数据表类型获取
+@app.route('/applet/video/get_video_type_list', methods=['GET', 'POST'])
+def get_video_type_list():
+    try:
+        data = [
+            {'dataListDesc': val.get('dataListDesc'), 'dataListCode': val.get('dataListCode')}
+            for key, val in config_.VIDEO_DATA_LIST_MAPPING.items()
+        ]
+        data.sort(key=lambda x: x['dataListCode'], reverse=False)
+        result = {'code': 200, 'message': 'success', 'data': data}
+        return json.dumps(result)
+    except Exception as e:
+        log_.error(traceback.format_exc())
+        result = {'code': -1, 'message': 'fail'}
+        return json.dumps(result)
+
+
+# 管理后台算法视频列表可视化 - 获取视频列表
+@app.route('/applet/video/get_online_list', methods=['GET', 'POST'])
+def get_video_online_list():
+    try:
+        request_data = json.loads(request.get_data())
+        ab_exp_code = request_data.get('abExpCode', None)
+        search_time = request_data.get('searchTime', None)
+        data_list_type = request_data.get('dataListType', None)
+        region_code = request_data.get('regionCode', None)
+        video_id = request_data.get('videoId', None)
+        page_num = request_data.get('pageNum', 1)
+        page_size = request_data.get('pageSize', 100)
+        if video_id is None:
+            result = get_video_list(ab_exp_code=ab_exp_code, search_time=search_time, data_list_type=data_list_type,
+                                    region_code=region_code, page_num=page_num, page_size=page_size)
+        else:
+            result = search_video(ab_exp_code=ab_exp_code, search_time=search_time, data_list_type=data_list_type,
+                                  region_code=region_code, video_id=video_id, page_num=page_num, page_size=page_size)
+        return json.dumps(result)
+
+    except Exception as e:
+        log_.error(traceback.format_exc())
+        result = {'code': -1, 'message': 'fail'}
+        return json.dumps(result)
+
+
 # app热榜
 @app.route('/app/video/hot_list', methods=['GET', 'POST'])
 def app_video_hot_list():

config.py

@@ -372,6 +372,44 @@ class BaseConfig(object):
     # 限流视频分发数记录 redis key前缀，完整格式：'com.weiqu.video.limit.distribute.count.{videoId}'
     KEY_NAME_PREFIX_LIMIT_VIDEO_DISTRIBUTE_COUNT = 'com.weiqu.video.limit.distribute.count.'
 
+    # 视频数据表类型 - 管理后台算法视频列表可视化
+    VIDEO_DATA_LIST_MAPPING = {
+        0: {
+            "dataListDesc": "全部", "dataListCode": 0,
+            "keyPrefixList": [
+                {"dataListCode": 1, "keyPrefix": RECALL_KEY_NAME_PREFIX_REGION_BY_H},
+                {"dataListCode": 2, "keyPrefix": RECALL_KEY_NAME_PREFIX_DUP1_REGION_24H_H},
+                {"dataListCode": 3, "keyPrefix": RECALL_KEY_NAME_PREFIX_DUP2_REGION_24H_H},
+                {"dataListCode": 4, "keyPrefix": RECALL_KEY_NAME_PREFIX_DUP_REGION_H},
+            ]
+        },
+        1: {"dataListDesc": "地域小时级", "dataListCode": 1, "keyPrefix": RECALL_KEY_NAME_PREFIX_REGION_BY_H},
+        2: {"dataListDesc": "地域相对24小时级", "dataListCode": 2, "keyPrefix": RECALL_KEY_NAME_PREFIX_DUP1_REGION_24H_H},
+        3: {"dataListDesc": "非地域相对24小时级", "dataListCode": 3, "keyPrefix": RECALL_KEY_NAME_PREFIX_DUP2_REGION_24H_H},
+        4: {"dataListDesc": "大列表", "dataListCode": 4, "keyPrefix": RECALL_KEY_NAME_PREFIX_DUP_REGION_H},
+    }
+    VIDEO_DATA_LIST_AB_EXP_CODE_MAPPING = {
+        "055": {"app_type": 0, "data_key": "data1", "rule_key": "rule2"},
+        "068": {"app_type": 5, "data_key": "data1", "rule_key": "rule2"},
+        "069": {"app_type": 19, "data_key": "data1", "rule_key": "rule2"},
+        "072": {"app_type": 4, "data_key": "data1", "rule_key": "rule2"},
+        "073": {"app_type": 6, "data_key": "data1", "rule_key": "rule2"},
+        "074": {"app_type": 18, "data_key": "data1", "rule_key": "rule2"},
+        "079": {"app_type": 0, "data_key": "data1", "rule_key": "rule3"},
+        "081": {"app_type": 5, "data_key": "data2", "rule_key": "rule2"},
+        "082": {"app_type": 5, "data_key": "data3", "rule_key": "rule2"},
+        "083": {"app_type": 5, "data_key": "data4", "rule_key": "rule2"},
+        "084": {"app_type": 0, "data_key": "data2", "rule_key": "rule2"},
+    }
+    REGION_CODE = {
+        '河北省': '130000', '山西省': '140000', '辽宁省': '210000', '吉林省': '220000', '黑龙江省': '230000', '江苏省': '320000',
+        '浙江省': '330000', '安徽省': '340000', '福建省': '350000', '江西省': '360000', '山东省': '370000', '河南省': '410000',
+        '湖北省': '420000', '湖南省': '430000', '广东省': '440000', '海南省': '460000', '四川省': '510000', '贵州省': '520000',
+        '云南省': '530000', '陕西省': '610000', '甘肃省': '620000', '青海省': '630000', '台湾省': '710000', '北京': '110000',
+        '天津': '120000', '内蒙古': '150000', '上海': '310000', '广西': '450000', '重庆': '500000', '西藏': '540000',
+        '宁夏': '640000', '新疆': '650000', '香港': '810000', '澳门': '820000'
+    }
+
 
 class DevelopmentConfig(BaseConfig):
     """开发环境配置"""

db_helper.py

@@ -1,9 +1,9 @@
 import traceback
 import time
 import redis
-import psycopg2
+# import psycopg2
 import pymysql
-from psycopg2 import pool as pgpool
+# from psycopg2 import pool as pgpool
 from config import set_config
 from log import Log
 
@@ -156,6 +156,29 @@ class RedisHelper(object):
         #     })
         return data
 
+    def get_all_data_from_zset(self, key_name, desc=True, with_scores=False):
+        """
+        获取zset中所有元素的值
+        :param key_name: key
+        :param desc: 分数排序方式，默认从大到小
+        :param with_scores: 是否获取元素的分数，默认 False，只获取元素的值
+        :return: data 元素值列表（不包含分数）,value(videoId)类型转换为int, 包含分数时不进行类型转换
+        """
+        conn = self.connect()
+        if not conn.exists(key_name):
+            return None
+        data = []
+        start = 0
+        step = 100
+        while True:
+            end = start + step - 1
+            temp = conn.zrange(key_name, start, end, desc, with_scores)
+            if not temp:
+                break
+            data.extend(temp)
+            start += step
+        return data
+
     def get_score_with_value(self, key_name, value):
         """
         在zset中，根据元素的value获取对应的score
@@ -168,6 +191,22 @@ class RedisHelper(object):
             return None
         return conn.zscore(key_name, value)
 
+    def get_rank_with_value(self, key_name, value, desc=False):
+        """
+        在zset中，根据元素的value获取对应排名
+        :param key_name: key
+        :param value: 元素的值
+        :param desc: 是否倒序 type-bool 默认：False-按照score从小到大
+        :return: rank value对应的rank，从0开始，不存在返回None
+        """
+        conn = self.connect()
+        if not conn.exists(key_name):
+            return None
+        if desc is True:
+            return conn.zrevrank(key_name, value)
+        else:
+            return conn.zrank(key_name, value)
+
     def update_score_with_value(self, key_name, value, score, expire_time=24*3600):
         """
         在zset中，修改元素value对应的score

manager_op.py

@@ -0,0 +1,175 @@
+from config import set_config
+from log import Log
+from db_helper import RedisHelper
+
+config_ = set_config()
+log_ = Log()
+
+
+def get_video_list(ab_exp_code, search_time, data_list_type, region_code, page_num=1, page_size=100):
+    """
+    管理后台算法视频列表可视化 - 获取视频列表
+    :param ab_exp_code: 实验id，type-string
+    :param search_time: 时间，type-string，格式: %y%m%d.%h，eg: '20220714.05'
+    :param data_list_type: 视频数据表类型，type-int
+    :param region_code: 地域（省份），type-string，eg: '110000
+    :param page_num: 页码，type-int，默认: 1
+    :param page_size: 每页请求数，type-int，默认: 100
+    :return: result
+    """
+    redis_helper = RedisHelper()
+    if ab_exp_code is None or search_time is None or data_list_type is None or region_code is None:
+        data = []
+        data_count = 0
+        result = {'code': 200, 'message': 'success', 'dataCount': data_count, 'data': data}
+        return result
+    data = []
+    key_params = config_.VIDEO_DATA_LIST_AB_EXP_CODE_MAPPING.get(ab_exp_code)
+    app_type = key_params.get('app_type')
+    data_key = key_params.get('data_key')
+    rule_key = key_params.get('rule_key')
+    if data_list_type == 0:
+        # 视频数据表类型 选择为全部
+        key_prefix_list = config_.VIDEO_DATA_LIST_MAPPING.get(data_list_type).get('keyPrefixList')
+        for item in key_prefix_list:
+            data_list_code = item.get('dataListCode')
+            key_prefix = item.get('keyPrefix')
+            key_name = f"{key_prefix}{region_code}.{app_type}.{data_key}.{rule_key}.{search_time}"
+            key_data = redis_helper.get_all_data_from_zset(key_name=key_name, desc=True, with_scores=True)
+            if key_data is None:
+                continue
+            videos = [
+                {
+                    'abExpCode': ab_exp_code,
+                    'searchTime': search_time,
+                    'dataListType': data_list_code,
+                    'regionCode': region_code,
+                    'videoId': int(video_id),
+                    'score': float(score),
+                }
+                for video_id, score in key_data
+            ]
+            data.extend(videos)
+    else:
+        key_prefix = config_.VIDEO_DATA_LIST_MAPPING.get(data_list_type).get('keyPrefix')
+        key_name = f"{key_prefix}{region_code}.{app_type}.{data_key}.{rule_key}.{search_time}"
+        key_data = redis_helper.get_all_data_from_zset(key_name=key_name, desc=True, with_scores=True)
+        if key_data is None:
+            videos = []
+        else:
+            videos = [
+                {
+                    'abExpCode': ab_exp_code,
+                    'searchTime': search_time,
+                    'dataListType': data_list_type,
+                    'regionCode': region_code,
+                    'videoId': int(video_id),
+                    'score': float(score),
+                }
+                for video_id, score in key_data
+            ]
+        data.extend(videos)
+    # 添加"rank"
+    final_data = []
+    for i, item in enumerate(data):
+        item['rank'] = i + 1
+        final_data.append(item)
+    result = {
+        "code": 200,
+        "message": "success",
+        "dataCount": len(final_data),
+        "data": final_data[(page_num-1)*page_size:page_num*page_size]
+    }
+    return result
+
+
+def search_video(ab_exp_code, search_time, data_list_type, region_code, video_id, page_num=1, page_size=100):
+    """
+    管理后台算法视频列表可视化 - 搜索视频id
+    :param ab_exp_code: 实验id，type-string
+    :param search_time: 时间，type-string，格式: %y%m%d.%h，eg: '20220714.05'
+    :param data_list_type: 视频数据表类型，type-int
+    :param region_code: 地域（省份），type-string，eg: '110000
+    :param video_id: 视频id，type-int
+    :param page_num: 页码，type-int，默认: 1
+    :param page_size: 每页请求数，type-int，默认: 100
+    :return: result
+    """
+    redis_helper = RedisHelper()
+    if ab_exp_code is None or search_time is None or data_list_type is None or region_code is None or video_id is None:
+        data = []
+        data_count = 0
+        result = {'code': 200, 'message': 'success', 'dataCount': data_count, 'data': data}
+        return result
+    data = []
+    key_params = config_.VIDEO_DATA_LIST_AB_EXP_CODE_MAPPING.get(ab_exp_code)
+    app_type = key_params.get('app_type')
+    data_key = key_params.get('data_key')
+    rule_key = key_params.get('rule_key')
+    if data_list_type == 0:
+        # 视频数据表类型 选择为全部
+        key_prefix_list = config_.VIDEO_DATA_LIST_MAPPING.get(data_list_type).get('keyPrefixList')
+        for item in key_prefix_list:
+            data_list_code = item.get('dataListCode')
+            key_prefix = item.get('keyPrefix')
+            key_name = f"{key_prefix}{region_code}.{app_type}.{data_key}.{rule_key}.{search_time}"
+            rank = redis_helper.get_rank_with_value(key_name=key_name, value=int(video_id), desc=True)
+            score = redis_helper.get_score_with_value(key_name=key_name, value=int(video_id))
+            if rank is None or score is None:
+                continue
+            videos = {
+                'abExpCode': ab_exp_code,
+                'searchTime': search_time,
+                'dataListType': data_list_code,
+                'regionCode': region_code,
+                'videoId': int(video_id),
+                'score': float(score),
+                'rank': int(rank) + 1,
+            }
+            data.append(videos)
+        data.sort(key=lambda x: x['dataListType'], reverse=False)
+    elif region_code == '0':
+        # 地域 选择为全部
+        code_list = [code for _, code in config_.REGION_CODE.items()]
+        key_prefix = config_.VIDEO_DATA_LIST_MAPPING.get(data_list_type).get('keyPrefix')
+        for code in code_list:
+            key_name = f"{key_prefix}{code}.{app_type}.{data_key}.{rule_key}.{search_time}"
+            rank = redis_helper.get_rank_with_value(key_name=key_name, value=int(video_id), desc=True)
+            score = redis_helper.get_score_with_value(key_name=key_name, value=int(video_id))
+            if rank is None or score is None:
+                continue
+            videos = {
+                'abExpCode': ab_exp_code,
+                'searchTime': search_time,
+                'dataListType': data_list_type,
+                'regionCode': code,
+                'videoId': int(video_id),
+                'score': float(score),
+                'rank': int(rank) + 1,
+            }
+            data.append(videos)
+        data.sort(key=lambda x: x['regionCode'], reverse=False)
+    else:
+        key_prefix = config_.VIDEO_DATA_LIST_MAPPING.get(data_list_type).get('keyPrefix')
+        key_name = f"{key_prefix}{region_code}.{app_type}.{data_key}.{rule_key}.{search_time}"
+        rank = redis_helper.get_rank_with_value(key_name=key_name, value=int(video_id), desc=True)
+        score = redis_helper.get_score_with_value(key_name=key_name, value=int(video_id))
+        if rank is not None and score is not None:
+            videos = {
+                'abExpCode': ab_exp_code,
+                'searchTime': search_time,
+                'dataListType': data_list_type,
+                'regionCode': region_code,
+                'videoId': int(video_id),
+                'score': float(score),
+                'rank': int(rank) + 1,
+            }
+            data.append(videos)
+
+    result = {
+        "code": 200,
+        "message": "success",
+        "dataCount": len(data),
+        "data": data[(page_num - 1) * page_size:page_num * page_size]
+    }
+    return result