优化匹配逻辑

applications/functions/ask_kimi.py

@@ -20,11 +20,6 @@ def ask_kimi(question):
             "key_words": [],  # 返回三个关键词
             "search_keys": [], # 标题可能的搜索关键词，返回 3 个
             "extra_keys": [], # 关心这个视频的用户还会关心哪些关键词， 返回 3 个
-            "tone": 标题的语气，用一个词概括,
-            "target_audience": 标题的受众群体，用一个词概括,
-            "target_age": 标题的受众年龄段，从 老年， 中年，青年，小孩， 不限， 这五个里面选择，
-            "target_gender": 受众性别,
-            "address": 受众可能属于哪个城市,
             "theme": 标题的主题， 用一个词概括
         }
         只需要返回一个 json，key 和上面的一样，

applications/functions/auto_white.py

@@ -41,7 +41,6 @@ def auto_white(root_share_id):
     }
     payload = json.dumps(dd)
     cookie = get_cookie()
-    print(cookie)
     headers = {
         'accept': 'application/json',
         'accept-language': 'en',

applications/functions/common.py

@@ -0,0 +1,50 @@
+"""
+@author: luojunhui
+"""
+import json
+import uuid
+import requests
+import urllib.parse
+
+from applications.functions.auto_white import auto_white
+
+
+def create_gzh_path(video_id, shared_uid):
+    """
+    :param video_id: 视频 id
+    :param shared_uid: 分享 id
+    """
+    root_share_id = str(uuid.uuid4())
+    url = f"pages/user-videos?id={video_id}&su={shared_uid}&fromGzh=1&rootShareId={root_share_id}&shareId={root_share_id}"
+    # 自动把 root_share_id 加入到白名单
+    auto_white(root_share_id)
+    return root_share_id, f"pages/category?jumpPage={urllib.parse.quote(url, safe='')}"
+
+
+def request_for_info(video_id):
+    """
+    请求数据
+    :param video_id:
+    :return:
+    """
+    url = "https://longvideoapi.piaoquantv.com/longvideoapi/openapi/video/batchSelectVideoInfo"
+    data = {
+        "videoIdList": [video_id]
+    }
+    header = {
+        "Content-Type": "application/json",
+    }
+    response = requests.post(url, headers=header, data=json.dumps(data))
+    return response.json()
+
+
+def choose_video(result):
+    """
+    :param result: 计算出来的结果
+    :return: uid, video_id
+    """
+    score1 = result['s1_score']
+    if score1 > 0:
+        return result['s1_uid'], result['s1_vid']
+    else:
+        return None, None

applications/functions/mysql.py

@@ -0,0 +1,56 @@
+"""
+@author: luojunhui
+mysql 方法
+"""
+import json
+
+import pymysql
+
+
+def select(sql):
+    """
+    查询
+    :param sql:
+    :return:
+    """
+    connection = pymysql.connect(
+        host="rm-bp1159bu17li9hi94.mysql.rds.aliyuncs.com",  # 数据库IP地址，内网地址
+        port=3306,  # 端口号
+        user="crawler",  # mysql用户名
+        passwd="crawler123456@",  # mysql用户登录密码
+        db="piaoquan-crawler",  # 数据库名
+        charset="utf8mb4"  # 如果数据库里面的文本是utf8编码的，charset指定是utf8
+    )
+    cursor = connection.cursor()
+    cursor.execute(sql)
+    data = cursor.fetchall()
+    return data
+
+
+def select_pq_videos():
+    """
+    查询
+    :return: info_list
+    """
+    connection = pymysql.connect(
+        host="rm-bp1k5853td1r25g3n690.mysql.rds.aliyuncs.com",  # 数据库IP地址，内网地址
+        port=3306,  # 端口号
+        user="wx2016_longvideo",  # mysql用户名
+        passwd="wx2016_longvideoP@assword1234",  # mysql用户登录密码
+        db="incentive",  # 数据库名
+        charset="utf8mb4"  # 如果数据库里面的文本是utf8编码的，charset指定是utf8
+    )
+    sql = "select video_id, key_words, search_keys, extra_keys from video_content"
+    cursor = connection.cursor()
+    cursor.execute(sql)
+    data = cursor.fetchall()
+    result = [
+        {
+            "video_id": line[0],
+            "key_words": json.loads(line[1]),
+            "search_keys": json.loads(line[2]),
+            "extra_keys": json.loads(line[3]),
+        }
+        for line in data
+    ]
+    return result

applications/match_alg/__init__.py

@@ -2,3 +2,4 @@
 @author: luojunhui
 匹配算法
 """
+from .rank import best_choice

applications/match_alg/rank.py

@@ -1,13 +1,62 @@
 """
 @author: luojunhui
 """
+from applications.match_alg.recall import recall_videos
 
 
-def best_choice():
+def jac_score(d1, d2):
+    """
+    通过交并集来判断
+    :param d1:
+    :param d2:
+    :return:
+    """
+    f1_keys = set(d1["key_words"])
+    f2_keys = set(d2["key_words"])
+    keys_union = f1_keys | f2_keys
+    keys_intersection = f1_keys & f2_keys
+    f1_search_keys = set(d1["search_keys"])
+    f2_search_keys = set(d2["search_keys"])
+    search_keys_union = f1_search_keys | f2_search_keys
+    search_keys_intersection = f1_search_keys & f2_search_keys
+    f1_extra_keys = set(d1["extra_keys"])
+    f2_extra_keys = set(d2["extra_keys"])
+    extra_keys_union = f1_extra_keys | f2_extra_keys
+    extra_keys_intersection = f1_extra_keys & f2_extra_keys
+    score_1 = len(keys_intersection) / len(keys_union)
+    score_2 = len(search_keys_intersection) / len(search_keys_union)
+    score_3 = len(extra_keys_intersection) / len(extra_keys_union)
+    return score_1 * 0.4 + score_2 * 0.4 + score_3 * 0.2, d2['video_id']
+
+
+def best_choice(params_obj, request_param, trace_id):
     """
     计算，返回出最合适的 video_id
     :return: video_id
     """
-    best_video_id = 0
-    return best_video_id
+    pq_list, search_list = recall_videos(params=request_param, trace_id=trace_id)
+
+    def best_video_id(target_list):
+        """
+        :param target_list:
+        :return:
+        """
+        score_list = []
+        for video_obj in target_list:
+            try:
+                score, video_id = jac_score(d1=params_obj, d2=video_obj)
+                score_list.append((video_id, score))
+            except Exception as e:
+                print(e)
+        sorted_list = sorted(score_list, key=lambda x: x[1], reverse=True)
+        return sorted_list[0]
 
+    best_search_tuple = best_video_id(search_list)
+    if best_search_tuple[1] > 0:
+        return best_search_tuple[0]
+    else:
+        best_pq_tuple = best_video_id(pq_list)
+        if best_pq_tuple[1] > 0:
+            return best_pq_tuple[0]
+        else:
+            return None

applications/match_alg/recall.py

@@ -1,16 +1,397 @@
 """
 @author: luojunhui
 """
+import os
+import json
+import time
+from concurrent.futures import ThreadPoolExecutor
 
+import requests
 
-def recall_videos(params):
+from applications.log import logging
+from applications.functions.mysql import select, select_pq_videos
+from applications.functions.ask_kimi import ask_kimi
+
+
+gh_id_dict = {
+    "gh_01f8afd03366": {
+        "uid": 69637520,
+        "nick_name": "非亲非故"
+    },
+    "gh_058e41145a0c": {
+        "uid": 69637476,
+        "nick_name": "甜腻梦话"
+    },
+    "gh_084a485e859a": {
+        "uid": 69637472,
+        "nick_name": "梦星月"
+    },
+    "gh_0921c03402cd": {
+        "uid": 69637531,
+        "nick_name": "你的女友"
+    },
+    "gh_0c89e11f8bf3": {
+        "uid": 69637508,
+        "nick_name": "粟米"
+    },
+    "gh_171cec079b2a": {
+        "uid": 69637501,
+        "nick_name": "海上"
+    },
+    "gh_183d80deffb8": {
+        "uid": 69637491,
+        "nick_name": "论趣"
+    },
+    "gh_1ee2e1b39ccf": {
+        "uid": 69637473,
+        "nick_name": "纵有疾风起"
+    },
+    "gh_234ef02cdee5": {
+        "uid": 69637513,
+        "nick_name": "夹逼"
+    },
+    "gh_26a307578776": {
+        "uid": 69637490,
+        "nick_name": "最宝贝的宝贝"
+    },
+    "gh_29074b51f2b7": {
+        "uid": 69637530,
+        "nick_name": "沉舸"
+    },
+    "gh_2b8c6aa035ae": {
+        "uid": 69637470,
+        "nick_name": "懶得取名"
+    },
+    "gh_34318194fd0e": {
+        "uid": 69637517,
+        "nick_name": "徒四壁"
+    },
+    "gh_3845af6945d0": {
+        "uid": 69637545,
+        "nick_name": "秋水娉婷"
+    },
+    "gh_3ac6d7208961": {
+        "uid": 69637497,
+        "nick_name": "小熊的少女梦"
+    },
+    "gh_3c7d38636846": {
+        "uid": 69637519,
+        "nick_name": "油腻腻"
+    },
+    "gh_3df10391639c": {
+        "uid": 69637541,
+        "nick_name": "六郎娇面"
+    },
+    "gh_40a0ad154478": {
+        "uid": 69637516,
+        "nick_name": "禁止"
+    },
+    "gh_424c8eeabced": {
+        "uid": 69637522,
+        "nick_name": "认命"
+    },
+    "gh_4568b5a7e2fe": {
+        "uid": 69637482,
+        "nick_name": "香腮"
+    },
+    "gh_45beb952dc74": {
+        "uid": 69637488,
+        "nick_name": "毋庸"
+    },
+    "gh_484de412b0ef": {
+        "uid": 69637481,
+        "nick_name": "婪"
+    },
+    "gh_4c058673c07e": {
+        "uid": 69637474,
+        "nick_name": "影帝"
+    },
+    "gh_538f78f9d3aa": {
+        "uid": 69637478,
+        "nick_name": "伤痕"
+    },
+    "gh_56a6765df869": {
+        "uid": 69637514,
+        "nick_name": "风月"
+    },
+    "gh_56ca3dae948c": {
+        "uid": 69637538,
+        "nick_name": "留下太多回忆"
+    },
+    "gh_5e543853d8f0": {
+        "uid": 69637543,
+        "nick_name": "不知春秋"
+    },
+    "gh_5ff48e9fb9ef": {
+        "uid": 69637494,
+        "nick_name": "寻她找他"
+    },
+    "gh_671f460c856c": {
+        "uid": 69637523,
+        "nick_name": "绝不改悔"
+    },
+    "gh_6b7c2a257263": {
+        "uid": 69637528,
+        "nick_name": "奶牙"
+    },
+    "gh_6d205db62f04": {
+        "uid": 69637509,
+        "nick_name": "怕羞"
+    },
+    "gh_6d9f36e3a7be": {
+        "uid": 69637498,
+        "nick_name": "望长安"
+    },
+    "gh_73be0287bb94": {
+        "uid": 69637537,
+        "nick_name": "戏剧"
+    },
+    "gh_744cb16f6e16": {
+        "uid": 69637505,
+        "nick_name": "反駁"
+    },
+    "gh_7b4a5f86d68c": {
+        "uid": 69637477,
+        "nick_name": "我很想你"
+    },
+    "gh_7bca1c99aea0": {
+        "uid": 69637511,
+        "nick_name": "从小就很傲"
+    },
+    "gh_7e5818b2dd83": {
+        "uid": 69637532,
+        "nick_name": "二八佳人"
+    },
+    "gh_89ef4798d3ea": {
+        "uid": 69637533,
+        "nick_name": "彼岸花"
+    },
+    "gh_901b0d722749": {
+        "uid": 69637518,
+        "nick_name": "深情不为我"
+    },
+    "gh_9161517e5676": {
+        "uid": 69637495,
+        "nick_name": "折磨"
+    },
+    "gh_93e00e187787": {
+        "uid": 69637504,
+        "nick_name": "理会"
+    },
+    "gh_9877c8541764": {
+        "uid": 69637506,
+        "nick_name": "我沿着悲伤"
+    },
+    "gh_9cf3b7ff486b": {
+        "uid": 69637492,
+        "nick_name": "hoit"
+    },
+    "gh_9e559b3b94ca": {
+        "uid": 69637471,
+        "nick_name": "我与你相遇"
+    },
+    "gh_9f8dc5b0c74e": {
+        "uid": 69637496,
+        "nick_name": "港口"
+    },
+    "gh_a182cfc94dad": {
+        "uid": 69637539,
+        "nick_name": "四海八荒"
+    },
+    "gh_a2901d34f75b": {
+        "uid": 69637535,
+        "nick_name": "听腻了谎话"
+    },
+    "gh_a307072c04b9": {
+        "uid": 69637521,
+        "nick_name": "踏步"
+    },
+    "gh_a6351b447819": {
+        "uid": 69637540,
+        "nick_name": "七猫酒馆"
+    },
+    "gh_ac43e43b253b": {
+        "uid": 69637499,
+        "nick_name": "一厢情愿"
+    },
+    "gh_adca24a8f429": {
+        "uid": 69637483,
+        "nick_name": "对你何止一句喜欢"
+    },
+    "gh_b15de7c99912": {
+        "uid": 69637536,
+        "nick_name": "糖炒板栗"
+    },
+    "gh_b32125c73861": {
+        "uid": 69637493,
+        "nick_name": "发尾"
+    },
+    "gh_b3ffc1ca3a04": {
+        "uid": 69637546,
+        "nick_name": "主宰你心"
+    },
+    "gh_b8baac4296cb": {
+        "uid": 69637489,
+        "nick_name": "生性"
+    },
+    "gh_b9b99173ff8a": {
+        "uid": 69637524,
+        "nick_name": "养一只月亮"
+    },
+    "gh_bd57b6978e06": {
+        "uid": 69637527,
+        "nick_name": "厌遇"
+    },
+    "gh_be8c29139989": {
+        "uid": 69637502,
+        "nick_name": "不负"
+    },
+    "gh_bfe5b705324a": {
+        "uid": 69637529,
+        "nick_name": "乐极"
+    },
+    "gh_bff0bcb0694a": {
+        "uid": 69637534,
+        "nick_name": "简迷离"
+    },
+    "gh_c69776baf2cd": {
+        "uid": 69637512,
+        "nick_name": "骄纵"
+    },
+    "gh_c91b42649690": {
+        "uid": 69637503,
+        "nick_name": "荟萃"
+    },
+    "gh_d2cc901deca7": {
+        "uid": 69637487,
+        "nick_name": "恶意调笑"
+    },
+    "gh_d5f935d0d1f2": {
+        "uid": 69637500,
+        "nick_name": "青少年哪吒"
+    },
+    "gh_da76772d8d15": {
+        "uid": 69637526,
+        "nick_name": "独揽风月"
+    },
+    "gh_de9f9ebc976b": {
+        "uid": 69637475,
+        "nick_name": "剑出鞘恩怨了"
+    },
+    "gh_e0eb490115f5": {
+        "uid": 69637486,
+        "nick_name": "赋别"
+    },
+    "gh_e24da99dc899": {
+        "uid": 69637484,
+        "nick_name": "恋雨夏季"
+    },
+    "gh_e2576b7181c6": {
+        "uid": 69637515,
+        "nick_name": "满天星"
+    },
+    "gh_e75dbdc73d80": {
+        "uid": 69637542,
+        "nick_name": "情战"
+    },
+    "gh_e9d819f9e147": {
+        "uid": 69637525,
+        "nick_name": "与卿"
+    },
+    "gh_efaf7da157f5": {
+        "uid": 69637547,
+        "nick_name": "心野性子浪"
+    },
+    "gh_f4594783f5b8": {
+        "uid": 69637544,
+        "nick_name": "自缚"
+    },
+    "gh_fe6ef3a65a48": {
+        "uid": 69637480,
+        "nick_name": "风间"
+    }
+}
+
+
+def ask_kimi_and_save_to_local(info_tuple):
+    """
+    save file to local
+    :return:
+    """
+    title, trace_id, save_path = info_tuple[0], info_tuple[1], info_tuple[2]
+    if os.path.exists(save_path):
+        logging(
+            code="1002",
+            info="该 video 信息已经挖掘完成---{}".format(title),
+            function="ask_kimi_and_save_to_local",
+            trace_id=trace_id,
+        )
+    else:
+        os.makedirs(os.path.dirname(save_path), exist_ok=True)
+        if not title:
+            result = {}
+        else:
+            result = ask_kimi(title)
+        logging(
+            code="1002",
+            info="kimi-result",
+            data=result,
+            trace_id=trace_id,
+            function="ask_kimi_and_save_to_local"
+        )
+        with open(save_path, "w", encoding="utf-8") as f:
+            f.write(json.dumps(result, ensure_ascii=False))
+
+
+def recall_videos(params, trace_id):
     """
     通过请求的数据来召回视频
+    :param trace_id:
     :param params: 请求参数
-    :return:
+    :return: file_list
     """
     title = params['title']
-    content = params['content']
-
+    # content = params['content']
+    ghId = params['ghId']
+    user_id = gh_id_dict[ghId]['uid']
 
+    # 在外面搜索视频
+    payload = {
+        "ghId": ghId,
+        "search_keys": [title],
+        "trace_id": trace_id
+    }
+    # print(payload)
+    url = "http://61.48.133.26:8111/search_videos"
+    requests.post(url, json=payload)
+    # print("请求完成")
+    time.sleep(30)
+    select_sql = "select video_id, video_title from crawler_video where platform='weixin_search' and user_id = '{}' order by update_time DESC limit 10".format(
+        user_id)
+    out_video_list = select(sql=select_sql)
+    dir_path = os.path.join(os.getcwd(), 'applications', 'static', "out_videos")
+    os.makedirs(os.path.dirname(dir_path), exist_ok=True)
+    done_list = os.listdir(dir_path)
+    process_list = [
+        (
+            i[1],
+            trace_id,
+            os.path.join(dir_path, "{}.json".format(i[0]))
+        ) for i in out_video_list if not "{}.json".format(i[0]) in done_list
+    ]
+    with ThreadPoolExecutor(max_workers=10) as pool:
+        pool.map(ask_kimi_and_save_to_local, process_list)
 
+    # 在两边召回视频
+    # pq_videos
+    recall_video_list = select_pq_videos()
+    dirs_1 = os.path.join(os.getcwd(), 'applications', 'static', 'out_videos')
+    file_list = [os.path.join(dirs_1, file) for file in os.listdir(dirs_1) if file.endswith(".json")]
+    search_list = []
+    for file in file_list:
+        with open(file, encoding="utf-8") as f:
+            obj = json.loads(f.read())
+            if obj:
+                obj['video_id'] = file.split("/")[-1].replace('.json', '')
+        search_list.append(obj)
+    return recall_video_list, search_list

applications/process.py

@@ -4,67 +4,11 @@
 """
 
 import os
-import json
-import uuid
-import requests
-import urllib.parse
-from datetime import datetime, timedelta
 
 from applications.log import logging
 from applications.functions.ask_kimi import ask_kimi
-from applications.functions.calculate import title_mix
-from applications.functions.auto_white import auto_white
-
-
-def create_gzh_path(video_id, shared_uid):
-    """
-    :param video_id: 视频 id
-    :param shared_uid: 分享 id
-    """
-    root_share_id = str(uuid.uuid4())
-    url = f"pages/user-videos?id={video_id}&su={shared_uid}&fromGzh=1&rootShareId={root_share_id}&shareId={root_share_id}"
-    # 自动把 root_share_id 加入到白名单
-    auto_white(root_share_id)
-    return root_share_id, f"pages/category?jumpPage={urllib.parse.quote(url, safe='')}"
-
-
-def request_for_info(video_id):
-    """
-    请求数据
-    :param video_id:
-    :return:
-    """
-    url = "https://longvideoapi.piaoquantv.com/longvideoapi/openapi/video/batchSelectVideoInfo"
-    data = {
-        "videoIdList": [video_id]
-    }
-    header = {
-        "Content-Type": "application/json",
-    }
-    response = requests.post(url, headers=header, data=json.dumps(data))
-    return response.json()
-
-
-def choose_video(result):
-    """
-    :param result: 计算出来的结果
-    :return: uid, video_id
-    """
-    # 判断 score
-    # score1, score2 = result['s1_score'], result['s2_score']
-    score1 = result['s1_score']
-    if score1 > 0:
-        return result['s1_uid'], result['s1_vid']
-    else:
-        return None, None
-    # if score1 == 0 and score2 == 0:
-    #     return None, None
-    # elif score1 == 0 and score2 > 0:
-    #     return result['s2_uid'], result['s2_vid']
-    # elif score1 > 0 and score2 == 0:
-    #     return result['s1_uid'], result['s1_vid']
-    # elif score1 > 0 and score2 > 0:
-    #     return result['s1_uid'], result['s1_vid']
+from applications.match_alg import best_choice
+from applications.functions.common import *
 
 
 class ProcessParams(object):
@@ -124,20 +68,13 @@ class ProcessParams(object):
             with open(save_path, "w", encoding="utf-8") as f:
                 f.write(json.dumps(result, ensure_ascii=False))
 
-    def process(self, data):
+    def deal(self, data):
         """执行代码"""
-        today = datetime.today()
-        yesterday = today - timedelta(days=1)
-        yesterday_str = yesterday.strftime("%Y%m%d")
-        logging(
-            code="1002",
-            info="昨日的时间戳是---{}".format(yesterday_str),
-            function="process",
-            trace_id=self.trace_id,
-        )
         params = self.get_params(data)
         title = params['title']
-        account_name = params['accountName']
+        # account_name = params['accountName']
+        # ghId = params['ghId']
+
         title_p = os.path.join(os.getcwd(), 'applications', 'static', "titles", "{}.json".format(title))
         if os.path.exists(title_p):
             logging(
@@ -146,33 +83,42 @@ class ProcessParams(object):
                 function="process",
                 trace_id=self.trace_id
             )
-            result = title_mix(title_p=title_p, dt=yesterday_str, trace_id=self.trace_id)
         else:
             self.ask_kimi_and_save_to_local(title)
-            result = title_mix(title_p=title_p, dt=yesterday_str, trace_id=self.trace_id)
-        uid, video_id = choose_video(result)
+
+        with open(title_p, encoding="utf-8") as f:
+            params_obj = json.loads(f.read())
+
+        best_video_id = best_choice(
+            params_obj=params_obj,
+            trace_id=self.trace_id,
+            request_param=params
+        )
         logging(
             code="1002",
-            info="best video_id --{}".format(video_id),
+            info="best video_id --{}".format(best_video_id),
             function="process",
             trace_id=self.trace_id
         )
-        if video_id and uid:
-            root_share_id, productionPath = create_gzh_path(video_id=video_id, shared_uid=uid)
-            logging(
-                code="1002",
-                info="root_share_id --{}, productionPath -- {}".format(root_share_id, productionPath),
-                function="process",
-                trace_id=self.trace_id
-            )
-            response = request_for_info(video_id)
+
+        if best_video_id:
+            print(best_video_id)
+            response = request_for_info(best_video_id)
             productionCover = response['data'][0]['shareImgPath']
             productionName = response["data"][0]['title']
             videoUrl = response['data'][0]['videoPath']
+            user_id = response['data'][0]['user']['uid']
             programAvatar = "/static/logo.png"
             programId = "wx69c36def517d687a"
             programName = "票圈最惊奇"
             source = "Web"
+            root_share_id, productionPath = create_gzh_path(video_id=best_video_id, shared_uid=user_id)
+            logging(
+                code="1002",
+                info="root_share_id --{}, productionPath -- {}".format(root_share_id, productionPath),
+                function="process",
+                trace_id=self.trace_id
+            )
             result = {
                 "productionCover": productionCover,
                 "productionName": productionName,
@@ -191,7 +137,7 @@ class ProcessParams(object):
                 trace_id=self.trace_id,
                 data={
                     "rootShareId": root_share_id,
-                    "videoId": video_id
+                    "videoId": best_video_id
                 }
             )
         else:

applications/routes.py

@@ -41,7 +41,7 @@ async def post_data():
     )
     p = ProcessParams(t_id=trace_id)
     data = await request.get_json()
-    processed_data = p.process(data)
+    processed_data = p.deal(data)
     return jsonify(processed_data)
 
 

requirement.txt → requirements.txt

@@ -2,6 +2,9 @@ aiofiles==23.2.1
 aiohttp==3.9.3
 aiosignal==1.3.1
 alembic==1.11.1
+aliyun-log-python-sdk==0.9.1
+aliyun-python-sdk-core==2.15.1
+aliyun-python-sdk-kms==2.16.2
 annotated-types==0.6.0
 anyio==4.3.0
 APScheduler==3.10.1
@@ -16,9 +19,13 @@ cffi==1.15.1
 charset-normalizer==3.3.2
 click==8.1.6
 colorama==0.4.6
+crcmod==1.7
 cryptography==41.0.2
+dateparser==1.2.0
 decorator==5.1.1
 distro==1.9.0
+elastic-transport==8.13.0
+elasticsearch==8.13.0
 exceptiongroup==1.2.0
 Flask==3.0.3
 Flask-APScheduler==1.12.4
@@ -44,24 +51,29 @@ importlib-metadata==6.8.0
 importlib_resources==6.1.2
 itsdangerous==2.1.2
 Jinja2==3.1.2
+jmespath==0.10.0
 Mako==1.2.4
 MarkupSafe==2.1.3
 marshmallow==3.20.1
 marshmallow-sqlalchemy==0.29.0
+mq-http-sdk==1.0.3
 multidict==6.0.5
 mypy-extensions==1.0.0
 numpy==1.24.4
 odps==3.5.1
 openai==1.21.2
+oss2==2.18.4
 packaging==23.1
 pandas==2.0.3
 pathspec==0.12.1
 Pillow==10.0.0
 platformdirs==4.2.0
 priority==2.0.0
+protobuf==3.20.3
 psutil==5.9.5
 pyarrow==15.0.2
 pycparser==2.21
+pycryptodome==3.20.0
 pydantic==2.6.4
 pydantic_core==2.16.3
 PyMySQL==1.1.0
@@ -69,6 +81,7 @@ pyodps==0.11.6
 python-dateutil==2.8.2
 pytz==2023.3
 Quart==0.19.5
+regex==2024.4.16
 requests==2.31.0
 schedule==1.2.1
 six==1.16.0