2024-09-25

发布新版本文章匹配小程序服务

applications/config/__init__.py

@@ -1,8 +1,6 @@
 """
 @author: luojunhui
 """
-import json
-
 import pyapollos
 
 

applications/deal/__init__.py

@@ -1,8 +0,0 @@
-"""
-@author: luojunhui
-接口信息
-"""
-from .record import Record
-from .response import Response
-from .minigram import Minigram
-from .getOffVideos import GetOffVideos

applications/deal/checkVideoStatusBeforePublish.py

@@ -1,4 +0,0 @@
-"""
-@author: luojunhui
-@description:
-"""

applications/deal/finalResponse.py

@@ -1,338 +0,0 @@
-"""
-@author: luojunhui
-"""
-import json
-import time
-import random
-import hashlib
-
-import uuid
-import urllib.parse
-
-from applications.functions.common import async_post
-from applications.functions.log import logging
-from static.config import db_article
-
-
-async def publishToPQ(video_obj):
-    """
-    publish video to pq
-    :return:
-    """
-    oss_path = video_obj['videoPath']
-    uid = video_obj['uid']
-    title = video_obj['title']
-    cover = video_obj['coverPath']
-    url = "https://vlogapi.piaoquantv.com/longvideoapi/crawler/video/send"
-    headers = {
-        "User-Agent": "PQSpeed/486 CFNetwork/1410.1 Darwin/22.6.0",
-        "cookie": "JSESSIONID=4DEA2B5173BB9A9E82DB772C0ACDBC9F; JSESSIONID=D02C334150025222A0B824A98B539B78",
-        "referer": "http://appspeed.piaoquantv.com",
-        "token": "524a8bc871dbb0f4d4717895083172ab37c02d2f",
-        "accept-language": "zh-CN,zh-Hans;q=0.9",
-        "Content-Type": "application/x-www-form-urlencoded",
-    }
-    payload = {
-        "coverImgPath": cover,
-        "deviceToken": "9ef064f2f7869b3fd67d6141f8a899175dddc91240971172f1f2a662ef891408",
-        "fileExtensions": "MP4",
-        "loginUid": uid,
-        "networkType": "Wi-Fi",
-        "platform": "iOS",
-        "requestId": "fb972cbd4f390afcfd3da1869cd7d001",
-        "sessionId": "362290597725ce1fa870d7be4f46dcc2",
-        "subSessionId": "362290597725ce1fa870d7be4f46dcc2",
-        "title": title,
-        "token": "524a8bc871dbb0f4d4717895083172ab37c02d2f",
-        "uid": uid,
-        "versionCode": "486",
-        "versionName": "3.4.12",
-        "videoFromScene": "1",
-        "videoPath": oss_path,
-        "viewStatus": "1",
-    }
-    response = await async_post(url, headers, payload)
-    print(response)
-    return response
-
-
-async def getPQVideoDetail(video_id):
-    """
-    获取票圈视频详情信息
-    :return:
-    """
-    url = "https://longvideoapi.piaoquantv.com/longvideoapi/openapi/video/batchSelectVideoInfo"
-    data = {
-        "videoIdList": [video_id]
-    }
-    header = {
-        "Content-Type": "application/json",
-    }
-    response = await async_post(url, header, json.dumps(data))
-    print(response)
-    return response
-
-
-class FinalResponse(object):
-    """
-    视频发布至pq并且获取video_id
-    """
-
-    def __init__(self, params, mysql_client):
-        """
-        :param params:
-        :param mysql_client:
-        :return:
-        """
-        self.kimi_title = None
-        self.video_obj_list = None
-        self.mini_type = None
-        self.gh_id = None
-        self.trace_id = None
-        self.params = params
-        self.mysql_client = mysql_client
-        self.mini_map = {
-            25: {
-                "avatar": "https://rescdn.yishihui.com/0temp/ttmhzfsh.png",
-                "id": "wx0b7d95eb293b783b",
-                "name": "天天美好祝福生活",
-                "index": 25,
-            },
-            29: {
-                "avatar": "https://rescdn.yishihui.com/0temp/cyfyld.png",
-                "id": "wx65c76bb4c67934db",
-                "name": "财运福运来到",
-                "index": 29,
-            },
-            31: {
-                "avatar": "https://rescdn.yishihui.com/0temp/mhzfshxf2.png",
-                "id": "wx2e4478b1641b3b15",
-                "name": "美好祝福生活幸福",
-                "index": 31,
-            },
-            36: {
-                "avatar": "https://rescdn.yishihui.com/0temp/zfyfyc.jpeg",
-                "id": "wxcddf231abd0dabdc",
-                "name": "祝福有福有财",
-                "index": 36,
-            },
-            27: {
-                "avatar": "https://rescdn.yishihui.com/0temp/xymhfqdd.png",
-                "id": "wx7187c217efef24a7",
-                "name": "幸运美好福气多多",
-                "index": 27,
-            },
-            33: {
-                "avatar": "https://rescdn.yishihui.com/0temp/ssnnfqd.jpeg",
-                "id": "wx5e67713277549b6f",
-                "name": "年年岁岁福气多",
-                "index": 33,
-            },
-        }
-
-    def checkParams(self):
-        """
-        校验参数
-        :return:
-        """
-        try:
-            self.trace_id = self.params['traceId']
-            self.kimi_title = self.params['kimiTitle']
-            self.gh_id = self.params['ghId']
-            self.mini_type = self.params['miniType']
-            self.video_obj_list = self.params['videoObjList']
-            return None
-        except Exception as e:
-            response = {
-                "traceId": self.trace_id,
-                "error": str(e),
-                "info": "参数校验失败"
-            }
-            return response
-
-    def createGzhPath(self, video_id, shared_uid, gh_id):
-        """
-        :param gh_id: 公众号账号的gh_id
-        :param video_id: 视频 id
-        :param shared_uid: 分享 id
-        """
-
-        def generate_source_id():
-            """
-            generate_source_id
-            :return:
-            """
-            timestamp = str(int(time.time() * 1000))
-            random_str = str(random.randint(1000, 9999))
-            hash_input = f"{timestamp}-{random_str}"
-            return hashlib.md5(hash_input.encode()).hexdigest()
-
-        root_share_id = str(uuid.uuid4())
-        if self.mini_type == 2:
-            source_id = (
-                    "touliu_tencentGzhArticle_{}_".format(gh_id) + generate_source_id()
-            )
-        elif self.mini_type == 1:
-            source_id = "longArticles_" + generate_source_id()
-        elif self.mini_type == 3:
-            source_id = "WeCom_" + generate_source_id()
-        else:
-            source_id = "Error mini_program_type {}".format(self.mini_type)
-        url = f"pages/user-videos?id={video_id}&su={shared_uid}&fromGzh=1&rootShareId={root_share_id}&shareId={root_share_id}&rootSourceId={source_id}"
-        # 自动把 root_share_id 加入到白名单
-        # auto_white(root_share_id)
-        return (
-            root_share_id,
-            source_id,
-            f"pages/category?jumpPage={urllib.parse.quote(url, safe='')}",
-        )
-
-    def chooseMiniProgram(self, mini_id):
-        """
-        获取小程序分享卡片
-        :return:
-        """
-        mini_info = self.mini_map[mini_id]
-        return (
-            mini_info["avatar"],
-            mini_info["id"],
-            mini_info["name"],
-            mini_info["index"],
-        )
-
-    async def generateSingleCard(self, video_id, title, index, gh_id, mini_id):
-        """
-        生成单个分享卡片
-        :return:
-        """
-        response = await getPQVideoDetail(video_id)
-        print(response)
-        productionCover = response["data"][0]["shareImgPath"]
-        videoUrl = response["data"][0]["videoPath"]
-        user_id = response["data"][0]["user"]["uid"]
-        programAvatar, programId, programName, pqMiniId = self.chooseMiniProgram(mini_id)
-        root_share_id, source_id, productionPath = self.createGzhPath(video_id, user_id, gh_id)
-        logging(
-            code="1002",
-            info="root_share_id --{}, productionPath -- {}".format(
-                root_share_id, productionPath
-            ),
-            function="process",
-            trace_id=self.trace_id,
-        )
-        result = {
-            "productionCover": productionCover,
-            "productionName": title,
-            "programAvatar": programAvatar,
-            "programId": programId,
-            "programName": programName,
-            "source": "web",
-            "rootShareId": root_share_id,
-            "productionPath": productionPath,
-            "videoUrl": videoUrl,
-            "mini_id": mini_id,
-            "paragraphPosition": index * 0.25,
-        }
-        update_result_sql = f"""
-                            UPDATE {db_article}
-                            SET
-                                result{index} = %s,
-                                success = %s
-                            WHERE
-                                trace_id = %s;
-                        """
-        await self.mysql_client.async_insert(
-            sql=update_result_sql,
-            params=(json.dumps(result, ensure_ascii=False), 1, self.trace_id),
-        )
-        logging(
-            code="2000",
-            info="统计 root_share_id && video_id",
-            function="process",
-            trace_id=self.trace_id,
-            data={
-                "rootShareId": root_share_id,
-                "videoId": video_id,
-                "sourceId": source_id,
-            },
-        )
-        return result
-
-    async def generateCards(self, video_list, kimi_title, gh_id):
-        """
-        生成一组卡片
-        :param video_list:
-        :param kimi_title:
-        :param gh_id:
-        :return:
-        """
-        L = []
-        if self.mini_type == 1:
-            mini_id_list = [25, 29, 31]
-            video_count = len(video_list)
-            mini_choice_index = random.sample(range(3), video_count)
-            mini_choice_item = [mini_id_list[i] for i in mini_choice_index]
-            for index, video_id in enumerate(video_list, 1):
-                result = await self.generateSingleCard(
-                    video_id=video_id,
-                    title=kimi_title,
-                    index=index,
-                    gh_id=gh_id,
-                    mini_id=mini_choice_item[index - 1],
-                )
-                L.append(result)
-        elif self.mini_type == 2:
-            for index, video_id in enumerate(video_list, 1):
-                result = await self.generateSingleCard(
-                    video_id=video_id,
-                    title=kimi_title,
-                    index=index,
-                    gh_id=gh_id,
-                    mini_id=33,
-                )
-                L.append(result)
-        elif self.mini_type == 3:
-            for index, video_id in enumerate(video_list, 1):
-                result = await self.generateSingleCard(
-                    video_id=video_id,
-                    title=kimi_title,
-                    index=index,
-                    gh_id=gh_id,
-                    mini_id=27,
-                )
-                L.append(result)
-        return L
-
-    async def getVideoIdList(self):
-        """
-        获取匹配文章的id_list
-        :return:
-        """
-        vid_list = []
-        for video_obj in self.video_obj_list:
-            video_publish_info = await publishToPQ(video_obj=video_obj)
-            video_id = video_publish_info['data']['id']
-            vid_list.append(video_id)
-        return vid_list
-
-    async def deal(self):
-        """
-        deal function
-        :return:
-        """
-        params_error = self.checkParams()
-        if params_error:
-            return params_error
-        else:
-            video_list = await self.getVideoIdList()
-            card_list = await self.generateCards(
-                video_list, self.kimi_title, self.gh_id
-            )
-            response = {
-                "trace_id": self.trace_id,
-                "miniprogramList": card_list
-            }
-            return response
-
-
-

applications/deal/getOffVideos.py

@@ -1,73 +0,0 @@
-"""
-@author: luojunhui
-"""
-import time
-
-
-class GetOffVideos(object):
-    """
-    下架视频
-    """
-
-    def __init__(self, params, mysql_client):
-        self.params = params
-        self.mysql_client = mysql_client
-        self.trace_id = None
-
-    def checkParams(self):
-        """
-
-        :return:
-        """
-        try:
-            self.trace_id = self.params['traceId']
-            return None
-        except Exception as e:
-            response = {
-                "error": "params error",
-                "info": str(e),
-                "data": self.params
-            }
-            return response
-
-    async def pushVideoIntoQueue(self):
-        """
-        将视频id记录到待下架表中
-        :return:
-        """
-        select_sql = f"""
-        select recall_video_id1, recall_video_id2, recall_video_id3 from {db_article}
-        where trace_id = '{self.trace_id}';
-        """
-        recall_video_info = await self.mysql_client.async_select(sql=select_sql)
-        recall_vid_tuple = recall_video_info[0]
-        for vid in recall_vid_tuple:
-            try:
-                update_sql = f"""
-                INSERT INTO get_off_videos
-                (video_id, publish_time, video_status, trace_id)
-                values 
-                (%s, %s, %s, %s);
-                """
-                await self.mysql_client.async_insert(
-                    sql=update_sql,
-                    params=(vid, int(time.time()), 1, self.trace_id)
-                )
-            except Exception as e:
-                print(e)
-
-    async def deal(self):
-        """
-
-        :return:
-        """
-        params_error = self.checkParams()
-        if params_error:
-            return params_error
-        else:
-            await self.pushVideoIntoQueue()
-            response = {
-                "status": "success",
-                "traceId": self.trace_id
-            }
-            return response

applications/deal/minigram.py

@@ -1,110 +0,0 @@
-"""
-@author: luojunhui
-"""
-minigram_map = {
-    1: {
-        # 25: {
-        #     "avatar": "https://rescdn.yishihui.com/0temp/ttmhzfsh.png",
-        #     "id": "wx0b7d95eb293b783b",
-        #     "name": "天天美好祝福生活",
-        #     "index": 25
-        # },
-        25: {
-                "avatar": "https://rescdn.yishihui.com/0temp/pqsp.png",
-                "id": "wxbdd2a2e93d9a6e25",
-                "name": "票圈视频",
-                "index": 25
-            },
-        29: {
-            "avatar": "https://rescdn.yishihui.com/0temp/cyfyld.png",
-            "id": "wx65c76bb4c67934db",
-            "name": "财运福运来到",
-            "index": 29
-        },
-        31: {
-            "avatar": "https://rescdn.yishihui.com/0temp/mhzfshxf2.png",
-            "id": "wx2e4478b1641b3b15",
-            "name": "美好祝福生活幸福",
-            "index": 31
-        }
-    },
-    2: {
-        36: {
-            "avatar": "https://rescdn.yishihui.com/0temp/zfyfyc.jpeg",
-            "id": "wxcddf231abd0dabdc",
-            "name": "祝福有福有财",
-            "index": 36
-        },
-        33: {
-            "avatar": "https://rescdn.yishihui.com/0temp/pqsp.png",
-            "id": "wxbdd2a2e93d9a6e25",
-            "name": "票圈视频",
-            "index": 33
-            }
-    },
-    3: {
-        27: {
-            "avatar": "https://rescdn.yishihui.com/0temp/xymhfqdd.png",
-            "id": "wx7187c217efef24a7",
-            "name": "幸运美好福气多多",
-            "index": 27
-        }
-    }
-}
-
-
-class Minigram(object):
-    """
-    小程序卡片
-    """
-
-    def __init__(self, params):
-        self.params = params
-        self.business_type = None
-        self.mini_code = None
-        self.trace_id = None
-
-    def check_params(self):
-        """
-        校验参数
-        :return:
-        """
-        try:
-            self.business_type = self.params['businessType']
-            self.mini_code = self.params['miniCode']
-            self.trace_id = self.params['traceId']
-            return None
-        except Exception as e:
-            response = {
-                "status": "fail",
-                "code": 1,
-                "message": str(e),
-                "info": "params check error"
-            }
-            return response
-
-    def choose_minigram(self):
-        """
-        分配小程序卡片
-        :return:
-        """
-        try:
-            minigram = minigram_map.get(self.business_type).get(self.mini_code)
-            response = {
-                "programAvatar": minigram['avatar'],
-                "programId": minigram['id'],
-                "programName": minigram['name'],
-                "trace_id": self.trace_id
-            }
-        except Exception as e:
-            response = {
-                "error": "invalid params",
-                "msg": str(e)
-            }
-        return response
-
-    async def deal(self):
-        """
-        :return:
-        """
-        return self.check_params() if self.check_params() else self.choose_minigram()

applications/deal/record.py

@@ -1,99 +0,0 @@
-"""
-@author: luojunhui
-"""
-import time
-
-from uuid import uuid4
-
-from applications.functions.log import logging
-from static.config import db_article
-
-
-class Record(object):
-    """
-    搜索接口处理逻辑
-    """
-
-    def __init__(self, params, mysql_client):
-        self.content_id = None
-        self.account_name = None
-        self.contents = None
-        self.title = None
-        self.gh_id = None
-        self.params = params
-        self.mysql_client = mysql_client
-        self.trace_id = "search-{}-{}".format(str(uuid4()), str(int(time.time())))
-
-    def check_params(self):
-        """
-        检查请求params
-        :return:
-        """
-        try:
-            self.gh_id = self.params['ghId']
-            self.title = self.params['title'].split("@@")[-1].replace("'", "")
-            self.contents = self.params['content'].replace("'", "")
-            self.account_name = self.params['accountName'].replace("'", "")
-            self.content_id = self.params['articleId']
-            logging(
-                code="1001",
-                info="搜索视频内容接口请求成功, 参数校验成功",
-                port="title_to_search",
-                trace_id=self.trace_id,
-                data=self.params
-            )
-            return None
-        except Exception as e:
-            result = {
-                "status": "fail",
-                "code": 1,
-                "message": str(e),
-                "info": "params check error"
-            }
-            logging(
-                code="4001",
-                info="搜索视频内容接口请求成功, 参数校验失败",
-                port="title_to_search",
-                trace_id=self.trace_id,
-                data=self.params
-            )
-            return result
-
-    async def input_queue(self):
-        """
-        把数据插入待处理队列
-        :return:
-        """
-        request_time = int(time.time())
-        insert_sql = f"""
-                        INSERT INTO {db_article}
-                            (trace_id, content_id, gh_id, account_name, article_title, article_text, content_status, success, request_time_stamp)
-                        VALUES 
-                            (%s, %s, %s, %s, %s, %s, %s, %s, %s);"""
-        await self.mysql_client.async_insert(
-            sql=insert_sql,
-            params=(self.trace_id, self.content_id, self.gh_id, self.account_name, self.title, self.contents, 0, 0, request_time)
-        )
-        logging(
-            code="1002",
-            info="成功记录请求数据到mysql中",
-            trace_id=self.trace_id
-        )
-
-    async def deal(self):
-        """
-        deal
-        :return:
-        """
-        params_error = self.check_params()
-        if params_error:
-            return params_error
-        else:
-            # 记录
-            await self.input_queue()
-            res = {
-                "status": "success input to article queue",
-                "code": 0,
-                "traceId": self.trace_id
-            }
-            return res

applications/deal/response.py

@@ -1,338 +0,0 @@
-"""
-@author: luojunhui
-"""
-
-import json
-import uuid
-import time
-import random
-import hashlib
-import urllib.parse
-
-from applications.functions.log import logging
-from static.config import db_article
-from applications.functions.common import request_for_info
-
-
-class Response(object):
-    """
-    召回逻辑
-    """
-
-    def __init__(self, trace_id, mysql_client, mini_program_type):
-        """
-        长文： 25， 29， 31
-        投流： 33
-        企微： 27
-        :param trace_id:
-        :param mysql_client:
-        :param mini_program_type:
-        """
-        self.trace_id = trace_id
-        self.mysql_client = mysql_client
-        self.mini_program_type = mini_program_type
-        self.mini_map = {
-            # 25: {
-            #     "avatar": "https://rescdn.yishihui.com/0temp/ttmhzfsh.png",
-            #     "id": "wx0b7d95eb293b783b",
-            #     "name": "天天美好祝福生活",
-            #     "index": 25,
-            # },
-            25: {
-                "avatar": "https://rescdn.yishihui.com/0temp/pqsp.png",
-                "id": "wxbdd2a2e93d9a6e25",
-                "name": "票圈视频",
-                "index": 25
-            },
-            29: {
-                "avatar": "https://rescdn.yishihui.com/0temp/cyfyld.png",
-                "id": "wx65c76bb4c67934db",
-                "name": "财运福运来到",
-                "index": 29,
-            },
-            31: {
-                "avatar": "https://rescdn.yishihui.com/0temp/mhzfshxf2.png",
-                "id": "wx2e4478b1641b3b15",
-                "name": "美好祝福生活幸福",
-                "index": 31,
-            },
-            36: {
-                "avatar": "https://rescdn.yishihui.com/0temp/zfyfyc.jpeg",
-                "id": "wxcddf231abd0dabdc",
-                "name": "祝福有福有财",
-                "index": 36,
-            },
-            27: {
-                "avatar": "https://rescdn.yishihui.com/0temp/xymhfqdd.png",
-                "id": "wx7187c217efef24a7",
-                "name": "幸运美好福气多多",
-                "index": 27,
-            },
-            # 33: {
-            #     "avatar": "https://rescdn.yishihui.com/0temp/ssnnfqd.jpeg",
-            #     "id": "wx5e67713277549b6f",
-            #     "name": "年年岁岁福气多",
-            #     "index": 33,
-            # },
-            33: {
-                "avatar": "https://rescdn.yishihui.com/0temp/pqsp.png",
-                "id": "wxbdd2a2e93d9a6e25",
-                "name": "票圈视频",
-                "index": 33
-            }
-        }
-        # self.test_id_set = {'gh_080bb43aa0dc', 'gh_e0eb490115f5', 'gh_6d9f36e3a7be', 'gh_29074b51f2b7', 'gh_6d205db62f04',
-        #  'gh_d49df5e974ca', 'gh_d4dffc34ac39', 'gh_89ef4798d3ea', 'gh_b15de7c99912', 'gh_9eef14ad6c16',
-        #  'gh_0c89e11f8bf3', 'gh_f902cea89e48', 'gh_6b7c2a257263', 'gh_183d80deffb8', 'gh_9f8dc5b0c74e',
-        #  'gh_1b27dd1beeca', 'gh_e24da99dc899', 'gh_dd4c857bbb36', 'gh_7e5818b2dd83'}
-
-    async def get_result(self):
-        """
-        获取结果
-        :return:
-        """
-        select_sql = f"""
-                SELECT gh_id, recall_video_id1, recall_video_id2, recall_video_id3, kimi_title, content_status, process_times
-                FROM {db_article}
-                WHERE trace_id = '{self.trace_id}';
-        """
-        info_tuple = await self.mysql_client.async_select(select_sql)
-        gh_id, vid1, vid2, vid3, kimi_title, content_status, process_times = info_tuple[
-            0
-        ]
-        response = {
-            "gh_id": gh_id,
-            "vid_list": [vid1, vid2, vid3],
-            "kimi_title": kimi_title,
-            "content_status": content_status,
-            "process_times": process_times,
-        }
-        print(response)
-        return response
-
-    def create_gzh_path(self, video_id, shared_uid, gh_id):
-        """
-        :param gh_id: 公众号账号的gh_id
-        :param video_id: 视频 id
-        :param shared_uid: 分享 id
-        """
-
-        def generate_source_id():
-            """
-            generate_source_id
-            :return:
-            """
-            timestamp = str(int(time.time() * 1000))
-            random_str = str(random.randint(1000, 9999))
-            hash_input = f"{timestamp}-{random_str}"
-            return hashlib.md5(hash_input.encode()).hexdigest()
-
-        root_share_id = str(uuid.uuid4())
-        if self.mini_program_type == 2:
-            source_id = (
-                "touliu_tencentGzhArticle_{}_".format(gh_id) + generate_source_id()
-            )
-        elif self.mini_program_type == 1:
-            source_id = "longArticles_" + generate_source_id()
-        elif self.mini_program_type == 3:
-            source_id = "WeCom_" + generate_source_id()
-        else:
-            source_id = "Error mini_program_type {}".format(self.mini_program_type)
-        url = f"pages/user-videos?id={video_id}&su={shared_uid}&fromGzh=1&rootShareId={root_share_id}&shareId={root_share_id}&rootSourceId={source_id}"
-        # 自动把 root_share_id 加入到白名单
-        # auto_white(root_share_id)
-        return (
-            root_share_id,
-            source_id,
-            f"pages/category?jumpPage={urllib.parse.quote(url, safe='')}",
-        )
-
-    def choose_mini_program(self, mini_id):
-        """
-        获取小程序分享卡片
-        :return:
-        """
-        mini_info = self.mini_map[mini_id]
-        return (
-            mini_info["avatar"],
-            mini_info["id"],
-            mini_info["name"],
-            mini_info["index"],
-        )
-
-    async def generate_card(self, video_id, kimi_title, index, gh_id, mini_id):
-        """
-        生成分享卡片
-        :return:
-        """
-        response = request_for_info(video_id)
-        productionCover = response["data"][0]["shareImgPath"]
-        productionName = kimi_title
-        videoUrl = response["data"][0]["videoPath"]
-        user_id = response["data"][0]["user"]["uid"]
-        programAvatar, programId, programName, pqMiniId = self.choose_mini_program(
-            mini_id
-        )
-        root_share_id, source_id, productionPath = self.create_gzh_path(
-            video_id, user_id, gh_id
-        )
-        source = "Web"
-        logging(
-            code="1002",
-            info="root_share_id --{}, productionPath -- {}".format(
-                root_share_id, productionPath
-            ),
-            function="process",
-            trace_id=self.trace_id,
-        )
-        result = {
-            "productionCover": productionCover,
-            "productionName": productionName,
-            "programAvatar": programAvatar,
-            "programId": programId,
-            "programName": programName,
-            "source": source,
-            "rootShareId": root_share_id,
-            "productionPath": productionPath,
-            "videoUrl": videoUrl,
-            "mini_id": mini_id,
-            "paragraphPosition": index * 0.25,
-        }
-        # if gh_id in self.test_id_set and index == 1:
-        # 只针对插入的第一个小程序位置做改变
-        if index == 1:
-            result['paragraphPosition'] = 0.01
-
-        update_result_sql = f"""
-                            UPDATE {db_article}
-                            SET
-                                result{index} = %s,
-                                success = %s
-                            WHERE
-                                trace_id = %s;
-                        """
-        await self.mysql_client.async_insert(
-            sql=update_result_sql,
-            params=(json.dumps(result, ensure_ascii=False), 1, self.trace_id),
-        )
-        logging(
-            code="2000",
-            info="统计 root_share_id && video_id",
-            function="process",
-            trace_id=self.trace_id,
-            data={
-                "rootShareId": root_share_id,
-                "videoId": video_id,
-                "sourceId": source_id,
-            },
-        )
-        return result
-
-    async def generate_cards(self, video_list, kimi_title, gh_id):
-        """
-        生成一组卡片
-        :param video_list:
-        :param kimi_title:
-        :param gh_id:
-        :return:
-        """
-        L = []
-        if self.mini_program_type == 1:
-            # mini_id_list = [25, 29, 31]
-            # video_count = len(video_list)
-            # mini_choice_index = random.sample(range(3), video_count)
-            # mini_choice_item = [mini_id_list[i] for i in mini_choice_index]
-            for index, video_id in enumerate(video_list, 1):
-                random_num = random.randint(1, 10)
-                if random_num in [1, 2, 3, 4, 5, 6]:
-                    mini_id = 25
-                elif random_num in [7, 8]:
-                    mini_id = 29
-                else:
-                    mini_id = 31
-                result = await self.generate_card(
-                    video_id=video_id,
-                    kimi_title=kimi_title,
-                    index=index,
-                    gh_id=gh_id,
-                    mini_id=mini_id,
-                )
-                L.append(result)
-        elif self.mini_program_type == 2:
-            for index, video_id in enumerate(video_list, 1):
-                result = await self.generate_card(
-                    video_id=video_id,
-                    kimi_title=kimi_title,
-                    index=index,
-                    gh_id=gh_id,
-                    mini_id=33,
-                )
-                L.append(result)
-        elif self.mini_program_type == 3:
-            for index, video_id in enumerate(video_list, 1):
-                result = await self.generate_card(
-                    video_id=video_id,
-                    kimi_title=kimi_title,
-                    index=index,
-                    gh_id=gh_id,
-                    mini_id=27,
-                )
-                L.append(result)
-        return L
-
-    async def deal(self):
-        """
-        Recall Deal
-        :return:
-        """
-        response = await self.get_result()
-        status_code = response.get("content_status")
-        process_times = response.get("process_times")
-        if status_code == 0:
-            if process_times > 3:
-                result = {
-                    "traceId": self.trace_id,
-                    "code": 0,
-                    "error": "匹配失败，处理超过五次，文章敏感",
-                }
-            else:
-                result = {
-                    "traceId": self.trace_id,
-                    "code": 0,
-                    "Message": "该请求还没处理",
-                }
-        elif status_code == 1:
-            result = {
-                "traceId": self.trace_id,
-                "code": 1,
-                "Message": "该请求正在处理中",
-            }
-        elif status_code == 2:
-            unEmptyList = [i for i in response["vid_list"] if i]
-            cards = await self.generate_cards(
-                unEmptyList, response.get("kimi_title"), response["gh_id"]
-            )
-            if cards:
-                result = {"traceId": self.trace_id, "miniprogramList": cards}
-            else:
-                result = {
-                    "traceId": self.trace_id,
-                    "code": 0,
-                    "error": "匹配失败，视频下载失败返回vid为0",
-                }
-        elif status_code == 3:
-            result = {
-                "traceId": self.trace_id,
-                "code": 0,
-                "error": "匹配失败，处理超过五次，文章敏感",
-            }
-        else:
-            result = {"traceId": self.trace_id, "Message": "UnKnow Error"}
-        logging(
-            code="1002",
-            info="返回结果",
-            function="process",
-            data=result,
-            trace_id=self.trace_id,
-        )
-        return result

applications/functions/common.py

@@ -2,128 +2,8 @@
 """
 @author: luojunhui
 """
-import time
-import json
-import uuid
-import random
-import hashlib
-import requests
 import aiohttp
 import asyncio
-import urllib.parse
-
-
-def create_gzh_path(video_id, shared_uid):
-    """
-    :param video_id: 视频 id
-    :param shared_uid: 分享 id
-    """
-
-    def auto_white(root_share_id_):
-        """
-        自动加入白名单， 保证公众号百分百出广告
-        :param root_share_id_:
-        :return:
-        """
-
-        def get_cookie():
-            """
-            获取 cookie
-            :return:
-            """
-            url = "https://admin.piaoquantv.com/manager/login?account=luojunhui&passWd=e10adc3949ba59abbe56e057f20f883e&muid=7"
-            payload = {}
-            headers = {
-                'accept': 'application/json, text/plain, */*',
-                'accept-language': 'en',
-                'priority': 'u=1, i',
-                'sec-ch-ua': '"Chromium";v="124", "Google Chrome";v="124", "Not-A.Brand";v="99"',
-                'sec-ch-ua-mobile': '?0',
-                'sec-ch-ua-platform': '"macOS"',
-                'sec-fetch-dest': 'empty',
-                'sec-fetch-mode': 'cors',
-                'sec-fetch-site': 'same-origin',
-                'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36'
-            }
-            response = requests.request("GET", url, headers=headers, data=payload)
-            return response.cookies.values()[0]
-
-        url = "https://admin.piaoquantv.com/manager/ad/own/white/rootShare/save"
-        dd = {
-            "rootShareId": root_share_id_,
-            "commit": "算法自动加入白名单--"
-        }
-        payload = json.dumps(dd)
-        cookie = get_cookie()
-        headers = {
-            'accept': 'application/json',
-            'accept-language': 'en',
-            'content-type': 'application/json;',
-            'cookie': "SESSION=" + cookie,
-            'origin': 'https://admin.piaoquantv.com',
-            'priority': 'u=1, i',
-            'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36'
-        }
-        response = requests.request("POST", url, headers=headers, data=payload)
-        return response.json()['content']
-
-    def generate_source_id():
-        """
-        generate_source_id
-        :return:
-        """
-        timestamp = str(int(time.time() * 1000))
-        random_str = str(random.randint(1000, 9999))
-        hash_input = f"{timestamp}-{random_str}"
-        return hashlib.md5(hash_input.encode()).hexdigest()
-
-    root_share_id = str(uuid.uuid4())
-    source_id = "longArticles_" + generate_source_id()
-    url = f"pages/user-videos?id={video_id}&su={shared_uid}&fromGzh=1&rootShareId={root_share_id}&shareId={root_share_id}&rootSourceId={source_id}"
-    # 自动把 root_share_id 加入到白名单
-    # auto_white(root_share_id)
-    return root_share_id, source_id, f"pages/category?jumpPage={urllib.parse.quote(url, safe='')}"
-
-
-def create_gzh_path_v2(video_id, shared_uid):
-    """
-    :param video_id: 视频 id
-    :param shared_uid: 分享 id
-    """
-
-    def generate_source_id():
-        """
-        generate_source_id
-        :return:
-        """
-        timestamp = str(int(time.time() * 1000))
-        random_str = str(random.randint(1000, 9999))
-        hash_input = f"{timestamp}-{random_str}"
-        return hashlib.md5(hash_input.encode()).hexdigest()
-
-    root_share_id = str(uuid.uuid4())
-    source_id = "touliu_tencentGzhArticle_" + generate_source_id()
-    url = f"pages/user-videos?id={video_id}&su={shared_uid}&fromGzh=1&rootShareId={root_share_id}&shareId={root_share_id}&rootSourceId={source_id}"
-    # 自动把 root_share_id 加入到白名单
-    # auto_white(root_share_id)
-    return root_share_id, source_id, f"pages/category?jumpPage={urllib.parse.quote(url, safe='')}"
-
-
-def request_for_info(video_id):
-    """
-    请求数据
-    :param video_id:
-    :return:
-    """
-    url = "https://longvideoapi.piaoquantv.com/longvideoapi/openapi/video/batchSelectVideoInfo"
-    data = {
-        "videoIdList": [video_id]
-    }
-    header = {
-        "Content-Type": "application/json",
-    }
-    response = requests.post(url, headers=header, data=json.dumps(data))
-    return response.json()
 
 
 def clean_title(strings):
@@ -166,53 +46,6 @@ def sensitive_flag(s_words, ori_title):
     return True
 
 
-def account_info_map(gh_id):
-    """
-    根据账号 id 来判断返回哪个小程序账号
-    :param gh_id:
-    :return:
-    """
-    pqlh = {
-        "name": "票圈乐活",
-        "id": "wxe8f8f0e23cecad0f",
-        "avatar": "https://rescdn.yishihui.com/0temp/lehuo.png"
-    }
-    xyfxhyjl = {
-        "name": "幸运福星好运锦鲤",
-        "id": "wx95dcbfc0753c06a8",
-        "avatar": "https://rescdn.yishihui.com/0temp/xyfxhyjl.png"
-    }
-    pqzf = {
-        "name": "票圈祝福",
-        "id": "wxf7261ed54f2e450e",
-        "avatar": "https://rescdn.yishihui.com/0temp/pqzf.png"
-    }
-    buy_accounts = [
-        "gh_084a485e859a",
-        "gh_e24da99dc899",
-        "gh_e0eb490115f5",
-        "gh_183d80deffb8",
-        "gh_5ff48e9fb9ef",
-        "gh_9f8dc5b0c74e",
-        "gh_6d9f36e3a7be"
-    ]
-    dyy = [
-        "gh_9877c8541764",
-        "gh_6d205db62f04",
-        "gh_c69776baf2cd",
-        "gh_7e5818b2dd83",
-        "gh_89ef4798d3ea",
-        "gh_a2901d34f75b",
-        "gh_b15de7c99912"
-    ]
-    if gh_id in buy_accounts:
-        return ""
-    elif gh_id in dyy:
-        return ""
-    else:
-        return ""
-
-
 async def request_etl(url, headers, json_data, retries=6):
     """
     :param url:

applications/functions/kimi.py

@@ -6,7 +6,7 @@ import json
 from openai import OpenAI
 
 from applications.functions.chatgpt import OpenAIServer
-from applications.functions.log import logging
+from applications.log import logging
 
 
 class KimiServer(object):

applications/functions/log.py → applications/log/__init__.py

@@ -1,6 +1,9 @@
 """
 @author: luojunhui
 """
+"""
+@author: luojunhui
+"""
 import time
 import json
 from aliyun.log import LogClient, PutLogsRequest, LogItem

applications/match_algorithm/rank.py

@@ -2,7 +2,6 @@
 @author: luojunhui
 """
 from applications.match_algorithm.title_similarity import jcd_title_similarity
-from applications.functions.log import logging
 
 
 def jac_score(d1, d2):

applications/match_algorithm/recall.py

@@ -1,42 +0,0 @@
-"""
-@author: luojunhui
-"""
-
-from applications.functions.log import logging
-
-
-async def recall_videos(trace_id, s_videos):
-    """
-    通过请求的数据来召回视频
-    :param s_videos:
-    :param trace_id:
-    :return: file_list
-    """
-    # title = params['title']
-    # content = params['content']
-    # ghId = params['ghId']
-    # user_id = gh_id_dict[ghId]['uid']
-
-    # 在两边召回视频
-    # pq_videos
-    # recall_video_list = MySQLServer().select_pq_videos()
-    recall_video_list = []
-    # dirs_1 = os.path.join(os.getcwd(), 'applications', 'static', 'out_videos')
-    # file_list = [os.path.join(dirs_1, "{}.json".format(vid)) for vid in s_videos]
-    # search_list = []
-    # for file in file_list:
-    #     with open(file, encoding="utf-8") as f:
-    #         obj = json.loads(f.read())
-    #         if obj:
-    #             obj['video_id'] = file.split("/")[-1].replace('.json', '')
-    #     search_list.append(obj)
-    logging(
-        code="1002",
-        info="召回视频",
-        data={
-            "pq_list": [],
-            "search_list": s_videos
-        },
-        trace_id=trace_id
-    )
-    return recall_video_list, s_videos

applications/schedule/__init__.py

@@ -1,6 +0,0 @@
-"""
-@author: luojunhui
-"""
-from .process_schedule import recall_videos
-from .search_schedule import search_videos
-# from .process_schedule import return_info_v2

applications/schedule/process_schedule.py

@@ -1,232 +0,0 @@
-"""
-@author: luojunhui
-对请求进行操作
-"""
-import json
-
-from applications.functions.common import request_for_info, create_gzh_path, create_gzh_path_v2
-from applications.functions.log import logging
-from static.config import db_article
-
-
-async def return_info_v2(video_id, trace_id, mysql_client):
-    """
-    :param mysql_client:
-    :param trace_id:
-    :param video_id:
-    :return:
-    """
-    response = request_for_info(video_id)
-    productionCover = response['data'][0]['shareImgPath']
-    productionName = response['data'][0]['title']
-    videoUrl = response['data'][0]['videoPath']
-    user_id = response['data'][0]['user']['uid']
-    programAvatar = "https://rescdn.yishihui.com/0temp/ssyqsh.png"
-    programId = "wx59d9e2c05f00f880"
-    programName = "刷刷有趣生活"
-    source = "PQ"
-    root_share_id, source_id, productionPath = create_gzh_path(video_id=video_id, shared_uid=user_id)
-    logging(
-        code="1002",
-        info="root_share_id --{}, productionPath -- {}".format(root_share_id, productionPath),
-        function="process",
-        trace_id=trace_id
-    )
-    result = {
-        "productionCover": productionCover,
-        "productionName": productionName,
-        "programAvatar": programAvatar,
-        "programId": programId,
-        "programName": programName,
-        "source": source,
-        "rootShareId": root_share_id,
-        "productionPath": productionPath,
-        "videoUrl": videoUrl,
-        "paragraphPosition": 0.25
-    }
-    update_result_sql = f"""
-                        UPDATE {db_article}
-                        SET
-                            result1 = %s,
-                            success = %s
-                        WHERE
-                            trace_id = %s;
-                    """
-    await mysql_client.async_insert(
-        sql=update_result_sql,
-        params=(json.dumps(result, ensure_ascii=False), 1, trace_id)
-    )
-    logging(
-        code="2000",
-        info="root_share_id和source_id",
-        function="process",
-        trace_id=trace_id,
-        data={
-            "rootShareId": root_share_id,
-            "videoId": video_id,
-            "sourceId": source_id
-        }
-    )
-    return result
-
-
-async def return_info(video_id, kimi_title, trace_id, mysql_client, index, gh_id):
-    """
-
-    :param gh_id: 公众号账号id
-    :param index:
-    :param mysql_client:
-    :param trace_id:
-    :param kimi_title:
-    :param video_id:
-    :return:
-    """
-    response = request_for_info(video_id)
-    productionCover = response['data'][0]['shareImgPath']
-    productionName = kimi_title
-    videoUrl = response['data'][0]['videoPath']
-    user_id = response['data'][0]['user']['uid']
-    if gh_id in ['gh_93e00e187787', 'gh_ac43e43b253b']:
-        # 投流的两个账号
-        programAvatar = "https://rescdn.yishihui.com/0temp/zfyfyc.jpeg"
-        programId = "wxcddf231abd0dabdc"
-        programName = "祝福有福有财"
-        root_share_id, source_id, productionPath = create_gzh_path_v2(video_id=video_id, shared_uid=user_id)
-    else:
-        programAvatar = "https://rescdn.yishihui.com/0temp/ssyqsh.png"
-        programId = "wx59d9e2c05f00f880"
-        programName = "刷刷有趣生活"
-        root_share_id, source_id, productionPath = create_gzh_path(video_id=video_id, shared_uid=user_id)
-    source = "Web"
-    logging(
-        code="1002",
-        info="root_share_id --{}, productionPath -- {}".format(root_share_id, productionPath),
-        function="process",
-        trace_id=trace_id
-    )
-    result = {
-        "productionCover": productionCover,
-        "productionName": productionName,
-        "programAvatar": programAvatar,
-        "programId": programId,
-        "programName": programName,
-        "source": source,
-        "rootShareId": root_share_id,
-        "productionPath": productionPath,
-        "videoUrl": videoUrl,
-        "paragraphPosition": index * 0.25
-    }
-    update_result_sql = f"""
-                    UPDATE {db_article}
-                    SET
-                        result{index} = %s,
-                        success = %s
-                    WHERE
-                        trace_id = %s;
-                """
-    await mysql_client.async_insert(
-        sql=update_result_sql,
-        params=(json.dumps(result, ensure_ascii=False), 1, trace_id)
-    )
-    logging(
-        code="2000",
-        info="统计 root_share_id && video_id",
-        function="process",
-        trace_id=trace_id,
-        data={
-            "rootShareId": root_share_id,
-            "videoId": video_id,
-            "sourceId": source_id
-        }
-    )
-    return result
-
-
-async def recall_videos(trace_id, mysql_client):
-    """
-    从 mysql 读取数据
-    :param trace_id:  唯一 id
-    :param mysql_client: mysql 服务、
-    :return:
-    """
-    select_sql = f"""
-        SELECT gh_id, recall_video_id1, recall_video_id2, recall_video_id3, kimi_title, content_status, process_times
-        FROM {db_article}
-        WHERE trace_id = '{trace_id}';
-    """
-    info_tuple = await mysql_client.async_select(select_sql)
-    gh_id, vid1, vid2, vid3, kimi_title, content_status, process_times = info_tuple[0]
-    vid_list = [vid1, vid2, vid3]
-    unEmptyList = [i for i in vid_list if i]
-    L = []
-    if content_status == 0:
-        if process_times > 5:
-            result = {
-                "traceId": trace_id,
-                "code": 0,
-                "error": "匹配失败，处理超过五次，文章敏感"
-            }
-        else:
-            result = {
-                "traceId": trace_id,
-                "code": 0,
-                "Message": "该请求还没处理"
-            }
-    elif content_status == 1:
-        result = {
-            "traceId": trace_id,
-            "code": 1,
-            "Message": "该请求正在处理中"
-        }
-    elif content_status == 2:
-        logging(
-            code="1002",
-            info="vid_list: {}".format(json.dumps(unEmptyList, ensure_ascii=False)),
-            function="process",
-            trace_id=trace_id
-        )
-        for index, best_video_id in enumerate(unEmptyList, 1):
-            # if kimi_title:
-            temp = await return_info(
-                video_id=best_video_id,
-                kimi_title=kimi_title,
-                trace_id=trace_id,
-                mysql_client=mysql_client,
-                index=index,
-                gh_id=gh_id
-            )
-            L.append(temp)
-            # else:
-            #     temp = await return_info_v2(
-            #         video_id=best_video_id,
-            #         trace_id=trace_id,
-            #         mysql_client=mysql_client
-            #     )
-            #     L.append(temp)
-            #     temp['paragraphPosition'] = 0.5
-            #     L.append(temp)
-            #     temp['paragraphPosition'] = 0.75
-            #     L.append(temp)
-        result = {
-            "traceId": trace_id,
-            "miniprogramList": L
-        }
-    elif content_status == 3:
-        result = {
-            "traceId": trace_id,
-            "code": 0,
-            "error": "匹配失败，处理超过五次，文章敏感"
-        }
-    else:
-        result = {
-            "traceId": trace_id,
-            "Message": "UnKnow Error"
-        }
-    logging(
-        code="1002",
-        info="返回结果",
-        function="process",
-        data=result,
-        trace_id=trace_id
-    )
-    return result

applications/schedule/search_schedule.py

@@ -1,361 +0,0 @@
-"""
-@author: luojunhui
-调用接口在微信内搜索视频
-"""
-import json
-import time
-
-from applications.match_algorithm.rank import title_similarity_rank
-from applications.search import *
-from static.config import gh_id_dict, db_article
-from applications.functions.log import logging
-from applications.functions.video_item import VideoProducer
-from applications.functions.kimi import KimiServer
-from applications.functions.common import request_etl
-
-
-class SearchABTest(object):
-    """
-    搜索策略实验方案
-    """
-
-    ori_title = None
-    article_summary = None
-    article_keys = None
-    gh_id = None
-    trace_id = None
-
-    def __init__(self, info, gh_id):
-        SearchABTest.set_class_properties(info, gh_id)
-
-    @classmethod
-    def set_class_properties(cls, info, gh_id):
-        """
-        初始化搜索策略实验类
-        :param info: kimi 挖掘的基本信息
-        :param gh_id: 公众号账号 id
-        :return:
-        """
-        cls.ori_title = info["ori_title"]
-        cls.article_summary = info["content_title"]
-        cls.article_keys = info["content_keys"]
-        cls.trace_id = info["trace_id"]
-        cls.gh_id = gh_id
-
-    @classmethod
-    async def base_line(cls):
-        """
-        兜底策略
-        """
-        result = await SearchMethod().search_v1(
-            text=cls.article_keys[0],
-            trace_id=cls.trace_id
-        )
-        if result:
-            return result
-        else:
-            sub_result = await SearchMethod().search_v1(
-                text=cls.article_keys[1],
-                trace_id=cls.trace_id)
-            if sub_result:
-                return sub_result
-            else:
-                return await SearchMethod().search_v1(
-                    text=cls.article_keys[2],
-                    trace_id=cls.trace_id
-                )
-
-    @classmethod
-    async def ab_0(cls):
-        """
-        默认原标题搜索
-        :return:
-        """
-        search_result = await SearchMethod().search_v1(
-            text=cls.ori_title,
-            trace_id=cls.trace_id
-        )
-        if search_result:
-            return search_result
-        else:
-            return await cls.base_line()
-
-    @classmethod
-    async def ab_1(cls):
-        """
-        使用 content_summary搜索
-        :return:
-        """
-        search_result = await SearchMethod().search_v1(
-            text=cls.article_summary,
-            trace_id=cls.trace_id
-        )
-        if search_result:
-            return search_result
-        else:
-            return await cls.ab_0()
-
-    @classmethod
-    async def ab_2(cls):
-        """
-        使用文本关键词搜索
-        :return:
-        """
-        search_result = await SearchMethod().search_v1(
-            text=cls.article_keys[0],
-            trace_id=cls.trace_id
-        )
-        if search_result:
-            return search_result
-        else:
-            return await cls.base_line()
-
-    @classmethod
-    async def ab_3(cls):
-        """
-        使用文本关键词搜索
-        :return:
-        """
-        search_result = await SearchMethod().search_v1(
-            text=cls.article_keys[1],
-            trace_id=cls.trace_id
-        )
-        if search_result:
-            return search_result
-        else:
-            return await cls.base_line()
-
-    @classmethod
-    async def ab_4(cls):
-        """
-        使用文本关键词搜索
-        :return:
-        """
-        search_result = await SearchMethod().search_v1(
-            text=cls.article_keys[2],
-            trace_id=cls.trace_id
-        )
-        if search_result:
-            return search_result
-        else:
-            return await cls.base_line()
-
-    @classmethod
-    async def ab_5(cls):
-        """
-        增量搜索, 返回result_list
-        :return:
-        """
-        result_list = await SearchMethod().search_v2(
-            text=cls.article_summary[:15],
-            trace_id=cls.trace_id
-        )
-        if len(result_list) > 3:
-            return result_list
-        else:
-            result_list += await SearchMethod().search_v2(
-                text=cls.ori_title[:15],
-                trace_id=cls.trace_id
-            )
-            if len(result_list) > 3:
-                return result_list
-            else:
-                result_list += await SearchMethod().search_v2(
-                    text=cls.article_keys[0],
-                    trace_id=cls.trace_id
-                )
-                if len(result_list) > 3:
-                    return result_list
-                else:
-                    result_list += await SearchMethod().search_v2(
-                        text=cls.article_keys[1],
-                        trace_id=cls.trace_id
-                    )
-                    if result_list:
-                        return result_list
-                    else:
-                        result_list += await SearchMethod().search_v2(
-                            text=cls.article_keys[2],
-                            trace_id=cls.trace_id
-                        )
-                        return result_list
-
-
-class SearchMethod(object):
-    """
-    搜索召回模式
-    """
-    s_words = []
-
-    @classmethod
-    async def search_v1(cls, text, trace_id):
-        """
-        dy ---> baidu ---> xigua
-        :param text:
-        :param trace_id:
-        :return:
-        """
-        douyin_result = douyin_search(keyword=text, sensitive_words=cls.s_words, trace_id=trace_id)
-        if douyin_result:
-            return {"platform": "dy_search", "result": douyin_result[0]}
-        else:
-            time.sleep(1)
-            baidu_result = hksp_search(key=text, sensitive_words=cls.s_words, trace_id=trace_id)
-            if baidu_result:
-                return {"platform": "baidu_search", "result": baidu_result[0]}
-            else:
-                xigua_result = xigua_search_v2(keyword=text, sensitive_words=cls.s_words)
-                if xigua_result:
-                    return {"platform": "xg_search", "result": xigua_result[0]}
-                else:
-                    return None
-
-    @classmethod
-    async def search_v2(cls, text, trace_id):
-        """
-        dy ---> baidu ---> xigua
-        :param trace_id:
-        :param text:
-        :return:
-        """
-        L = []
-        douyin_result = douyin_search(keyword=text, sensitive_words=cls.s_words, trace_id=trace_id)
-        for vid_obj in douyin_result:
-            L.append({"platform": "dy_search", "result": vid_obj})
-        if len(L) >= 3:
-            return L
-        else:
-            baidu_result = hksp_search(key=text, sensitive_words=cls.s_words, trace_id=trace_id)
-            if baidu_result:
-                L.append({"platform": "baidu_search", "result": baidu_result[0]})
-            xigua_result = xigua_search_v2(keyword=text, sensitive_words=cls.s_words)
-            if xigua_result:
-                L.append({"platform": "xg_search", "result": xigua_result[0]})
-            return L
-
-
-async def video_sender(video_obj, user, trace_id, platform, content_id):
-    """
-    异步处理微信 video_obj
-    公众号和站内账号一一对应
-    :param content_id：
-    :param platform:
-    :param user:
-    :param trace_id:
-    :param video_obj:
-    :return:
-    """
-    Video = VideoProducer()
-    if platform == "xg_search":
-        mq_obj = Video.xg_video_producer(
-            video_obj=video_obj,
-            user=user,
-            trace_id=trace_id,
-        )
-    elif platform == "baidu_search":
-        mq_obj = Video.baidu_video_producer(
-            video_obj=video_obj,
-            user=user,
-            trace_id=trace_id,
-        )
-    elif platform == "wx_search":
-        mq_obj = Video.wx_video_producer(
-            video_obj=video_obj,
-            user=user,
-            trace_id=trace_id,
-        )
-    elif platform == "dy_search":
-        mq_obj = Video.dy_video_producer(
-            video_obj=video_obj,
-            user=user,
-            trace_id=trace_id,
-        )
-    else:
-        mq_obj = {}
-    mq_obj['trace_id'] = trace_id
-    mq_obj['content_id'] = content_id
-    header = {
-        "Content-Type": "application/json",
-    }
-    response = await request_etl(
-        url="http://192.168.203.137:4612/etl",
-        headers=header,
-        json_data=mq_obj
-    )
-    return response
-    # response = await request_etl(
-    #     url="http://localhost:4612/etl",
-    #     headers=header,
-    #     json_data=mq_obj
-    # )
-    # return response
-
-
-async def search_videos(params, trace_id, gh_id, mysql_client):
-    """
-    search and send msg to ETL
-    :param mysql_client:
-    :param params:
-    :param gh_id: 通过账号 id 来控制实验策略
-    :param trace_id:
-    :return:
-    """
-    K = KimiServer()
-    kimi_info = await K.search_kimi_schedule(params=params)
-    kimi_title = kimi_info['k_title']
-    content_title = kimi_info['content_title'].replace("'", "").replace('"', "")
-    content_keys = json.dumps(kimi_info['content_keys'], ensure_ascii=False)
-    update_kimi_sql = f"""
-                    UPDATE {db_article} SET
-                    kimi_title = %s,
-                    kimi_summary = %s,
-                    kimi_keys = %s
-                    WHERE trace_id = %s;
-                """
-    await mysql_client.async_insert(
-        sql=update_kimi_sql,
-        params=(kimi_title, content_title, content_keys, trace_id)
-    )
-    kimi_info["trace_id"] = trace_id
-    SearchAB = SearchABTest(info=kimi_info, gh_id=gh_id)
-    # 启三个搜索，每个搜索都保证要搜索到， 分别用key1， key2， key3去搜索
-    recall_list = await SearchAB.ab_5()
-    logging(
-        code="1006",
-        info="搜索到{}条视频".format(len(recall_list)),
-        data=recall_list,
-        trace_id=trace_id
-    )
-    # 按照标题相似度排序
-    ranked_list = title_similarity_rank(content_title=params['title'].split("@@")[-1], recall_list=recall_list)
-    index = 0
-    for recall_obj in ranked_list:
-        if recall_obj:
-            platform = recall_obj['platform']
-            recall_video = recall_obj['result']
-            if recall_video:
-                response = await video_sender(
-                    video_obj=recall_video,
-                    user=gh_id_dict.get(gh_id),
-                    trace_id=trace_id,
-                    platform=platform,
-                    content_id=params['content_id']
-                )
-                if response['status'] == "success":
-                    index += 1
-                    logging(
-                        code="1007",
-                        info="成功请求etl",
-                        data=recall_video,
-                        trace_id=trace_id
-                    )
-                if index >= 3:
-                    print("already downloaded 3 videos")
-                    logging(
-                        code="1008",
-                        info="成功下载三条视频",
-                        trace_id=trace_id
-                    )
-                    return index
-    return index
-

applications/search/dy_search.py

@@ -5,7 +5,7 @@ import json
 import requests
 
 from applications.functions.common import sensitive_flag
-from applications.functions.log import logging
+from applications.log import logging
 
 
 def douyin_search(keyword, sensitive_words, trace_id):

applications/search/hksp_search.py

@@ -12,7 +12,7 @@ from uuid import uuid4
 from fake_useragent import FakeUserAgent
 
 from applications.functions.common import sensitive_flag
-from applications.functions.log import logging
+from applications.log import logging
 
 
 def tunnel_proxies():

applications/spider/__init__.py

@@ -4,7 +4,7 @@
 from datetime import datetime
 
 from applications.functions.video_item import VideoProducer
-from applications.functions.log import logging
+from applications.log import logging
 from applications.match_algorithm.rank import title_similarity_rank
 from .spiderAB import SearchABTest
 from .spiderSchedule import SearchMethod

etlTask.py

@@ -23,5 +23,5 @@ if __name__ == '__main__':
     while True:
         asyncio.run(main())
         now_str = datetime.datetime.now().__str__()
-        print("{}    请求执行完成， 等待10s".format(now_str))
-        time.sleep(10)
+        print("{}    请求执行完成， 等待30s".format(now_str))
+        time.sleep(30)

historyTask.py

@@ -23,5 +23,5 @@ if __name__ == '__main__':
     while True:
         asyncio.run(main())
         now_str = datetime.datetime.now().__str__()
-        print("{}    请求执行完成， 等待10s".format(now_str))
-        time.sleep(10)
+        print("{}    请求执行完成， 等待60s".format(now_str))
+        time.sleep(60)

kimiTask.py

@@ -20,5 +20,5 @@ if __name__ == '__main__':
     while True:
         asyncio.run(main())
         now_str = datetime.datetime.now().__str__()
-        print("{}    请求执行完成， 等待10s".format(now_str))
-        time.sleep(10)
+        print("{}    请求执行完成， 等待60s".format(now_str))
+        time.sleep(60)

publishtask.py

@@ -23,5 +23,5 @@ if __name__ == '__main__':
     while True:
         asyncio.run(main())
         now_str = datetime.datetime.now().__str__()
-        print("{}    请求执行完成， 等待10s".format(now_str))
-        time.sleep(10)
+        print("{}    请求执行完成， 等待60s".format(now_str))
+        time.sleep(60)

requirements.txt

@@ -1,4 +1,4 @@
-aiofiles
+aiofiles~=24.1.0
 aiohttp~=3.10.4
 aliyun-log-python-sdk
 aliyun-python-sdk-core
@@ -8,4 +8,7 @@ requests~=2.32.3
 tqdm~=4.66.5
 pymysql~=1.1.1
 pyapollos~=0.1.5
-aiomysql~=0.2.0
+aiomysql~=0.2.0
+oss2~=2.19.0
+lxml~=5.3.0
+openai~=1.47.1

server/api/record.py

@@ -2,7 +2,7 @@ import time
 
 from uuid import uuid4
 
-from applications.functions.log import logging
+from applications.log import logging
 
 
 class Record(object):
@@ -83,8 +83,9 @@ class Record(object):
             )
         )
         logging(
-            code="1002",
-            info="成功记录请求数据到mysql中",
+            code="1001",
+            info="请求文章存储到 long_articles_match_videos中",
+            function="Record",
             trace_id=self.trace_id
         )
 
@@ -106,8 +107,19 @@ class Record(object):
                     self.contents
                 )
             )
+            logging(
+                code="1002",
+                info="请求文章存储到 long_articles_text中",
+                function="Record",
+                trace_id=self.trace_id
+            )
         except Exception as e:
-            print(e)
+            logging(
+                code="1002",
+                info="请求文章 id 已经存储在 long_article_text中    {}".format(e),
+                function="Record",
+                trace_id=self.trace_id
+            )
 
     async def deal(self):
         """

server/api/response.py

@@ -8,7 +8,7 @@ import random
 import hashlib
 import urllib.parse
 
-from applications.functions.log import logging
+from applications.log import logging
 
 
 class Response(object):
@@ -58,7 +58,7 @@ class Response(object):
         return {
             "ghId": gh_id,
             "contentStatus": content_status,
-            "response": json.loads(response),
+            "response": response,
             "processTimes": process_times
         }
 
@@ -148,7 +148,7 @@ class Response(object):
         :return:
         """
         gh_id = result['ghId']
-        response = result['response']
+        response = json.loads(result['response'])
         match self.mini_program_type:
             case 1:
                 L = []

tasks/history_task.py

@@ -2,11 +2,10 @@
 @author: luojunhui
 """
 import json
-import time
 import asyncio
 
 from applications.config import Config
-from applications.functions.log import logging
+from applications.log import logging
 from applications.functions.pqFunctions import publishToPQ
 
 

tasks/kimi_task.py

@@ -4,7 +4,7 @@
 import json
 import asyncio
 from applications.functions.kimi import KimiServer
-from applications.functions.log import logging
+from applications.log import logging
 from applications.config import Config
 
 

tasks/publish_task.py

@@ -7,7 +7,7 @@ import json
 import time
 
 from applications.config import Config
-from applications.functions.log import logging
+from applications.log import logging
 from applications.functions.pqFunctions import publishToPQ
 
 

tasks/spider_task.py

@@ -5,7 +5,7 @@ import asyncio
 import json
 
 from applications.config import Config
-from applications.functions.log import logging
+from applications.log import logging
 from applications.spider import searchVideos
 
 
@@ -234,6 +234,7 @@ class spiderTask(object):
         :return:
         """
         task_list = await self.getTask()
+        print(task_list)
         if task_list:
             tasks = [self.processTask(params) for params in task_list]
             await asyncio.gather(*tasks)

tasks/task2.py

@@ -1,170 +0,0 @@
-"""
-@author: luojunhui
-"""
-import asyncio
-
-from static.config import db_article, db_video
-from applications.functions.log import logging
-from static.config import mysql_coroutines
-
-
-class MatchTask2(object):
-    """
-    定时执行任务
-    """
-
-    def __init__(self, mysql_client):
-        """
-        :param mysql_client:
-        """
-        self.mysql_client = mysql_client
-
-    async def get_task(self):
-        """
-        获取任务
-        :return:
-        """
-        select_sql = f"""
-            SELECT trace_id, content_id, gh_id, article_title, article_text, content_status, process_times
-            FROM {db_article} 
-            WHERE content_status = 0 and process_times <= 5
-            ORDER BY request_time_stamp
-            ASC
-            LIMIT {mysql_coroutines};
-        """
-        task_list = await self.mysql_client.async_select(sql=select_sql)
-        task_obj_list = [
-            {
-                "trace_id": item[0],
-                "content_id": item[1],
-                "gh_id": item[2],
-                "title": item[3],
-                "text": item[4],
-                "content_status": item[5],
-                "process_times": item[6]
-            } for item in task_list
-        ]
-        logging(
-            code="9001",
-            info="本次任务获取到 {} 条视频".format(len(task_obj_list)),
-            data=task_obj_list
-        )
-        return task_obj_list
-
-    async def get_history_videos(self, content_id):
-        """
-        check whether the contents videos exists
-        :param content_id:
-        :return:
-        """
-        select_sql = f"""
-            SELECT video_id
-            FROM {db_video}
-            where content_id = '{content_id}' and video_status = 1 order by request_time DESC;
-        """
-        content_videos = await self.mysql_client.async_select(select_sql)
-        videos = [vid for vid in content_videos]
-        if len(videos) >= 3:
-            return videos
-        else:
-            return None
-
-    async def use_exists_contents_videos(self, video_id_list, params):
-        """
-        使用已经存在的视频id
-        :return:
-        """
-        trace_id = params['trace_id']
-        content_id = params['content_id']
-        select_sql = f"""
-            SELECT kimi_title
-            FROM {db_article}
-            WHERE content_id = '{content_id}' and kimi_title is not null limit 1;
-        """
-        info = await self.mysql_client.async_select(sql=select_sql)
-        kimi_title = info[0]
-        update_sql = f"""
-            UPDATE {db_article}
-            SET 
-                kimi_title=%s,
-                recall_video_id1=%s, 
-                recall_video_id2=%s, 
-                recall_video_id3=%s,
-                content_status=%s,
-                process_times = %s
-            WHERE  trace_id = %s
-        """
-        vid1, vid2, vid3 = video_id_list[0], video_id_list[1], video_id_list[2]
-        await self.mysql_client.async_insert(
-            sql=update_sql,
-            params=(
-                kimi_title,
-                video_id_list[0],
-                "NULL" if vid2 is None else vid2,
-                "NULL" if vid3 is None else vid3,
-                2,
-                int(params['process_times']) + 1,
-                trace_id
-            )
-        )
-        logging(
-            code="9002",
-            info="已从历史文章更新，文章id： {}".format(content_id),
-            trace_id=trace_id
-        )
-
-    async def process_video_id(self, title, trace_id, process_times):
-        """
-        如果video_id在标题中，则做特殊处理
-        :return:
-        """
-        video_id = title.split("video_id=")[-1]
-        update_sql = f"""
-            UPDATE  
-                {db_article}
-            SET 
-                recall_video_id1 = %s,
-                content_status = %s,
-                process_times = %s
-            WHERE  
-                trace_id = %s;"""
-        await self.mysql_client.async_insert(
-            sql=update_sql,
-            params=(video_id, 2, int(process_times) + 1, trace_id)
-        )
-
-    async def process_task(self, params):
-        """
-        异步执行
-        :param params:
-        :return:
-        """
-        content_id = params['content_id']
-        trace_id = params['trace_id']
-        # 判断该篇文章是否存在未下架的视频，且判断是否有3条， 如果没有三条，则启动新抓取任务，后续优化点
-        video_id_list = await self.get_history_videos(content_id=content_id)
-        if video_id_list:
-            # 说明已经存在了结果, 将该条记录下的video_id拿出来
-            logging(
-                code="9001",
-                info="存在历史文章",
-                trace_id=trace_id
-            )
-            await self.use_exists_contents_videos(video_id_list=video_id_list, params=params)
-        else:
-            pass
-
-    async def deal(self):
-        """
-        处理
-        :return:
-        """
-        task_list = await self.get_task()
-        if task_list:
-            tasks = [self.process_task(params) for params in task_list]
-            await asyncio.gather(*tasks)
-        else:
-            logging(
-                code="9008",
-                info="没有要处理的请求"
-            )

tasks/task3.py

@@ -1,176 +0,0 @@
-"""
-@author: luojunhui
-"""
-import asyncio
-
-from static.config import db_article, db_video, mysql_coroutines
-from applications.functions.log import logging
-from applications.functions.pqFunctions import *
-
-
-class MatchTask3(object):
-    """
-    处理已经匹配过小程序的文章
-    """
-
-    def __init__(self, mysql_client):
-        """
-        :param mysql_client:
-        """
-        self.mysql_client = mysql_client
-
-    async def getTaskList(self):
-        """
-        获取任务
-        :return:
-        """
-        select_sql1 = f"""
-            SELECT 
-                ART.trace_id, 
-                ART.content_id, 
-                ART.gh_id, 
-                ART.article_title, 
-                ART.article_text, 
-                ART.content_status, 
-                ART.process_times
-            FROM {db_article} ART
-            JOIN (
-                select content_id, count(1) as cnt 
-                from {db_video}
-                where oss_status = 1
-                group by content_id
-            ) VID on ART.content_id = VID.content_id and VID.cnt >= 3
-            WHERE ART.content_status = 0 and ART.process_times <= 3
-            ORDER BY request_time_stamp
-            LIMIT {mysql_coroutines};
-        """
-        tasks = await self.mysql_client.async_select(sql=select_sql1)
-        task_obj_list = [
-            {
-                "trace_id": item[0],
-                "content_id": item[1],
-                "gh_id": item[2],
-                "title": item[3],
-                "text": item[4],
-                "content_status": item[5],
-                "process_times": item[6]
-            } for item in tasks
-        ]
-        logging(
-            code="9001",
-            info="本次任务获取到 {} 条视频".format(len(task_obj_list)),
-            data=task_obj_list
-        )
-        return task_obj_list
-
-    async def getHistoryVideoOssPath(self, content_id):
-        """
-        check whether the contents videos exists
-        :param content_id:
-        :return:
-        """
-        select_sql = f"""
-            SELECT video_title, uid, video_path, cover_path
-            FROM {db_video}
-            where content_id = '{content_id}' and oss_status = 1 order by request_time DESC;
-        """
-        content_videos = await self.mysql_client.async_select(select_sql)
-        video_list = [
-            {
-                "title": line[0],
-                "uid": line[1],
-                "videoPath": line[2],
-                "coverPath": line[3]
-            }
-            for line in content_videos
-        ]
-        if len(video_list) >= 3:
-            return video_list
-        else:
-            return None
-
-    async def useExistOssPath(self, video_info_list, params):
-        """
-        使用已经存在的视频id
-        :return:
-        """
-        trace_id = params['trace_id']
-        content_id = params['content_id']
-        select_sql = f"""
-            SELECT kimi_title
-            FROM {db_article}
-            WHERE content_id = '{content_id}' and kimi_title is not null limit 1;
-        """
-        info = await self.mysql_client.async_select(sql=select_sql)
-        kimi_title = info[0]
-        video_id_list = await getNewVideoIds(video_info_list)
-        vid1, vid2, vid3 = video_id_list[0], video_id_list[1], video_id_list[2]
-        update_sql = f"""
-            UPDATE {db_article}
-            SET 
-                kimi_title=%s,
-                recall_video_id1=%s, 
-                recall_video_id2=%s, 
-                recall_video_id3=%s,
-                content_status=%s,
-                process_times = %s
-            WHERE  trace_id = %s
-        """
-
-        await self.mysql_client.async_insert(
-            sql=update_sql,
-            params=(
-                kimi_title,
-                vid1,
-                vid2,
-                vid3,
-                2,
-                int(params['process_times']) + 1,
-                trace_id
-            )
-        )
-        logging(
-            code="9002",
-            info="已从历史文章更新，文章id： {}".format(content_id),
-            trace_id=trace_id
-        )
-
-    async def processTask(self, params):
-        """
-        异步执行
-        :param params:
-        :return:
-        """
-        content_id = params['content_id']
-        trace_id = params['trace_id']
-        # 判断该篇文章是否存在未下架的视频，且判断是否有3条， 如果没有三条，则启动新抓取任务，后续优化点
-        oss_path_list = await self.getHistoryVideoOssPath(content_id=content_id)
-        if oss_path_list:
-            # 说明已经存在了结果, 将该条记录下的video_oss拿出来
-            logging(
-                code="9001",
-                info="存在历史文章",
-                trace_id=trace_id
-            )
-            try:
-                await self.useExistOssPath(video_info_list=oss_path_list, params=params)
-            except Exception as e:
-                print(e)
-        else:
-            pass
-
-    async def deal(self):
-        """
-        处理
-        :return:
-        """
-        task_list = await self.getTaskList()
-        if task_list:
-            tasks = [self.processTask(params) for params in task_list]
-            await asyncio.gather(*tasks)
-        else:
-            logging(
-                code="9008",
-                info="没有要处理的请求"
-            )
-

tasks/task4.py

@@ -1,177 +0,0 @@
-"""
-@author: luojunhui
-"""
-import asyncio
-
-from static.config import db_article, db_video, mysql_coroutines
-from applications.functions.log import logging
-from applications.functions.pqFunctions import *
-
-
-class MatchTask4(object):
-    """
-    处理已经匹配过小程序的文章
-    """
-
-    def __init__(self, mysql_client):
-        """
-        :param mysql_client:
-        """
-        self.mysql_client = mysql_client
-
-    async def getTaskList(self):
-        """
-        获取任务
-        :return:
-        """
-        select_sql1 = f"""
-            SELECT 
-                ART.trace_id, 
-                ART.content_id, 
-                ART.gh_id, 
-                ART.article_title, 
-                ART.article_text, 
-                ART.content_status, 
-                ART.process_times
-            FROM {db_article} ART
-            JOIN (
-                select content_id, count(1) as cnt 
-                from {db_video}
-                where oss_status = 1
-                group by content_id
-            ) VID on ART.content_id = VID.content_id and VID.cnt >= 3
-            WHERE ART.content_status = 0 and ART.process_times <= 3
-            ORDER BY request_time_stamp
-            DESC
-            LIMIT {mysql_coroutines};
-        """
-        tasks = await self.mysql_client.async_select(sql=select_sql1)
-        task_obj_list = [
-            {
-                "trace_id": item[0],
-                "content_id": item[1],
-                "gh_id": item[2],
-                "title": item[3],
-                "text": item[4],
-                "content_status": item[5],
-                "process_times": item[6]
-            } for item in tasks
-        ]
-        logging(
-            code="9001",
-            info="本次任务获取到 {} 条视频".format(len(task_obj_list)),
-            data=task_obj_list
-        )
-        return task_obj_list
-
-    async def getHistoryVideoOssPath(self, content_id):
-        """
-        check whether the contents videos exists
-        :param content_id:
-        :return:
-        """
-        select_sql = f"""
-            SELECT video_title, uid, video_path, cover_path
-            FROM {db_video}
-            where content_id = '{content_id}' and oss_status = 1 order by request_time DESC;
-        """
-        content_videos = await self.mysql_client.async_select(select_sql)
-        video_list = [
-            {
-                "title": line[0],
-                "uid": line[1],
-                "videoPath": line[2],
-                "coverPath": line[3]
-            }
-            for line in content_videos
-        ]
-        if len(video_list) >= 3:
-            return video_list
-        else:
-            return None
-
-    async def useExistOssPath(self, video_info_list, params):
-        """
-        使用已经存在的视频id
-        :return:
-        """
-        trace_id = params['trace_id']
-        content_id = params['content_id']
-        select_sql = f"""
-            SELECT kimi_title
-            FROM {db_article}
-            WHERE content_id = '{content_id}' and kimi_title is not null limit 1;
-        """
-        info = await self.mysql_client.async_select(sql=select_sql)
-        kimi_title = info[0]
-        video_id_list = await getNewVideoIds(video_info_list)
-        vid1, vid2, vid3 = video_id_list[0], video_id_list[1], video_id_list[2]
-        update_sql = f"""
-            UPDATE {db_article}
-            SET 
-                kimi_title=%s,
-                recall_video_id1=%s, 
-                recall_video_id2=%s, 
-                recall_video_id3=%s,
-                content_status=%s,
-                process_times = %s
-            WHERE  trace_id = %s
-        """
-
-        await self.mysql_client.async_insert(
-            sql=update_sql,
-            params=(
-                kimi_title,
-                vid1,
-                vid2,
-                vid3,
-                2,
-                int(params['process_times']) + 1,
-                trace_id
-            )
-        )
-        logging(
-            code="9002",
-            info="已从历史文章更新，文章id： {}".format(content_id),
-            trace_id=trace_id
-        )
-
-    async def processTask(self, params):
-        """
-        异步执行
-        :param params:
-        :return:
-        """
-        content_id = params['content_id']
-        trace_id = params['trace_id']
-        # 判断该篇文章是否存在未下架的视频，且判断是否有3条， 如果没有三条，则启动新抓取任务，后续优化点
-        oss_path_list = await self.getHistoryVideoOssPath(content_id=content_id)
-        if oss_path_list:
-            # 说明已经存在了结果, 将该条记录下的video_oss拿出来
-            logging(
-                code="9001",
-                info="存在历史文章",
-                trace_id=trace_id
-            )
-            try:
-                await self.useExistOssPath(video_info_list=oss_path_list, params=params)
-            except Exception as e:
-                print(e)
-        else:
-            pass
-
-    async def deal(self):
-        """
-        处理
-        :return:
-        """
-        task_list = await self.getTaskList()
-        if task_list:
-            tasks = [self.processTask(params) for params in task_list]
-            await asyncio.gather(*tasks)
-        else:
-            logging(
-                code="9008",
-                info="没有要处理的请求"
-            )
-