调试

applications/deal/__init__.py

@@ -5,5 +5,4 @@
 from .record import Record
 from .response import Response
 from .minigram import Minigram
-from .preResponse import PreResponse
-from .finalResponse import FinalResponse
+from .getOffVideos import GetOffVideos

applications/deal/getOffVideos.py

@@ -0,0 +1,75 @@
+"""
+@author: luojunhui
+"""
+import time
+
+from static.config import db_article
+
+
+class GetOffVideos(object):
+    """
+    下架视频
+    """
+
+    def __init__(self, params, mysql_client):
+        self.params = params
+        self.mysql_client = mysql_client
+        self.trace_id = None
+
+    def checkParams(self):
+        """
+
+        :return:
+        """
+        try:
+            self.trace_id = self.params['traceId']
+            return None
+        except Exception as e:
+            response = {
+                "error": "params error",
+                "info": str(e),
+                "data": self.params
+            }
+            return response
+
+    async def pushVideoIntoQueue(self):
+        """
+        将视频id记录到待下架表中
+        :return:
+        """
+        select_sql = f"""
+        select recall_video_id1, recall_video_id2, recall_video_id3 from {db_article}
+        where trace_id = '{self.trace_id}';
+        """
+        recall_video_info = await self.mysql_client.async_select(sql=select_sql)
+        recall_vid_tuple = recall_video_info[0]
+        for vid in recall_vid_tuple:
+            try:
+                update_sql = f"""
+                INSERT INTO get_off_videos
+                (video_id, publish_time, video_status, trace_id)
+                values 
+                (%s, %s, %s, %s);
+                """
+                await self.mysql_client.async_insert(
+                    sql=update_sql,
+                    params=(vid, int(time.time()), 1, self.trace_id)
+                )
+            except Exception as e:
+                print(e)
+
+    async def deal(self):
+        """
+
+        :return:
+        """
+        params_error = self.checkParams()
+        if params_error:
+            return params_error
+        else:
+            await self.pushVideoIntoQueue()
+            response = {
+                "status": "success",
+                "traceId": self.trace_id
+            }
+            return response

applications/deal/preResponse.py

@@ -1,68 +0,0 @@
-"""
-@author: luojunhui
-返回预分配信息
-"""
-import json
-
-from static.config import db_article
-
-
-class PreResponse(object):
-    """
-    预匹配处理
-    """
-
-    def __init__(self, params, mysql_client):
-        """
-        输入trace_id, 返回trace_id所绑定的视频信息
-        :param params:
-        :param mysql_client:
-        """
-        self.trace_id = None
-        self.params = params
-        self.mysql_client = mysql_client
-
-    def checkParams(self):
-        """
-        校验参数
-        :return:
-        """
-        try:
-            self.trace_id = self.params['traceId']
-            return None
-        except Exception as e:
-            response = {
-                "error": "params check failed",
-                "info": str(e)
-            }
-            return response
-
-    async def findVideoInfo(self):
-        """
-
-        :return:
-        """
-        sql = f"""
-        select video_info1, video_info2, video_info3 from {db_article}
-        where trace_id = '{self.trace_id}';
-        """
-        video_info = await self.mysql_client.async_select(sql)
-        video_info_list = video_info[0]
-        vid_list = [json.loads(i) for i in video_info_list]
-        return vid_list
-
-    async def deal(self):
-        """
-        deal function
-        :return:
-        """
-        params_error = self.checkParams()
-        if params_error:
-            return params_error
-        else:
-            video_info_list = await self.findVideoInfo()
-            response = {
-                "traceId": self.trace_id,
-                "videoObjList": video_info_list
-            }
-            return response

applications/functions/pqFunctions.py

@@ -0,0 +1,83 @@
+"""
+@author: luojunhui
+"""
+import json
+
+from applications.functions.common import async_post
+
+
+async def publishToPQ(video_obj):
+    """
+    publish video to pq
+    :return:
+    """
+    oss_path = video_obj['videoPath']
+    uid = video_obj['uid']
+    title = video_obj['title']
+    cover = video_obj['coverPath']
+    url = "https://vlogapi.piaoquantv.com/longvideoapi/crawler/video/send"
+    headers = {
+        "User-Agent": "PQSpeed/486 CFNetwork/1410.1 Darwin/22.6.0",
+        "cookie": "JSESSIONID=4DEA2B5173BB9A9E82DB772C0ACDBC9F; JSESSIONID=D02C334150025222A0B824A98B539B78",
+        "referer": "http://appspeed.piaoquantv.com",
+        "token": "524a8bc871dbb0f4d4717895083172ab37c02d2f",
+        "accept-language": "zh-CN,zh-Hans;q=0.9",
+        "Content-Type": "application/x-www-form-urlencoded",
+    }
+    payload = {
+        "coverImgPath": cover,
+        "deviceToken": "9ef064f2f7869b3fd67d6141f8a899175dddc91240971172f1f2a662ef891408",
+        "fileExtensions": "MP4",
+        "loginUid": uid,
+        "networkType": "Wi-Fi",
+        "platform": "iOS",
+        "requestId": "fb972cbd4f390afcfd3da1869cd7d001",
+        "sessionId": "362290597725ce1fa870d7be4f46dcc2",
+        "subSessionId": "362290597725ce1fa870d7be4f46dcc2",
+        "title": title,
+        "token": "524a8bc871dbb0f4d4717895083172ab37c02d2f",
+        "uid": uid,
+        "versionCode": "486",
+        "versionName": "3.4.12",
+        "videoFromScene": "1",
+        "videoPath": oss_path,
+        "viewStatus": "1",
+    }
+    response = await async_post(url, headers, payload)
+    return response
+
+
+async def getPQVideoDetail(video_id):
+    """
+    获取票圈视频详情信息
+    :return:
+    """
+    url = "https://longvideoapi.piaoquantv.com/longvideoapi/openapi/video/batchSelectVideoInfo"
+    data = {
+        "videoIdList": [video_id]
+    }
+    header = {
+        "Content-Type": "application/json",
+    }
+    response = await async_post(url, header, json.dumps(data))
+    return response
+
+
+async def getNewVideoIds(video_obj_list):
+    """
+    video
+    :return:
+    """
+    vid_list = []
+    for video_obj in video_obj_list:
+        # video_obj 里面的信息对于历史数据可能不全，需要从pq获取
+        try:
+            if len(vid_list) >= 3:
+                return vid_list
+            else:
+                pq_response = await publishToPQ(video_obj)
+                video_id = pq_response['data']['id']
+                vid_list.append(video_id)
+        except:
+            continue
+    return vid_list

applications/routes.py

@@ -3,7 +3,7 @@
 """
 from quart import Blueprint, jsonify, request
 
-from applications.deal import Response, Record, Minigram, PreResponse, FinalResponse
+from applications.deal import Response, Record, Minigram, GetOffVideos
 
 my_blueprint = Blueprint('LongArticles', __name__)
 
@@ -56,25 +56,15 @@ def Routes(mysql_client):
         response = await M.deal()
         return jsonify(response)
 
-    @my_blueprint.route("/pre_match", methods=['POST'])
-    async def pre_match_minigram():
+    @my_blueprint.route("/get_off_videos", methods=['POST'])
+    async def get_off_videos():
         """
+        自动下架视频记录
         :return:
         """
         data = await request.get_json()
-        M = PreResponse(params=data, mysql_client=mysql_client)
-        response = await M.deal()
-        return jsonify(response)
-
-    @my_blueprint.route("/final_match", methods=['POST'])
-    async def final_match_minigram():
-        """
-
-        :return:
-        """
-        data = await request.get_json()
-        FMM = FinalResponse(params=data, mysql_client=mysql_client)
-        response = await FMM.deal()
+        GOV = GetOffVideos(params=data, mysql_client=mysql_client)
+        response = await GOV.deal()
         return jsonify(response)
 
     return my_blueprint

applications/schedule/search_schedule.py

@@ -284,7 +284,7 @@ async def video_sender(video_obj, user, trace_id, platform, content_id):
     # )
     # return response
     response = await request_etl(
-        url="http://localhost:4612/etlV2",
+        url="http://localhost:4612/etl",
         headers=header,
         json_data=mq_obj
     )

match_video_task.py

@@ -6,7 +6,7 @@ import datetime
 import asyncio
 
 import aiomysql
-from tasks import MatchTask1, MatchTask2
+from tasks import MatchTask1, MatchTask3
 
 
 class TaskMySQLClient(object):
@@ -84,12 +84,11 @@ async def main2():
     """
     TMC = TaskMySQLClient()
     await TMC.init_pool()
-    PD = MatchTask2(TMC)
+    PD = MatchTask3(TMC)
     await PD.deal()
 
 
 if __name__ == '__main__':
-    # asyncio.run(main())
     while True:
         asyncio.run(main())
         now_str = datetime.datetime.now().__str__()

static/config.py

@@ -755,12 +755,12 @@ gh_id_dict = {
 
 
 # prod
-# db_article = "long_articles_video"
-# db_video = "article_match_videos"
+db_article = "long_articles_video"
+db_video = "article_match_videos"
 
 # dev
-db_article = "long_articles_video_dev"
-db_video = "article_match_videos_dev"
+# db_article = "long_articles_video_dev"
+# db_video = "article_match_videos_dev"
 
 # spider coroutines
 spider_coroutines = 10

tasks/__init__.py

@@ -3,4 +3,5 @@
 定时任务
 """
 from .task1 import MatchTask1
-from .task2 import MatchTask2
+from .task2 import MatchTask2
+from .task3 import MatchTask3

tasks/chadui.py

@@ -1,9 +1,6 @@
 """
 @author: luojunhui
 """
-"""
-@author: luojunhui
-"""
 import asyncio
 
 from static.config import db_article, db_video

tasks/new_task.py → tasks/task3.py

@@ -2,89 +2,10 @@
 @author: luojunhui
 """
 import asyncio
-import json
 
-from static.config import db_article, db_video
+from static.config import db_article, db_video, mysql_coroutines
 from applications.functions.log import logging
-from static.config import mysql_coroutines
-from applications.functions.common import async_post
-
-
-async def publishToPQ(video_obj):
-    """
-    publish video to pq
-    :return:
-    """
-    oss_path = video_obj['videoPath']
-    uid = video_obj['uid']
-    title = video_obj['title']
-    cover = video_obj['coverPath']
-    url = "https://vlogapi.piaoquantv.com/longvideoapi/crawler/video/send"
-    headers = {
-        "User-Agent": "PQSpeed/486 CFNetwork/1410.1 Darwin/22.6.0",
-        "cookie": "JSESSIONID=4DEA2B5173BB9A9E82DB772C0ACDBC9F; JSESSIONID=D02C334150025222A0B824A98B539B78",
-        "referer": "http://appspeed.piaoquantv.com",
-        "token": "524a8bc871dbb0f4d4717895083172ab37c02d2f",
-        "accept-language": "zh-CN,zh-Hans;q=0.9",
-        "Content-Type": "application/x-www-form-urlencoded",
-    }
-    payload = {
-        "coverImgPath": cover,
-        "deviceToken": "9ef064f2f7869b3fd67d6141f8a899175dddc91240971172f1f2a662ef891408",
-        "fileExtensions": "MP4",
-        "loginUid": uid,
-        "networkType": "Wi-Fi",
-        "platform": "iOS",
-        "requestId": "fb972cbd4f390afcfd3da1869cd7d001",
-        "sessionId": "362290597725ce1fa870d7be4f46dcc2",
-        "subSessionId": "362290597725ce1fa870d7be4f46dcc2",
-        "title": title,
-        "token": "524a8bc871dbb0f4d4717895083172ab37c02d2f",
-        "uid": uid,
-        "versionCode": "486",
-        "versionName": "3.4.12",
-        "videoFromScene": "1",
-        "videoPath": oss_path,
-        "viewStatus": "1",
-    }
-    response = await async_post(url, headers, payload)
-    return response
-
-
-async def getPQVideoDetail(video_id):
-    """
-    获取票圈视频详情信息
-    :return:
-    """
-    url = "https://longvideoapi.piaoquantv.com/longvideoapi/openapi/video/batchSelectVideoInfo"
-    data = {
-        "videoIdList": [video_id]
-    }
-    header = {
-        "Content-Type": "application/json",
-    }
-    response = await async_post(url, header, json.dumps(data))
-    return response
-
-
-async def getNewVideoIds(video_obj_list):
-    """
-    video
-    :return:
-    """
-    vid_list = []
-    for video_obj in video_obj_list:
-        # video_obj 里面的信息对于历史数据可能不全，需要从pq获取
-        try:
-            if len(vid_list) >= 3:
-                return vid_list
-            else:
-                pq_response = await publishToPQ(video_obj)
-                video_id = pq_response['data']['id']
-                vid_list.append(video_id)
-        except:
-            continue
-    return vid_list
+from applications.functions.pqFunctions import *
 
 
 class MatchTask3(object):

test_code/final_match.py

@@ -1,46 +0,0 @@
-"""
-@author: luojunhui
-"""
-import time
-
-import requests
-
-url = "http://localhost:8111/final_match"
-
-header = {
-    "Content-Type": "application/json",
-}
-
-
-body = {
-    "kimiTitle": "🔥96岁北大教授夫人选择断食离世，生前遗言震撼人心！😢",
-    "traceId": "search-d2f0ec07-8d91-4440-a438-1d9e9bbb9941-1723033241",
-    "videoObjList": [
-        {
-            "coverPath": "long_articles/image/919ea8b6-399c-4117-90b1-9cbfa04655c4",
-            "title": "漫话寿高多辱（下）陈司寇老师对待生死的达观态度值得老年人借鉴",
-            "uid": "69637509",
-            "videoPath": "long_articles/video/c6ed59d1-54b2-4e7f-9a09-0f2f40b28afa"
-        },
-        {
-            "coverPath": "long_articles/image/c067097b-cc1e-4773-9d5c-069503d96b89",
-            "title": "96岁北大教授夫人绝食4天后离世，临终遗言引人深思",
-            "uid": "69637509",
-            "videoPath": "long_articles/video/e88d8334-b838-4b0e-ac25-258ca6f1d789"
-        },
-        {
-            "coverPath": "long_articles/image/b8b042e5-fac0-4a43-b869-8ec7f51d38b6",
-            "title": "96岁北大教授夫人绝食4天后离世，临终遗言引人深思",
-            "uid": "69637509",
-            "videoPath": "long_articles/video/e2fd64c2-10d9-48f4-aed1-3cf26c7bc050"
-        }
-    ],
-    "ghId": "gh_6d205db62f04",
-    "miniType": 1
-}
-b = time.time()
-response = requests.post(url=url, headers=header, json=body)
-a = time.time()
-
-print(a - b)
-print(response.json())

test_code/generate_id.py

@@ -1,19 +0,0 @@
-"""
-@author: luojunhui
-pages/category?jumpPage=pages%2Fuser-videos%3Fid%3D21932340%26su%3D70731104%26fromGzh%3D1%26rootShareId%3D703f489c-b942-48f6-898e-6f77642b48e5%26shareId%3D703f489c-b942-48f6-898e-6f77642b48e5%26rootSourceId%3DlongArticles_2930ef1ac0c0f52dcf45ab823abf3ac7
-"""
-import os
-import json
-
-path_dir = os.listdir("vv")
-
-fw = open("id.txt", "a+", encoding="utf-8")
-for file in path_dir:
-    print(file)
-    ori_id = file.replace(".json", "")
-    new_p = os.path.join("vv", file)
-    with open(new_p, encoding="utf-8") as f:
-        data = json.loads(f.read())
-    new_id = data['data']['id']
-    fw.write("{},{}\n".format(ori_id, new_id))
-

tt.py

@@ -6,7 +6,7 @@ import time
 import aiomysql
 import asyncio
 
-from tasks.new_task import MatchTask3
+from tasks.task3 import MatchTask3
 
 
 class TaskMySQLClient(object):
@@ -80,4 +80,5 @@ async def main():
 if __name__ == '__main__':
     while True:
         asyncio.run(main())
+        print("执行完成")
         time.sleep(10)