增加日志、失败重试次数

common/pq_utility.py

@@ -4,33 +4,33 @@ import urllib.parse
 class PQ():
     @classmethod
     def insert_piaoquantv(cls, oss_object_key, title, pq_id):
-        url = "https://vlogapi.piaoquantv.com/longvideoapi/crawler/video/send"
-        payload = dict(pageSource='vlog-pages/post/post-video-post', videoPath=oss_object_key, width='720',
-                       height='1280', fileExtensions='mp4', viewStatus='1', title=title,
-                       careModelStatus='1',
-                       token='f04f58d6e664cbc9902660a1e8d20ce6cd7fdb0f', loginUid=pq_id,
-                       versionCode='719',
-                       machineCode='weixin_openid_o0w175aZ4FJtqVsA1tcozJDJHdDU', appId='wx89e7eb06478361d7',
-                       clientTimestamp='1703337579331',
-                       machineInfo='{"sdkVersion":"3.2.5","brand":"iPhone","language":"zh_CN","model":"iPhone 12 Pro<iPhone13,3>","platform":"ios","system":"iOS 15.6.1","weChatVersion":"8.0.44","screenHeight":844,"screenWidth":390,"pixelRatio":3,"windowHeight":762,"windowWidth":390,"softVersion":"4.1.719"}',
-                       sessionId='1703337560040-27bfe208-a389-f476-db1d-840681e04b32',
-                       subSessionId='1703337569952-8f56d53c-b36d-760e-8abe-0b4a027cd5bd', senceType='1089',
-                       hotSenceType='1089', id='1050', channel='pq')
+        for i in range(3):
+            url = "https://vlogapi.piaoquantv.com/longvideoapi/crawler/video/send"
+            payload = dict(pageSource='vlog-pages/post/post-video-post', videoPath=oss_object_key, width='720',
+                           height='1280', fileExtensions='mp4', viewStatus='1', title=title,
+                           careModelStatus='1',
+                           token='f04f58d6e664cbc9902660a1e8d20ce6cd7fdb0f', loginUid=pq_id,
+                           versionCode='719',
+                           machineCode='weixin_openid_o0w175aZ4FJtqVsA1tcozJDJHdDU', appId='wx89e7eb06478361d7',
+                           clientTimestamp='1703337579331',
+                           machineInfo='{"sdkVersion":"3.2.5","brand":"iPhone","language":"zh_CN","model":"iPhone 12 Pro<iPhone13,3>","platform":"ios","system":"iOS 15.6.1","weChatVersion":"8.0.44","screenHeight":844,"screenWidth":390,"pixelRatio":3,"windowHeight":762,"windowWidth":390,"softVersion":"4.1.719"}',
+                           sessionId='1703337560040-27bfe208-a389-f476-db1d-840681e04b32',
+                           subSessionId='1703337569952-8f56d53c-b36d-760e-8abe-0b4a027cd5bd', senceType='1089',
+                           hotSenceType='1089', id='1050', channel='pq')
 
-        payload['videoPath'] = oss_object_key
-        payload['title'] = title
-        data = urllib.parse.urlencode(payload)
-        headers = {
-            'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 15_6_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 MicroMessenger/8.0.44(0x18002c2d) NetType/WIFI Language/zh_CN',
-            'Accept-Encoding': 'gzip,compress,br,deflate',
-            'Referer': 'https://servicewechat.com/wx89e7eb06478361d7/726/page-frame.html',
-            'Content-Type': 'application/x-www-form-urlencoded',
-            'Cookie': 'JSESSIONID=A60D96E7A300A25EA05425B069C8B459'
-        }
-        response = requests.post(url, data=data, headers=headers)
-        data = response.json()
-        code = data["code"]
-        if code == 0:
-            return True
-        else:
-            return False
+            payload['videoPath'] = oss_object_key
+            payload['title'] = title
+            data = urllib.parse.urlencode(payload)
+            headers = {
+                'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 15_6_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 MicroMessenger/8.0.44(0x18002c2d) NetType/WIFI Language/zh_CN',
+                'Accept-Encoding': 'gzip,compress,br,deflate',
+                'Referer': 'https://servicewechat.com/wx89e7eb06478361d7/726/page-frame.html',
+                'Content-Type': 'application/x-www-form-urlencoded',
+                'Cookie': 'JSESSIONID=A60D96E7A300A25EA05425B069C8B459'
+            }
+            response = requests.post(url, data=data, headers=headers)
+            data = response.json()
+            code = data["code"]
+            if code == 0:
+                return True
+        return False

single_video/gongzhonghao/gongzhonghao_video.py

@@ -6,15 +6,11 @@ import os
 import sys
 import requests
 import json
-import urllib.parse
 from urllib.parse import urlparse
-
-from common import Oss
+sys.path.append(os.getcwd())
+from common import Oss, Common
 from common.pq_utility import PQ
 from common.url_manage import urlManage
-from common.userAgent import get_random_user_agent
-
-sys.path.append(os.getcwd())
 
 config = configparser.ConfigParser()
 config.read('/root/single_video_crawler/config.ini')  # 替换为您的配置文件路径
@@ -84,19 +80,18 @@ class gongzhonghaoVdieo():
                 if status == False:
                     return "视频下载失败"
                 oss_object_key = Oss.video_sync_upload_oss(video_path_url, video_id)
-                status = oss_object_key.get("status")
-                if status != 200:
-                    return "发送OSS失败"
+                oss_object_key.get("status")
                 # 获取 oss 视频地址
                 oss_object_key = oss_object_key.get("oss_object_key")
+                Common.logger().info(f'准备发送站内参数:{oss_object_key},{title},{vx_message[3]}')
                 piaoquantv = PQ.insert_piaoquantv(oss_object_key, title, vx_message[3])
                 if piaoquantv == False:
-                    return "发送账号失败"
+                    return "视频发送到站内失败"
                 if os.path.isfile(video_path_url):
                     os.remove(video_path_url)
-                return
             else:
-                return "无法获取视频链接"
-        except Exception:
-            return "URL处理失败"
+                return "无法获取到视频ID"
+        except Exception as e:
+            Common.logger().info(f'报错信息:{e}')
+            return f"处理报错，报错信息{e}"
 

single_video/kuaishou/kuaishou_video.py

@@ -4,7 +4,7 @@ import re
 import requests
 from bs4 import BeautifulSoup
 
-from common import Oss
+from common import Oss, Common
 from common.pq_utility import PQ
 from common.url_manage import urlManage
 from common.userAgent import get_random_user_agent
@@ -14,10 +14,17 @@ class kuaishouVideo():
     # 获取快手 标题+视频链接
     @classmethod
     def get_videoList(cls, vx_message, channel):
-        try:
-            data_link = vx_message[1]
+        data_link = vx_message[1]
+        content_id = ''
+        for i in range(3):
             content_id = urlManage.url_manage(data_link, channel)
             if content_id:
+                break
+        if content_id == '':
+            return "无法获取到视频ID"
+        Common.logger().info(f'渠道:{channel},视频ID:{content_id}')
+        for i in range(3):
+            try:
                 url = f"https://www.kuaishou.com/short-video/{content_id}"
                 headers = {
                   'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
@@ -36,16 +43,13 @@ class kuaishouVideo():
                 response = requests.get(url, headers=headers)
                 if "VisionVideoDetailPhoto" not in response.text:
                     return "cookie过期"
-
                 status_code = response.status_code
                 if status_code != 200:
                     return "cookie过期"
                 data = response.text
                 soup = BeautifulSoup(data, 'html.parser')
-
                 script_tags = soup.find_all('script', text=lambda x: x and "__APOLLO_STATE__" in x)
                 pattern = re.compile(r'window\.__APOLLO_STATE__\s*=\s*({.*?});', re.DOTALL)
-
                 for script in script_tags:
                     # 获取<script>标签内容
                     script_content = script.string
@@ -63,16 +67,12 @@ class kuaishouVideo():
                         # 随机生成视频oss_id
                         video_id = urlManage.random_id()
                         oss_object_key = Oss.video_url_upload_oss(photoUrl, video_id)
-                        status = oss_object_key.get("status")
-                        if status != 200:
-                            return "发送OSS失败"
                         # 获取 oss 视频地址
                         oss_object_key = oss_object_key.get("oss_object_key")
+                        Common.logger().info(f'准备发送站内参数:{oss_object_key},{caption},{vx_message[3]}')
                         piaoquantv = PQ.insert_piaoquantv(oss_object_key, caption, vx_message[3])
                         if piaoquantv == False:
-                            return "发送账号失败"
-                        return
-            else:
-                return "无法获取视频ID"
-        except Exception:
-            return "URL处理失败"
+                            return "视频发送到站内失败"
+            except Exception as e:
+                Common.logger().info(f'报错信息:{e}')
+

single_video/shipinhao/shipinhao_author.py

@@ -1,39 +1,29 @@
 import os
-import json
-import random
 import sys
 import time
-import uuid
-import datetime
 
 import requests
-import cv2
-
-from common import Oss
+import json
+sys.path.append(os.getcwd())
+from common import Oss, Common
 from common.pq_utility import PQ
 from common.url_manage import urlManage
 
-sys.path.append(os.getcwd())
-
-
-
-
-import requests
-import json
-
 class shipinhaoVdieo():
 
-
     @classmethod
     def get_videoList(cls, vx_message, channel):
-        try:
-            data_link = vx_message[1]
-            data_link = json.loads(data_link)
-
-            nickname = data_link.get('nickname', '')
-            title = data_link.get('title', '')
+        data_link = vx_message[1]
+        data_link = json.loads(data_link)
+        nickname = data_link.get('nickname', '')
+        title = data_link.get('title', '')
+        Common.logger().info(f'渠道:{channel},用户名:{title}, 标题:{title}')
+        if title == '':
+            return "标题为空"
+        video_url = ''
+        for i in range(3):
+            time.sleep(1)
             url = "http://61.48.133.26:30001/SearchContentFristPageMediaInfo"
-
             payload = json.dumps({
                 "content": title,
                 "show_detail": "1"
@@ -41,36 +31,27 @@ class shipinhaoVdieo():
             headers = {
                 'Content-Type': 'application/json'
             }
-
             response = requests.request("POST", url, headers=headers, data=payload)
             data_list = response.json()
             list = data_list.get('data', [])
-            if list:
-                video_url = ''
-                sub_boxes = list[0].get('subBoxes', [])
-                for sub_boxe in sub_boxes:
-                    name_title = sub_boxe["items"][0].get('source',[])["title"]
-                    if name_title == nickname:
-                        video_url = sub_boxe["items"][0]['videoUrl']
-                if video_url == '':
-                    return "无法获取视频链接"
-                # 随机生成视频oss_id
-                video_id = urlManage.random_id()
-                oss_object_key = Oss.video_url_upload_oss(video_url, video_id)
-                status = oss_object_key.get("status")
-                if status != 200:
-                    return "发送OSS失败"
-                    # 获取 oss 视频地址
-                oss_object_key = oss_object_key.get("oss_object_key")
-                piaoquantv = PQ.insert_piaoquantv(oss_object_key, title, vx_message[3])
-                if piaoquantv == False:
-                    return "发送账号失败"
-                return
-            else:
-                return "无法获取视频链接"
-        except Exception:
-            return "URL处理失败"
-
+            sub_boxes = list[0].get('subBoxes', [])
+            for sub_boxe in sub_boxes:
+                name_title = sub_boxe["items"][0].get('source',[])["title"]
+                if name_title == nickname:
+                    video_url = sub_boxe["items"][0]['videoUrl']
+                    break
+        if video_url == '':
+            return "无法搜索到视频"
+        # 随机生成视频oss_id
+        video_id = urlManage.random_id()
+        oss_object_key = Oss.video_url_upload_oss(video_url, video_id)
+        Common.logger().info(f'上传OSS成功')
+        oss_object_key.get("status")
+        oss_object_key = oss_object_key.get("oss_object_key")
+        Common.logger().info(f'准备发送站内参数:{oss_object_key},{title},{vx_message[3]}')
+        piaoquantv = PQ.insert_piaoquantv(oss_object_key, title, vx_message[3])
+        if piaoquantv == False:
+            return "视频发送到站内失败"
 
 
 

single_video/xigua/xigua_video.py

@@ -7,6 +7,7 @@ import time
 
 import requests
 
+from common import Common
 from common.pq_utility import PQ
 
 sys.path.append(os.getcwd())
@@ -54,8 +55,14 @@ class xiguaVideo():
         try:
             data_link = vx_message[1]
             time.sleep(1)
-            content_id = urlManage.url_manage(data_link, channel)
-            if content_id:
+            content_id = ''
+            for i in range(3):
+                content_id = urlManage.url_manage(data_link, channel)
+                if content_id:
+                    break
+            if content_id == '':
+                return "无法获取到视频ID"
+            for i in range(3):
                 headers = {
                     'Cookie': 'ttwid=1%7C7j8Kxe-NNRFurQ2qk8x48WH3FToSbPDi1_YAU1IC2ws%7C1711524169%7C845d7775e8226786557408dc77df0ba8a8f7b18bdba3c37b693438dc6049ad95;',
                     'Referer': f'https://www.ixigua.com/{content_id}',
@@ -66,13 +73,13 @@ class xiguaVideo():
                 response = requests.get(url, headers=headers)
                 status_code = response.status_code
                 if status_code != 200:
-                    return "cookie过期"
+                    continue
                 response.encoding = "utf-8"
                 try:
                     html_data = response.text
                     title_match = re.search(r'<title[^>]*>(.*?)</title>', html_data)
                 except Exception:
-                    return "cookie过期"
+                    continue
                 if title_match:
                     title_content = title_match.group(1)
                     title_content = title_content.split(" - ")[0]
@@ -98,18 +105,15 @@ class xiguaVideo():
                 if status == False:
                     return "视频下载失败"
                 oss_object_key = Oss.video_sync_upload_oss(video_path_url, video_id)
-                status = oss_object_key.get("status")
-                if status != 200:
-                    return "发送OSS失败"
                 # 获取 oss 视频地址
                 oss_object_key = oss_object_key.get("oss_object_key")
+                Common.logger().info(f'准备发送站内参数:{oss_object_key},{title_content},{vx_message[3]}')
                 piaoquantv = PQ.insert_piaoquantv(oss_object_key, title_content, vx_message[3])
                 if piaoquantv == False:
-                    return "发送账号失败"
+                    return "视频发送到站内失败"
                 if os.path.isfile(video_path_url):
                     os.remove(video_path_url)
                 return
-            else:
-                return "无法获取视频ID"
-        except Exception :
-            return "URL处理失败"
+        except Exception as e:
+            Common.logger().info(f'报错信息:{e}')
+            return f"处理报错，报错信息{e}"