Merge remote-tracking branch 'origin/master'

shipinhao/shipinhao_author/shipinhao_scheduling.py

@@ -1,5 +1,6 @@
 import os
 import json
+import random
 import sys
 import datetime
 import time
@@ -152,6 +153,7 @@ class ShiPinHaoAccount:
                     "ajax": "1",
                 }
                 response = requests.request("GET", url, headers=headers, params=params)
+                time.sleep(random.randint(10, 30))
                 res_json = response.json()
                 # 开始判断视频是否有信息，是否频控
                 if res_json["base_resp"]["err_msg"] == "invalid session":
@@ -244,8 +246,8 @@ class ShiPinHaoAccount:
             "like_cnt": 0,
             "share_cnt": 0,
             "user_id": self.user_dict["uid"],
-            "cover_url": video_obj["media"][0]["cover_url"],
-            "video_url": video_obj["media"][0]["video_url"],
+            "cover_url": video_obj["media"][0]["cover_url"] if video_obj['media'][0]['cover_url'] else video_obj['media'][0]['thumb_url'],
+            "video_url": video_obj["media"][0]["url"],
             "avatar_url": video_obj["head_url"],
             "width": video_obj["media"][0]["width"],
             "height": video_obj["media"][0]["height"],

shipinhao/shipinhao_main/run_sph_author.py

@@ -1,8 +1,6 @@
-# -*- coding: utf-8 -*-
-# @Author: wangkun
-# @Time: 2023/6/30
 import argparse
-from multiprocessing import Process
+import random
+import time
 from mq_http_sdk.mq_client import *
 from mq_http_sdk.mq_consumer import *
 from mq_http_sdk.mq_exception import MQExceptionBase
@@ -95,13 +93,14 @@ def main(log_type, crawler, topic_name, group_id, env):
                     message="开始抓取"
                 )
                 for user_dict in user_list:
+                    time.sleep(random.randint(10, 20))
                     try:
                         AliyunLogger.logging(
                             code="1000",
                             platform=crawler,
                             mode=log_type,
                             env=env,
-                            message="开始抓取视频号{}".format(user_dict["name"]),
+                            message="开始抓取视频号{}".format(user_dict["link"]),
                         )
                         # 初始化
                         SPHA = ShiPinHaoAccount(
@@ -117,7 +116,7 @@ def main(log_type, crawler, topic_name, group_id, env):
                             platform=crawler,
                             mode=log_type,
                             env=env,
-                            message="完成抓取视频号{}".format(user_dict["name"]),
+                            message="完成抓取视频号{}".format(user_dict["link"]),
                         )
                     except Exception as e:
                         AliyunLogger.logging(
@@ -125,11 +124,11 @@ def main(log_type, crawler, topic_name, group_id, env):
                             platform=crawler,
                             mode=log_type,
                             env=env,
-                            message="抓取视频号{}出现问题， 报错为{}".format(user_dict["name"], e),
+                            message="抓取视频号{}出现问题， 报错为{}".format(user_dict["link"], e),
                         )
 
                 AliyunLogger.logging(
-                    code="1004", platform=crawler, mode=log_type, message="结束一轮抓取"
+                    code="1004", platform=crawler, mode=log_type, env=env,message="结束一轮抓取"
                 )
 
         except MQExceptionBase as err: