Selaa lähdekoodia

kuaishou author etl

ehlxr 2 vuotta sitten
vanhempi
commit
9b679c9595
1 muutettua tiedostoa jossa 18 lisäystä ja 6 poistoa
  1. 18 6
      kuaishou/kuaishou_author/kuaishou_author_scheduling.py

+ 18 - 6
kuaishou/kuaishou_author/kuaishou_author_scheduling.py

@@ -10,6 +10,7 @@ import requests
 import json
 import urllib3
 from requests.adapters import HTTPAdapter
+from common.mq import MQ
 sys.path.append(os.getcwd())
 from common.common import Common
 from common.feishu import Feishu
@@ -81,6 +82,7 @@ class KuaishouauthorScheduling:
     @classmethod
     def get_videoList(cls, log_type, crawler, user_dict, rule_dict, env):
         pcursor = ""
+        mq = MQ(topic_name="topic_crawler_etl_" + env)
         while True:
             url = "https://www.kuaishou.com/graphql"
             payload = json.dumps({
@@ -198,12 +200,22 @@ class KuaishouauthorScheduling:
                         Common.logger(log_type, crawler).info('视频已下载\n')
                         Common.logging(log_type, crawler, env, '视频已下载\n')
                     else:
-                        cls.download_publish(log_type=log_type,
-                                             crawler=crawler,
-                                             user_dict=user_dict,
-                                             video_dict=video_dict,
-                                             rule_dict=rule_dict,
-                                             env=env)
+                        video_dict["out_user_id"] = video_dict["user_id"]
+                        video_dict["platform"] = crawler
+                        video_dict["strategy"] = log_type
+                        video_dict["out_video_id"] = video_dict["video_id"]
+                        video_dict["width"] = video_dict["video_width"]
+                        video_dict["height"] = video_dict["video_height"]
+                        video_dict["crawler_rule"] = json.dumps(rule_dict)
+                        video_dict["user_id"] = user_dict["uid"]
+                        video_dict["publish_time"] = video_dict["publish_time_str"]
+                        mq.send_msg(video_dict)
+                        # cls.download_publish(log_type=log_type,
+                        #                      crawler=crawler,
+                        #                      user_dict=user_dict,
+                        #                      video_dict=video_dict,
+                        #                      rule_dict=rule_dict,
+                        #                      env=env)
                 except Exception as e:
                     Common.logger(log_type, crawler).warning(f"抓取单条视频异常:{e}\n")
                     Common.logging(log_type, crawler, env, f"抓取单条视频异常:{e}\n")