add platform

account_crawler_task.py

@@ -50,45 +50,40 @@ def insert_data_into_feishu_sheet(platform: str, data_list: list[list[str]]) ->
         ranges="A2:J{}".format(2 + len(video_array)),
     )
 
+def deal_each_platform(platform: str) -> None:
+    """
+    deal each platform
+    :param platform: str, channels or toutiao
+    """
+    match platform:
+        case "toutiao":
+            crawler = ToutiaoAccountCrawler()
+        case "sph":
+            crawler = ChannelsAccountCrawler()
+        case "hksp":
+            crawler = HaoKanAccountCrawler()
+        case _:
+            raise RuntimeError("platform error")
 
-if __name__ == "__main__":
-    # crawler channels
-    channels_account_crawler = ChannelsAccountCrawler()
-    channels_account_crawler.deal()
-
-    # insert data into sph sheet
-    video_list = channels_account_crawler.get_video_list_with_score(platform="sph")
-    insert_data_into_feishu_sheet(platform="sph", data_list=video_list)
-    channel_video_id_list = [i[0] for i in video_list]
-    if channel_video_id_list:
-        channels_account_crawler.update_video_status(
-            video_id_tuple=tuple(channel_video_id_list), ori_status=0, new_status=1
+    # start process
+    crawler.deal()
+
+    # get videos with score to sync to feishu
+    video_list = crawler.get_video_list_with_score(platform=platform)
+    if video_list:
+        insert_data_into_feishu_sheet(platform=platform, data_list=video_list)
+        video_id_list= [i[0] for i in video_list]
+        # update status
+        crawler.update_video_status(
+            video_id_tuple=tuple(video_id_list), ori_status=0, new_status=1
         )
 
-    # crawler toutiao
-    toutiao_account_crawler = ToutiaoAccountCrawler()
-    toutiao_account_crawler.deal()
 
-    # insert data into toutiao sheet
-    video_list = toutiao_account_crawler.get_video_list_with_score(platform="toutiao")
-    insert_data_into_feishu_sheet(platform="toutiao", data_list=video_list)
-    toutiao_video_id_list = [i[0] for i in video_list]
-    if toutiao_video_id_list:
-        toutiao_account_crawler.update_video_status(
-            video_id_tuple=tuple(toutiao_video_id_list), ori_status=0, new_status=1
-        )
-
-    # crawler haokanshipin
-    haokan_account_crawler = HaoKanAccountCrawler()
-    haokan_account_crawler.deal()
-    video_list = haokan_account_crawler.get_video_list_with_score(platform="hksp")
-    insert_data_into_feishu_sheet(platform="hksp", data_list=video_list)
-    haokan_video_id_list = [i[0] for i in video_list]
-    if haokan_video_id_list:
-        haokan_account_crawler.update_video_status(
-            video_id_tuple=tuple(haokan_video_id_list), ori_status=0, new_status=1
-        )
 
+if __name__ == "__main__":
+    platform_list = ["sph", "hksp", "toutiao"]
+    for platform in platform_list:
+        deal_each_platform(platform=platform)
 
 
 

tasks/crawler_accounts_by_association.py

@@ -203,7 +203,7 @@ class ChannelsAccountCrawler(CrawlerAccounts):
                 self.process_search_response(video, seed_title=title)
             except Exception as e:
                 log(
-                    task="channels account crawler",
+                    task="crawler_channels_account_videos",
                     function="process_search_response",
                     message="search by title failed",
                     data={
@@ -220,7 +220,7 @@ class ChannelsAccountCrawler(CrawlerAccounts):
                 self.search_video_in_channels(title=item["title"])
             except Exception as e:
                 log(
-                    task="channels account crawler",
+                    task="crawler_channels_account_videos",
                     function="search_video_in_channels",
                     message="search video in channels failed",
                     data={