LongArticlesJob/account_crawler_task.py

"""
@author: luojunhui
@description: try to get some more accounts
"""

import datetime

from applications.api.feishu_api import FeishuSheetApi
from tasks.crawler_accounts_by_association import ChannelsAccountCrawler
from tasks.crawler_accounts_by_association import ToutiaoAccountCrawler
from tasks.crawler_accounts_by_association import HaoKanAccountCrawler

document_token = "BGQCsOXwHhVRq5tswjgcI8NInqd"
toutiao_sheet_id = "pIJSt7"
channels_sheet_id = "ee0163"
haokan_sheet_id = 'tfftfD'


def insert_data_into_feishu_sheet(platform: str, data_list: list[list[str]]) -> None:
    """
    insert data info into feishu sheet
    :param platform: str, channels or toutiao
    :param data_list: list[list[str]],
    """
    video_array = [
        list(i) + [datetime.date.today().strftime("%Y-%m-%d")] for i in data_list
    ]
    feishu_sheet = FeishuSheetApi()
    feishu_sheet.fetch_token()
    match platform:
        case "toutiao":
            sheet_id = toutiao_sheet_id
        case "sph":
            sheet_id = channels_sheet_id
        case 'hksp':
            sheet_id = haokan_sheet_id
        case _:
            raise RuntimeError("platform error")

    feishu_sheet.prepend_value(
        sheet_token=document_token,
        sheet_id=sheet_id,
        values=[["******"]],
        ranges="A2:A2",
    )
    feishu_sheet.insert_value(
        sheet_token=document_token,
        sheet_id=sheet_id,
        values=video_array,
        ranges="A2:J{}".format(2 + len(video_array)),
    )

def deal_each_platform(platform: str) -> None:
    """
    deal each platform
    :param platform: str, channels or toutiao
    """
    match platform:
        case "toutiao":
            crawler = ToutiaoAccountCrawler()
        case "sph":
            crawler = ChannelsAccountCrawler()
        case "hksp":
            crawler = HaoKanAccountCrawler()
        case _:
            raise RuntimeError("platform error")

    # start process
    crawler.deal()

    # get videos with score to sync to feishu
    video_list = crawler.get_video_list_with_score(platform=platform)
    if video_list:
        insert_data_into_feishu_sheet(platform=platform, data_list=video_list)
        video_id_list= [i[0] for i in video_list]
        # update status
        crawler.update_video_status(
            video_id_tuple=tuple(video_id_list), ori_status=0, new_status=1
        )



if __name__ == "__main__":
    platform_list = ["sph", "hksp", "toutiao"]
    for platform in platform_list:
        deal_each_platform(platform=platform)