LongArticlesJob/account_crawler_task.py

"""
@author: luojunhui
@description: try to get some more accounts
"""

import datetime

from applications.api.feishu_api import FeishuSheetApi
from tasks.crawler_accounts_by_association import ChannelsAccountCrawler
from tasks.crawler_accounts_by_association import ToutiaoAccountCrawler
from tasks.crawler_accounts_by_association import HaoKanAccountCrawler

document_token = "BGQCsOXwHhVRq5tswjgcI8NInqd"
toutiao_sheet_id = "pIJSt7"
channels_sheet_id = "ee0163"
haokan_sheet_id = 'tfftfD'


def insert_data_into_feishu_sheet(platform: str, data_list: list[list[str]]) -> None:
    """
    insert data info into feishu sheet
    :param platform: str, channels or toutiao
    :param data_list: list[list[str]],
    """
    video_array = [
        list(i) + [datetime.date.today().strftime("%Y-%m-%d")] for i in data_list
    ]
    feishu_sheet = FeishuSheetApi()
    feishu_sheet.fetch_token()
    match platform:
        case "toutiao":
            sheet_id = toutiao_sheet_id
        case "channels":
            sheet_id = channels_sheet_id
        case 'hksp':
            sheet_id = haokan_sheet_id
        case _:
            raise RuntimeError("platform error")

    feishu_sheet.prepend_value(
        sheet_token=document_token,
        sheet_id=sheet_id,
        values=[["******"]],
        ranges="A2:A2",
    )
    feishu_sheet.insert_value(
        sheet_token=document_token,
        sheet_id=sheet_id,
        values=video_array,
        ranges="A2:I{}".format(2 + len(video_array)),
    )


if __name__ == "__main__":
    # crawler channels
    channels_account_crawler = ChannelsAccountCrawler()
    channels_account_crawler.deal()

    # insert data into toutiao sheet
    video_list = channels_account_crawler.get_video_list_with_score(platform="channels")
    insert_data_into_feishu_sheet(platform="toutiao", data_list=video_list)
    video_id_list = [i[0] for i in video_list]
    channels_account_crawler.update_video_status(
        video_id_tuple=tuple(video_id_list), ori_status=0, new_status=1
    )

    # crawler toutiao
    toutiao_account_crawler = ToutiaoAccountCrawler()
    toutiao_account_crawler.deal()

    # insert data into toutiao sheet
    video_list = toutiao_account_crawler.get_video_list_with_score(platform="toutiao")
    insert_data_into_feishu_sheet(platform="toutiao", data_list=video_list)
    video_id_list = [i[0] for i in video_list]
    toutiao_account_crawler.update_video_status(
        video_id_tuple=tuple(video_id_list), ori_status=0, new_status=1
    )

    # crawler haokanshipin
    haokan_account_crawler = HaoKanAccountCrawler()
    haokan_account_crawler.deal()
    video_list = haokan_account_crawler.get_video_list_with_score(platform="hksp")
    insert_data_into_feishu_sheet(platform="hksp", data_list=video_list)
    video_id_list = [i[0] for i in video_list]
    haokan_account_crawler.update_video_status(
        video_id_tuple=tuple(video_id_list), ori_status=0, new_status=1
    )