LongArticlesJob/applications/pipeline/account_pipeline.py

"""
@author: luojunhui
@description: account crawler pipeline
"""
from applications.db import DatabaseConnector

empty_dict = {}

def whether_duplicate_account_id(account_id: str, platform: str, db_client: DatabaseConnector) -> bool:
    """
    whether duplicate account id
    """
    sql = f"""
        select id, status from video_meta_accounts
        where account_id = %s and platform = %s;
    """
    duplicate_id, status = db_client.fetch(query=sql, params=(account_id, platform))[0]
    if duplicate_id and status:
        return True
    return False

def scrape_account_entities_process(account_item: dict, db_client: DatabaseConnector) -> dict:
    """
    scrape_account_entities_process,
    """
    account_id = account_item['account_id']
    platform = account_item['platform']

    # whether account exists
    if whether_duplicate_account_id(account_id, platform, db_client):
        print("duplicate account id: {}".format(account_id))
        return empty_dict

    # account analysis

    return account_item