""" @author: luojunhui @description: account crawler pipeline """ from applications.db import DatabaseConnector empty_dict = {} def whether_duplicate_account_id(account_id: str, platform: str, db_client: DatabaseConnector) -> bool: """ whether duplicate account id """ sql = f""" select id from video_meta_accounts where account_id = %s and platform = %s; """ duplicate_id = db_client.fetch(query=sql, params=(account_id, platform)) if duplicate_id: return True return False def scrape_account_entities_process(account_item: dict, db_client: DatabaseConnector) -> dict: """ scrape_account_entities_process, """ account_id = account_item['account_id'] platform = account_item['platform'] # whether account exists if whether_duplicate_account_id(account_id, platform, db_client): return empty_dict # account analysis return account_item