""" @author: luojunhui @description: account crawler pipeline """ from applications.db import DatabaseConnector empty_dict = {} def whether_duplicate_account_id(account_id: str, platform: str, db_client: DatabaseConnector) -> bool: """ whether duplicate account id """ sql = f""" select id, status from video_meta_accounts where account_id = %s and platform = %s; """ fetch_response = db_client.fetch(query=sql, params=(account_id, platform)) if fetch_response: duplicate_id, status = fetch_response[0] if duplicate_id and status: return True return False else: return False def scrape_account_entities_process(account_item: dict, db_client: DatabaseConnector) -> dict: """ scrape_account_entities_process, """ account_id = account_item['account_id'] platform = account_item['platform'] # whether account exists if whether_duplicate_account_id(account_id, platform, db_client): print("duplicate account id: {}".format(account_id)) return empty_dict # account analysis return account_item