LongArticlesJob/applications/utils/save_to_db.py

"""
@author: luojunhui
"""

import traceback
from applications.aliyunLogApi import log


def insert_into_single_video_source_table(db_client, video_item):
    """
    insert video into single video source table
    """
    insert_sql = f"""
        INSERT INTO publish_single_video_source
        (content_trace_id, article_title, out_account_id, out_account_name, read_cnt, like_cnt, article_url, cover_url, 
        video_oss_path, publish_timestamp, crawler_timestamp, url_unique_md5, category, tags, platform, source_account, 
        category_status, audit_status, audit_video_id, mini_program_title)
        values
        (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s);
    """
    try:
        db_client.save(
            query=insert_sql,
            params=(
                video_item["content_trace_id"],
                video_item["article_title"],
                video_item["out_account_id"],
                video_item["out_account_name"],
                video_item["read_cnt"],
                video_item["like_cnt"],
                video_item["article_url"],
                video_item["cover_url"],
                video_item["video_oss_path"],
                video_item["publish_timestamp"],
                video_item["crawler_timestamp"],
                video_item["url_unique_md5"],
                video_item["category"],
                video_item["tags"],
                video_item["platform"],
                video_item["source_account"],
                video_item["category_status"],
                video_item["audit_status"],
                video_item["audit_video_id"],
                video_item["mini_program_title"],
            ),
        )
    except Exception as e:
        log(
            task="{}_video_crawler".format(video_item["platform"]),
            function="save_each_video",
            message="save video failed",
            data={
                "error": str(e),
                "traceback": traceback.format_exc(),
                "video_id": video_item["url_unique_md5"],
                "oss_path": video_item["video_oss_path"],
            },
        )

def insert_into_video_meta_accounts_table(db_client, account_item):
    """
    insert account into account meta table
    """
    insert_sql = f"""
        insert into video_meta_accounts
            (platform, account_id, account_name, max_cursor, account_init_date, status, priority)
        values
            (%s, %s, %s, %s, %s, %s, %s);
    """
    try:
        db_client.save(
            query=insert_sql,
            params=(
                account_item["platform"],
                account_item["account_id"],
                account_item["account_name"],
                account_item["max_cursor"],
                account_item["account_init_date"],
                account_item["status"],
                account_item["priority"],
            ),
        )
    except Exception as e:
        log(
            task="{}_account_crawler".format(account_item["platform"]),
            function="save_each_account",
            message="save account failed",
            data={
                "error": str(e),
                "traceback": traceback.format_exc(),
                "account_id": account_item["account_id"],
            },
        )

def insert_into_candidate_account_pool_table(db_client, account_item):
    """
    insert recommendation into recommendation table
    """
    # check whether duplicate video
    fetch_query = f"""
        select id from crawler_candidate_account_pool
        where account_id = %s and platform = %s;
    """
    duplicate_id = db_client.fetch(
        query=fetch_query, params=(
            account_item["account_id"],
            account_item["platform"]
        )
    )
    if duplicate_id:
        print("duplicate id: {}".format(duplicate_id))
        return

    # insert into table
    insert_query = f"""
        insert into crawler_candidate_account_pool
            (account_name, account_id, title_list, platform, crawler_date)
            values
            (%s, %s, %s, %s, %s)
    """
    try:
        db_client.save(
            query=insert_query,
            params=(
               account_item["account_name"],
               account_item["account_id"],
               account_item["title_list"],
               account_item["platform"],
               account_item["crawler_date"]
            )
        )
    except Exception as e:
        log(
            task="{}_account_crawler".format(account_item["platform"]),
            function="save_each_account",
            message="save account failed",
            data={
                "error": str(e),
                "traceback": traceback.format_exc(),
                "item": account_item
            }
        )