LongArticleTaskServer/applications/tasks/data_recycle_tasks/recycle_outside_account_articles.py

import time, json

import traceback
import urllib.parse
from tqdm.asyncio import tqdm

from .recycle_daily_publish_articles import UpdateRootSourceIdAndUpdateTimeTask
from .recycle_daily_publish_articles import Const
from applications.crawler.wechat import get_article_list_from_account
from applications.crawler.wechat import get_article_detail
from applications.pipeline import insert_outside_article_into_recycle_pool
from applications.api import feishu_robot

account_name_set = {
    "念念私语",
    "发现趣论奇闻",
    "一晴方春",
    "生活技巧悦读",
    "妙招百科享生活",
    "精选问候祝福寄语",
    "生活实用妙招收录",
    "生活妙计通",
    "逸事趣闻说",
    "天天一起跳广场舞",
    "零点生活志",
    "居家生活实录",
    "生活妙招实录",
    "生活妙招点子库",
    "生活百科巧事通",
    "早晨送你暖心祝福",
    "生活情感感悟",
    "家庭百科大全",
    "清晨早安温馨问候",
    "经典祝福语大全集",
    "日常技巧全书",
    "无忧生活锦囊",
    "家庭妙方实录",
    "生活技巧宝藏库",
    "妙招集锦全书",
    "乐享技巧馆",
    "暖心问候语录",
    "乐享技巧汇",
    "巧手理想家",
    "品质生活有妙招",
    "生活妙招万家通",
    "生活妙招大赏",
    "温暖祝福语大全",
    "好愿祝福语录",
    "精选日常祝福语",
    "邻家生活点滴",
    "创新生活妙招百科",
    "小技巧生活录",
    "妙生活家手记",
    "生活窍门事事通",
    "巧手技巧百科",
    "生活技巧小支招",
    "趣生活百科",
    "生活妙用技巧合集",
    "日常祝福语大全",
    "妙招达人养成记",
    "妙生活锦囊",
    "生活百科一点知",
    "生活妙计科普馆",
    "巧妈生活妙招助手",
    "小窍门生活录",
    "好生活点点通",
    "实用技巧馆",
    "生活情感肆读",
    "生活技巧小奥秘",
    "送温暖祝福精选",
    "生活妙招选集",
    "每日祝福语问候",
    "日常生活点子库",
    "每日好友祝福集",
    "节气祝福问候录",
    "趣味生活铺子",
    "居家生活妙招大全",
    "落日情绪屋",
    "邻家生活技巧",
    "每日精选祝福语录",
    "邻家生活有妙招",
    "妙招集合录",
    "多学生活技巧",
    "祝福问候手册",
    "点滴生活坊",
    "早安心语合集",
    "微情话语录",
    "邻家妙招知识宝典",
    "生活妙招好能手",
    "节庆祝福语大全",
    "精选早安祝福合集",
    "节日问候语大全",
    "居家生活妙招技巧",
    "精选早安问候语合集",
    "治愈情感宝典",
    "生活小妙知",
    "生活妙想指南",
    "技巧达人站",
    "幸福语录精选",
    "技巧生活手册",
    "早安祝福语集",
    "家居生活小窍门全集",
    "祝福问候大集锦",
    "乐活研究社",
    "实用生活365",
    "无忧生活小支招",
    "家有妙计百变通",
    "生活技巧智慧库",
    "点滴乐活小妙招",
    "早安问候精选祝福",
    "生活妙招巧思汇",
    "热门广场舞大全",
    "中老年妙招大全",
    "技巧百变生活馆",
    "生活巧手指南",
    "有妙招享生活",
    "巧手生活录",
    "生活技巧我知道",
    "每日早晚安祝福语录",
    "情暖祝福语录",
    "有趣生活妙招屋",
    "家居生活趣用",
    "日常生活妙招百科",
    "灵巧生活一点通",
    "拾遗情感铺",
    "广场歌舞热榜",
    "邻里妙方百科",
    "妙招技巧帮",
    "经典广场舞热榜",
    "早安祝福集大全",
    "祝福语每日送",
    "情感情报库",
    "生活妙招技巧汇",
    "实用妙招宝",
    "巧思收集录",
    "日用妙招点点通",
    "家庭技巧生活录",
    "每日早安祝福集",
    "清晨祝福合集",
    "技巧生活集萃",
    "生活妙思巧手集",
    "晨间送祝语",
    "便捷小妙招",
    "趣招一点通",
    "妙招收集馆",
    "情感慢读",
    "安心祝福集",
    "生活技巧点子库",
    "懂点技巧吧",
    "技巧能手妙招库",
    "事事妙招集锦",
    "祝福语合集",
    "技巧生活录",
    "生活妙思小帮手",
    "暖心祝福寄语录",
    "生活妙计百宝库",
    "每日祝福语选集",
    "巧手常识集锦",
    "日常祝福问候语",
    "日常问候心愿语录",
    "小窍门宝典",
    "情感阅读舍",
    "精选日常祝福",
    "生活小机智",
    "生活乐享君",
    "早上好心情祝福",
    "幸福小窍门",
    "绸缪情感铺",
    "晨间暖心语录",
    "生活妙方宝典",
    "生活妙招点通铺",
    "美好微祝福语录",
    "生活最有招",
    "祝福语温暖问候集锦",
    "节日问候温馨祝福",
    "每日幸福语录集锦",
    "生活巧招点子库",
    "生活情感课堂",
    "情感避风湾",
    "实用生活小方法",
    "什锦生活录",
    "晨间祝福语精选",
    "巧居生活妙招站",
    "巧知生活集",
    "日常妙招收集录",
    "技巧生活百知",
    "居家小妙术",
    "时光说情感",
    "好愿祝福温馨问候",
    "最美祝愿问候",
    "生活妙招乐享",
    "情绪解忧屋",
    "生活妙招空间站",
    "乐活技巧馆",
    "如意祝福选集",
    "日常技巧大讲堂",
    "三分钟技巧集",
    "情感生活百态",
    "早安问候语精选大全",
    "招招妙招百事通",
    "经典热门广场舞曲",
    "日常生活技能大全",
    "一点儿窍门馆",
    "邻家生活技巧合集",
    "每日送祝福手册",
    "早上好常用祝福问候",
    "妙招巧生活杂货铺",
    "生活全知通",
    "友友刷刷看",
    "退休也乐呵",
    "奇闻怪异集",
    "退休养老攻略",
    "竹边生活记",
    "诡秘奇闻记",
    "趣味生活简记",
    "生活墨记",
    "企退老人心声",
    "生活静语",
    "生活趣时光",
    "生活向暖",
    "老年退休小贴士",
    "退休乐时光",
    "生活趣谈会",
    "生活百科闲谈",
    "生活志记",
    "退休生活报告",
    "奇闻趣世界",
    "日常饮食百科",
    "趣享生活时光",
    "趣读奇闻汇",
    "生活点滴小栈",
    "浮光生活记",
    "生活解忧坊",
    "生活多彩时光",
    "退休生活那些事",
    "生活茶话集",
    "美好生活闲谈",
    "民间奇闻集",
    "生活百味记",
    "悠享生活指南",
    "杂谈异闻社",
    "饮食智慧屋",
    "漫谈奇闻社",
    "退休生活驿站",
    "幸福养身大全",
    "生活念记",
    "生活栖风",
    "生活沐暖",
    "生活絮事记",
    "生活集韵",
    "轻享生活记",
    "幽巷奇闻谈",
    "饮食匠心录",
    "生活畅聊集",
    "鉴赏奇闻集",
    "生活拾碎光",
    "企业退休杂谈",
    "智慧退休计划",
    "老年退休指南",
    "饮食创意坊",
    "趣闻奇谈汇",
    "生活漫读集",
    "生活趣事乐园",
    "慢品生活味",
    "企退老年乐园",
    "生活悠闲记",
    "生活饮食百科",
    "畅享饮食集",
    "幸福生活乐园",
    "退休知识宝典",
    "生活流年记",
    "生活知百味",
    "生活感悟集",
    "精致饮食指南",
    "生活知识宝典",
    "\\N",
    "沧桑时光生活",
    "自得美好生活",
    "生活暖暖舒心",
    "每日技巧集",
    "温暖问候精选",
    "百惠优生活",
    "生活爱问百科",
    "技巧便生活",
    "生活一点知",
    "精选祝福问候大全",
    "巧手生活碎片",
    "巧思知识库",
    "温暖祝福寄语",
    "智巧日常百宝库",
    "招招巧手汇",
    "早安祝福精选手册",
    "每日祝福寄语",
    "生活技能百科全知",
    "生活妙招窍门指南",
    "美好生活智慧录",
    "生活妙招小工匠",
    "幸知情感书房",
    "便捷生活通",
    "心享好生活",
    "祝福语录问候精选",
    "解忧生活铺",
    "家庭妙招站",
    "智享生活巧招",
    "早上好祝福问候心语",
    "技巧达人生活馆",
    "暖心幸福语集",
    "养身百科常谈",
    "省时省力小技巧",
    "妙招干货合集",
    "万事生活通",
    "每日精选祝福寄语",
    "祝福语正能量问候大全",
    "生活妙手多",
    "妙生活集市录",
    "生活无忧大全",
    "巧手生活技巧指南",
    "窍门百事通",
    "生活锦囊妙集",
    "技巧生活大全",
    "知著书局",
    "生活妙招百家录",
    "邻里生活妙招宝典",
    "技巧百变馆",
    "趣招技巧生活通",
    "家庭窍门实录",
    "快乐祝福语录集",
    "合家欢乐祝福问候",
    "日常家居技能",
    "好友祝福正能量语录",
    "百科巧事万事通",
    "巧手生活百科",
    "巧手生活体验馆",
    "每日祝福语早安语录",
    "情感经典说",
    "纷云说",
    "家庭妙招优选",
    "生活能手妙招",
    "居家生活实用小妙招",
    "巧生活妙招馆",
    "妙招干货铺",
    "速学妙招录",
    "科普生活小帮手",
    "祝福语热门精选",
    "美满祝福语录",
    "畅想生活招",
    "技巧百科巧招",
    "经典情感祝福语录",
    "耀舟实用妙招汇",
    "祝福心语选集",
    "节日问候语选集",
    "居家技能全书",
    "祝福贺词精选",
    "一起跳个广场舞",
    "小窍门大帮手",
    "精选问候语送祝福",
    "生活妙招收录馆",
    "家用妙招技巧集",
    "邻家生活小妙招",
    "温馨问候语集",
    "真心祝福暖心问候",
    "技巧实用生活馆",
    "实用生活妙招全录",
    "百科妙招一点通",
    "妙招百科集",
    "花好月圆吉祥祝福",
    "妙招知识通",
    "创意生活技巧集",
    "颜夕漫读",
    "美好祝福日常集锦",
    "邻里技能宝典",
    "早上好暖心祝福语录",
    "妙享生活妙招社",
    "微看情感好文",
    "节日祝福常用问候语录",
    "暖心祝愿语录",
    "乐活妙招小帮手",
    "友友过来看",
    "静好生活社",
    "漫读生活指南",
    "中老年饮食杂谈",
    "生活之百科",
    "生活微光志",
    "生活图鉴",
    "生活漫记簿",
    "奇闻秘传",
    "生活轻描记",
    "退休生活百科",
    "生活拾光机",
    "日常慧窍门",
    "奇闻汐语",
    "家庭饮食宝典",
    "分享建康知识",
    "老年健康生活",
    "异闻奇谈录",
    "江湖奇闻记",
    "烟火慢生活",
    "奇闻逸事阁",
    "饮食养身秘诀",
    "慢享生活记",
    "生活山河集",
    "养老微光录",
    "甄选生活册",
    "居家生活指南",
    "老年退休手册",
    "生活知趣多",
    "流年新生活",
    "林下思忆",
    "忆往深情",
    "生活百科小常识",
    "春耕秋实录",
    "生活技巧锦囊大全",
    "趣享生活社",
    "每日圆满祝福",
    "微读情感驿站",
    "巧手来当家",
    "祝福问候早安手册",
    "日用窍门大全",
    "365天早安祝福问候",
    "趣事说奇闻",
    "技巧百科说",
    "趣闻奇谈录",
    "生活能手宝典",
    "日日送福语",
    "小妙招锦囊",
    "幸福生活事事通",
    "生活巧思屋",
    "温情祝福合集",
    "日常家居小智慧",
    "祝福语暖心语录精选",
    "生活妙招日志",
    "生活妙招因子",
    "早晚安正能量问候大全",
    "优选早安问候语",
    "巧手生活知识馆",
    "礼貌问候祝福语",
    "日常生活小工匠",
    "百科技巧生活屋",
    "美好祝福问候语集",
    "幸福问候祝福馆",
    "早安祝福温馨问候",
    "优享生活妙招社",
    "生活集事通",
    "邻家妙招集合录",
    "情感碎语",
    "一招妙生活",
    "日常祝福语录合集",
    "明月观尘",
    "技巧百科生活屋",
    "奇妙知识生活库",
    "祝福早安每日集锦",
    "暖心问候心语",
    "生活常识技能馆",
    "每日一祝福精选",
    "小技巧百科馆",
    "生活妙招物语",
    "妙招百科全录",
    "日常窍门百事通",
    "祝福节日问候馆",
    "巧手新生活",
    "退休时光宝典",
    "快乐开心最重要",
    "生活浅酌",
    "退休日常百科",
    "记录农村日常",
    "退休常识宝典",
    "闲谈养老生活",
    "退休老年乐园",
    "精选退休大全",
    "饮食趣发现",
    "精选退休录",
    "奇闻实记",
    "退休谈生活",
    "每日建康小妙招",
    "诡夜奇闻录",
    "每日饮食推荐",
    "生活闲语",
    "天天饮食搭配",
    "镜像奇闻录",
    "饮食慢品",
    "老年饮食合集",
    "妙趣生活集",
    "生活锦记",
    "生活行语",
    "退休好岁月",
    "生活智行记",
    "饮食新风尚",
    "经典家常食谱",
    "优选生活笔记",
    "趣谈奇闻集",
    "落月情绵",
    "安然若似",
    "看生活有妙招",
    "云露华浓",
    "悦读时光书房",
    "生活百科妙招通",
    "俏生活有妙招",
    "实用生活妙招指南",
    "妙招集锦百宝箱",
    "情感避风溏",
    "岁月乐活集",
    "奇闻妙趣谈",
    "小客精选团",
    "邻家妙招巧事通",
    "趣味生活指南",
    "生活技巧百招馆",
    "小妙招大智慧",
    "幸福语录祝语",
    "实用窍门馆",
    "优生活妙招学习馆",
    "邻家妙招实用技巧",
    "技能生活小贴士",
    "静听情感语录",
    "家居技能秘籍",
    "温馨祝福语早上好",
    "精选节日祝福语录",
    "美满祝福大全",
    "家庭日用妙招",
    "祝福贴心语录",
    "老年金色岁月",
    "退休都来看",
    "快乐金秋生活",
    "人老天地宽",
    "奇闻怪谈志",
    "异度奇闻录",
    "退休悦读汇",
    "养老生活感悟",
    "退休岁月杂谈",
    "生活小窍门推荐",
    "家庭饮食百科",
    "老年生活食谱",
    "奇闻幻彩集",
    "趣说奇闻汇",
    "退休铭记",
    "退休生活小贴士",
    "感受退休生活",
    "生活微光闲语",
    "智慧妙招生活",
    "甄选生活社",
    "生活识百味",
    "奇闻趣集",
    "每日妙招共享",
    "鉴赏生活日记",
    "生活窍门50条",
    "奇事异闻录",
    "奇闻好看",
    "秘探奇闻录",
    "记录退休秘籍",
    "家庭食谱合集",
    "与心浮沉",
    "唯美微情感",
    "春月不惜",
    "心怡趣论奇闻",
    "老年生活有依",
    "别样新生活",
    "居家幸福老人",
    "暖心情感屋",
    "点滴妙招乐生活",
    "日常祝福温馨问候",
    "生活必备技巧集",
    "小妙招大用途",
    "会点生活小妙招",
    "生活百科常识库",
    "巧思巧手汇聚集",
    "深情驿站",
    "早安祝福问候精选",
    "一招一生活",
    "温馨问候祝福语录",
    "每日问候暖心语录",
    "奇闻趣谈社",
    "生活妙招技巧通",
    "无忧巧生活",
    "妙招百事帮",
    "一招一巧生活馆",
    "老年生活愉快",
    "养老生活讲堂",
    "农村知识铺",
    "居家常用秘籍",
    "品质退休生活",
    "万象奇闻录",
    "退休养老之路",
    "名厨美食推荐",
    "记录养老生活",
    "中老年退休知识",
    "实用退休知识",
    "生活辰光记",
    "极光奇闻",
    "奇闻放大镜",
    "退休生活讲堂",
    "奇事百闻录",
    "退休生活新视角",
    "生活色彩斑斓",
    "迎风好生活",
    "闲庭信步生活",
    "生活其乐融融",
    "雾里云淡",
    "雨凉思情",
    "矜柔人生",
    "生活智慧妙招姐",
    "圆满祝福精选",
    "温馨问候语集锦",
    "实用生活技能合集",
    "生活趣事笔记",
    "生活解语",
    "奇谈趣闻社",
    "奇人见闻",
    "退休生活好帮手",
    "奇闻集锦",
    "退休老年社群",
    "奇闻故事栈",
    "企退生活日记",
    "分享生活小常识",
    "中老年美食圈",
    "退休生活常识",
    "精选退休美文",
    "休闲养老生活",
    "养老退休指南",
    "安享退休事",
    "悠闲退休时光",
    "夜谈奇闻志",
    "南山遗梦",
    "奇特人生",
    "奇闻集合",
}


class RecycleOutsideAccountArticlesTask(Const):
    def __init__(self, pool, log_client, date_string):
        self.pool = pool
        self.log_client = log_client
        self.date_string = date_string

    async def get_outside_accounts(self):
        query = """
            select
                t2.group_source_name as account_source,
                t3.name as name,
                t3.gh_id as gh_id
            from wx_statistics_group_source t1
                join wx_statistics_group_source_account t2 on t2.group_source_name = t1.account_source_name
                join publish_account t3 on t3.id = t2.account_id
            where
                t1.mode_type = '代运营服务号' and
                (
                    t2.group_source_name like '%云誉%'
                        or t2.group_source_name like '%微小盟%'
                        or t2.group_source_name like '%阿雅达%'
                        or t2.group_source_name like '%创易%'
                    )
            and t3.status = 1 and t3.name != '';
        """
        return await self.pool.async_fetch(query=query, db_name="aigc")

    async def recycle_single_account(self, account):
        """recycle single account"""
        if account["name"] not in account_name_set:
            return

        query = """
                select max(update_time) as publish_timestamp \
                from outside_account_articles
                where gh_id = %s;
                """
        response = await self.pool.async_fetch(query=query, params=(account["gh_id"],))
        if response:
            max_publish_timestamp = response[0]["publish_timestamp"]
        else:
            max_publish_timestamp = int(time.time()) - self.NEW_ACCOUNT_CRAWL_PERIOD

        cursor = None
        while True:
            response = await get_article_list_from_account(
                account_id=account["gh_id"], index=cursor
            )
            response_code = response["code"]
            match response_code:
                case self.ACCOUNT_FORBIDDEN_CODE:
                    # await feishu_robot.bot(
                    #     title="发布账号封禁",
                    #     detail={
                    #         "账号名称": account["name"],
                    #         "账号id": account["gh_id"],
                    #     },
                    # )
                    return
                case self.ARTICLE_SUCCESS_CODE:
                    msg_list = response.get("data", {}).get("data", [])
                    if not msg_list:
                        return

                    await insert_outside_article_into_recycle_pool(
                        self.pool, self.log_client, msg_list, account
                    )

                    # check last article
                    last_article = msg_list[-1]
                    last_publish_timestamp = last_article["AppMsg"]["BaseInfo"][
                        "UpdateTime"
                    ]
                    if last_publish_timestamp <= max_publish_timestamp:
                        return

                    cursor = response["data"].get("next_cursor")
                    if not cursor:
                        return
                case self.CRAWL_CRASH_CODE:
                    await self.log_client.log(
                        contents={
                            "task": "recycle_daily_publish_articles",
                            "data": {
                                "gh_id": account["gh_id"],
                            },
                            "message": "爬虫挂掉",
                            "status": "fail",
                        }
                    )
                case _:
                    return

    async def deal(self):
        subscription_accounts = await self.get_outside_accounts()
        for account in tqdm(subscription_accounts, desc="recycle each account"):
            try:
                await self.recycle_single_account(account)

            except Exception as e:
                print(
                    f"{account['name']}\t{account['gh_id']}: recycle account error:", e
                )



class UpdateOutsideRootSourceIdAndUpdateTimeTask(UpdateRootSourceIdAndUpdateTimeTask):
    def __init__(self, pool, log_client):
        super().__init__(pool, log_client)

    async def get_outside_article_list_v2(self) -> list[dict]:
        query = """
                select content_url, wx_sn 
                from outside_account_articles where publish_timestamp in %s
                    and account_name in %s
                order by update_time desc;
            """
        article_list = await self.pool.async_fetch(
            query=query, params=(tuple([0, -1, -3]), tuple(account_name_set))
        )
        return article_list

    async def check_each_article(self, article: dict):
        url = article["content_url"]
        wx_sn = article["wx_sn"]
        try:
            response = await get_article_detail(url)
            response_code = response["code"]

            if response_code == self.ARTICLE_DELETE_CODE:
                publish_timestamp_s = self.DELETE_STATUS
                root_source_id_list = []
            elif response_code == self.ARTICLE_ILLEGAL_CODE:
                publish_timestamp_s = self.ILLEGAL_STATUS
                root_source_id_list = []
            elif response_code == self.ARTICLE_SUCCESS_CODE:
                data = response["data"]["data"]
                publish_timestamp_ms = data["publish_timestamp"]
                publish_timestamp_s = int(publish_timestamp_ms / 1000)
                mini_program = data.get("mini_program", [])
                if mini_program:
                    root_source_id_list = [
                        urllib.parse.parse_qs(urllib.parse.unquote(i["path"])).get(
                            "rootSourceId", [""]
                        )[0]
                        for i in mini_program
                    ]
                else:
                    root_source_id_list = []
            else:
                publish_timestamp_s = self.UNKNOWN_STATUS
                root_source_id_list = []
        except Exception as e:
            publish_timestamp_s = self.REQUEST_FAIL_STATUS
            root_source_id_list = None
            error_msg = traceback.format_exc()
            await self.log_client.log(
                contents={
                    "task": "get_official_article_detail",
                    "data": {
                        "url": url,
                        "wx_sn": wx_sn,
                        "error_msg": error_msg,
                        "error": str(e),
                    },
                    "function": "check_each_article",
                    "status": "fail",
                }
            )
        query = """
            update outside_account_articles set publish_timestamp = %s, root_source_id_list = %s
            where wx_sn = %s;
        """
        await self.pool.async_save(
            query=query,
            params=(
                publish_timestamp_s,
                json.dumps(root_source_id_list, ensure_ascii=False),
                wx_sn,
            ),
        )
        if publish_timestamp_s == self.REQUEST_FAIL_STATUS:
            article["wx_sn"] = wx_sn
            return article
        else:
            return None

    async def fallback_mechanism(self):
        # 若还是无 publish_timestamp，用update_time当作 publish_timestamp
        update_sql_2 = f"""
            update outside_account_articles
            set publish_timestamp = update_time
            where publish_timestamp < %s;
        """
        affected_rows_2 = await self.pool.async_save(query=update_sql_2, params=(0,))
        if affected_rows_2:
            await feishu_robot.bot(
                title="执行兜底修改发布时间戳",
                detail={
                    # "通过msgId修改": affected_rows_1,
                    "通过create_timestamp修改": affected_rows_2,
                },
                mention=False,
            )

    async def deal(self):
        task_list = await self.get_outside_article_list_v2()
        for task in tqdm(task_list, desc="get article detail step1: "):
            try:
                await self.check_each_article(task)
            except Exception as e:
                try:
                    await self.log_client.log(
                        contents={
                            "task": "get_official_article_detail_step1",
                            "data": {
                                "detail": {
                                    "url": task["ContentUrl"],
                                    "wx_sn": task["wx_sn"],
                                },
                                "error_msg": traceback.format_exc(),
                                "error": str(e),
                            },
                            "function": "check_each_article",
                            "status": "fail",
                        }
                    )
                except Exception as e:
                    print(e)
                    print(traceback.format_exc())