piaoquan_crawler/xiaoniangao/xiaoniangao_author/xiaoniangao_author_test.py

import json
import os
import random
import sys
import time
import uuid
import requests

from common.mq import MQ

sys.path.append(os.getcwd())
from common.pipeline import PiaoQuanPipelineTest
from common.public import get_config_from_mysql, clean_title
from common.scheduling_db import MysqlHelper


def tunnel_proxies():
    # 隧道域名:端口号
    tunnel = "q796.kdltps.com:15818"

    # 用户名密码方式
    username = "t17772369458618"
    password = "5zqcjkmy"
    tunnel_proxies = {
        "http": "http://%(user)s:%(pwd)s@%(proxy)s/"
                % {"user": username, "pwd": password, "proxy": tunnel},
        "https": "http://%(user)s:%(pwd)s@%(proxy)s/"
                 % {"user": username, "pwd": password, "proxy": tunnel},
    }

    return tunnel_proxies


class XiaoNianGaoAuthor:
    def __init__(self, platform, mode, rule_dict, env, user_list):
        self.platform = platform
        self.mode = mode
        self.rule_dict = rule_dict
        self.env = env
        self.user_list = user_list
        self.mq = MQ(topic_name="topic_crawler_etl_" + self.env)
        self.download_count = 0

    def get_author_list(self):
        # 每轮只抓取定量的数据，到达数量后自己退出
        max_count = int(self.rule_dict.get("videos_cnt", {}).get("min", 300))
        for user_dict in self.user_list:
            print(user_dict)
            account_level = user_dict['account_level']
            if account_level and account_level != "P3":
                if self.download_count <= max_count:
                    self.get_video_list(user_dict)
                    time.sleep(random.randint(1, 15))
                else:
                    message = "本轮已经抓取足够数量的视频，已经自动退出"
                    print(message)
                    return

    def get_video_list(self, user_dict):
        next_t = -1
        # 只抓取更新的视频，如果刷到已经更新的立即退出
        url = "https://kapi-xng-app.xiaoniangao.cn/v1/album/user_public"
        headers = {
            "Host": "kapi-xng-app.xiaoniangao.cn",
            "content-type": "application/json; charset=utf-8",
            "accept": "*/*",
            "verb": "POST",
            "accept-language": "zh-cn",
            "date": "Wed, 01 Nov 2023 11:53:22 GMT",
            "x-token-id": "",
            "x-signaturemethod": "hmac-sha1",
        }
        while True:
            payload = {
                "token": "",
                "limit": 20,
                "start_t": next_t,
                "visited_mid": int(user_dict["link"]),
                "share_width": 300,
                "share_height": 240,
            }
            response = requests.request(
                "POST",
                url,
                headers=headers,
                data=json.dumps(payload),
                proxies=tunnel_proxies(),
            )
            if "data" not in response.text or response.status_code != 200:
                message = f"get_videoList:{response.text}"
                print(message)
                return
            elif "list" not in response.json()["data"]:
                message = f"get_videoList:{response.json()}"
                print(message)
                return
            elif len(response.json()["data"]["list"]) == 0:
                message = f"没有更多数据啦~"
                print(message)
                return
            else:
                next_t = response.json()["data"]["next_t"]
                feeds = response.json()["data"]["list"]
                for video_obj in feeds:
                    try:
                        message = f"扫描到一条视频"
                        print(message)
                        flag = self.process_video_obj(video_obj, user_dict)
                        if not flag:
                            return
                    except Exception as e:
                        message = "抓取单条视频异常, 报错原因是: {}".format(e)
                        print(message)

    def process_video_obj(self, video_obj, user_dict):
        trace_id = self.platform + str(uuid.uuid1())
        # 标题，表情随机加在片头、片尾，或替代句子中间的标点符号
        xiaoniangao_title = clean_title(video_obj.get("title", ""))
        # 随机取一个表情/符号
        emoji = random.choice(
            get_config_from_mysql(self.mode, self.platform, self.env, "emoji")
        )
        # 生成最终标题，标题list[表情+title, title+表情]随机取一个
        video_title = random.choice(
            [f"{emoji}{xiaoniangao_title}", f"{xiaoniangao_title}{emoji}"]
        )
        # 发布时间
        publish_time_stamp = int(int(video_obj.get("t", 0)) / 1000)
        publish_time_str = time.strftime(
            "%Y-%m-%d %H:%M:%S", time.localtime(publish_time_stamp)
        )
        # 用户名 / 头像
        user_name = (
            video_obj.get("user", {})
            .get("nick", "")
            .strip()
            .replace("\n", "")
            .replace("/", "")
            .replace(" ", "")
            .replace(" ", "")
            .replace("&NBSP", "")
            .replace("\r", "")
        )
        video_dict = {
            "video_title": video_title,
            "video_id": video_obj.get("vid", ""),
            "duration": int(video_obj.get("du", 0) / 1000),
            "play_cnt": video_obj.get("play_pv", 0),
            "like_cnt": video_obj.get("favor", {}).get("total", 0),
            "comment_cnt": video_obj.get("comment_count", 0),
            "share_cnt": video_obj.get("share", 0),
            "user_name": user_name,
            "publish_time_stamp": publish_time_stamp,
            "publish_time_str": publish_time_str,
            "update_time_stamp": int(time.time()),
            "video_width": int(video_obj.get("w", 0)),
            "video_height": int(video_obj.get("h", 0)),
            "avatar_url": video_obj.get("user", {}).get("hurl", ""),
            "profile_id": video_obj["id"],
            "profile_mid": video_obj.get("user", {}).get("mid", ""),
            "cover_url": video_obj.get("url", ""),
            "video_url": video_obj.get("v_url", ""),
            "session": f"xiaoniangao-author-{int(time.time())}",
            "out_user_id": video_obj["id"],
            "platform": self.platform,
            "strategy": self.mode,
            "out_video_id": video_obj.get("vid", ""),
        }
        if (
                int(time.time()) - publish_time_stamp
                > 3600 * 24 * int(self.rule_dict.get("period", {}).get("max", 1000))
        ):
            message = "发布时间超过{}天".format(
                int(self.rule_dict.get("period", {}).get("max", 1000))
            )
            print(message)
            return False
        pipeline = PiaoQuanPipelineTest(
            platform=self.platform,
            mode=self.mode,
            rule_dict=self.rule_dict,
            env=self.env,
            item=video_dict,
            trace_id=trace_id,
        )
        account_level = user_dict['account_level']
        if account_level == "P0" or account_level == "P1":
            flag = True
        else:
            flag = pipeline.process_item()
        # flag = pipeline.process_item()
        if flag:
            video_dict["width"] = video_dict["video_width"]
            video_dict["height"] = video_dict["video_height"]
            video_dict["crawler_rule"] = json.dumps(self.rule_dict)
            video_dict["user_id"] = user_dict["uid"]
            video_dict["publish_time"] = video_dict["publish_time_str"]
            print(video_dict)
            # self.mq.send_msg(video_dict)
            self.download_count += 1
            message = "成功发送 MQ 至 ETL"
            print(message)
        return True


if __name__ == "__main__":
    select_user_sql = (
        f"""select * from crawler_user_v3 where task_id=21"""
    )
    user_list = MysqlHelper.get_values(
        "author", "xiaoniangao", select_user_sql, "prod", ""
    )
    XNGA = XiaoNianGaoAuthor(
        platform="xiaoniangao",
        mode="author",
        rule_dict={},
        env="prod",
        user_list=user_list
    )
    XNGA.get_author_list()