video_editor_agc/extract_data/kuaishou/kuaishou_author.py

# -*- coding: utf-8 -*-
# @Time: 2024/01/18
import datetime
import os
import random
import sys
import time
from datetime import datetime
import requests
import json
import urllib3
sys.path.append(os.getcwd())
from common.aliyun_oss_uploading import Oss
from common.common import Common
from common.material import Material
from common.feishu import Feishu
from common.db import MysqlHelper
from requests.adapters import HTTPAdapter


class kuaishouAuthor():

    """
    oss视频地址 存入数据库
    """
    @classmethod
    def insert_videoUrl(cls, video_id, account_id, oss_object_key, mark):
        current_time = datetime.now()
        formatted_time = current_time.strftime("%Y-%m-%d %H:%M")
        insert_sql = f"""INSERT INTO agc_video_url (video_id, account_id, oss_object_key, time, status, mark) values ("{video_id}", "{account_id}", "{oss_object_key}", "{formatted_time}", 1, "{mark}")"""
        MysqlHelper.update_values(
            sql=insert_sql,
            env="prod",
            machine="",
        )

    """
    获取快手用户主页id
    """
    @classmethod
    def get_kuaishou_videoUserId(cls, mark):
        select_user_sql = f"""select user_id, channel from agc_channel_data where mark = '{mark}' and  channel  = '快手' ORDER BY id DESC;"""
        user_list = MysqlHelper.get_values(select_user_sql, "prod")
        return user_list

    """
    查询该video_id是否在数据库存在
    """
    @classmethod
    def select_videoUrl_id(cls, video_id):
        select_user_sql = f"""select video_id from agc_video_url where video_id='{video_id}' ;"""
        user_list = MysqlHelper.get_values(select_user_sql, "prod")
        if user_list:
            return True
        else:
            return False

    """快手读取数据 将数据存储到oss上"""
    @classmethod
    def get_kuaishou_videoList(cls, data):
        try:
            mark = data['mark']
            token = data['token']
            feishu_id = data['feishu_id']
            channel_id = data['channel'][0]
            channel = data['channel'][1]
            Material.insert_user(feishu_id, channel_id, mark, channel)
            cookie = Material.get_cookie(feishu_id, token, channel)
            # 获取 用户主页id
            user_list = cls.get_kuaishou_videoUserId(mark)
            if len(user_list) == 0:
                return
            for i in user_list:
                account_id = i[0].replace('(', '').replace(')', '').replace(',', '')
                Common.logger("kuaishou").info(f"用户主页ID：{account_id}")
                pcursor = ""
                count = 0
                while True:
                    if count > 5:
                        break
                    time.sleep(random.randint(10, 50))
                    url = "https://www.kuaishou.com/graphql"
                    payload = json.dumps({
                        "operationName": "visionProfilePhotoList",
                        "variables": {
                            "userId": account_id,
                            "pcursor": pcursor,
                            "page": "profile"
                        },
                        "query": "fragment photoContent on PhotoEntity {\n  id\n  duration\n  caption\n  originCaption\n  likeCount\n  viewCount\n  commentCount\n  realLikeCount\n  coverUrl\n  photoUrl\n  photoH265Url\n  manifest\n  manifestH265\n  videoResource\n  coverUrls {\n    url\n    __typename\n  }\n  timestamp\n  expTag\n  animatedCoverUrl\n  distance\n  videoRatio\n  liked\n  stereoType\n  profileUserTopPhoto\n  musicBlocked\n  __typename\n}\n\nfragment feedContent on Feed {\n  type\n  author {\n    id\n    name\n    headerUrl\n    following\n    headerUrls {\n      url\n      __typename\n    }\n    __typename\n  }\n  photo {\n    ...photoContent\n    __typename\n  }\n  canAddComment\n  llsid\n  status\n  currentPcursor\n  tags {\n    type\n    name\n    __typename\n  }\n  __typename\n}\n\nquery visionProfilePhotoList($pcursor: String, $userId: String, $page: String, $webPageArea: String) {\n  visionProfilePhotoList(pcursor: $pcursor, userId: $userId, page: $page, webPageArea: $webPageArea) {\n    result\n    llsid\n    webPageArea\n    feeds {\n      ...feedContent\n      __typename\n    }\n    hostName\n    pcursor\n    __typename\n  }\n}\n"
                    })
                    headers = {
                        'Accept': '*/*',
                        'Content-Type': 'application/json',
                        'Origin': 'https://www.kuaishou.com',
                        'Cookie': cookie,
                        'Content-Length': '1260',
                        'Accept-Language': 'zh-CN,zh-Hans;q=0.9',
                        'Host': 'www.kuaishou.com',
                        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.6.1 Safari/605.1.15',
                        'Referer': f'https://www.kuaishou.com/profile/{account_id}',
                        'Accept-Encoding': 'gzip, deflate, br',
                        'Connection': 'keep-alive'
                    }
                    urllib3.disable_warnings()
                    s = requests.session()
                    # max_retries=3 重试3次
                    s.mount('http://', HTTPAdapter(max_retries=3))
                    s.mount('https://', HTTPAdapter(max_retries=3))
                    response = s.post(url=url, headers=headers, data=payload, verify=False,
                                      timeout=10)
                    response.close()
                    if response.status_code != 200:
                        Common.logger("kuaishou").info(
                            f"接口请求失败，请更换cookie，{response.status_code}")
                        Feishu.bot('recommend', '快手', f'{mark}:快手cookie失效，请及时更换～', mark)
                        return
                    elif "feeds" not in response.json()["data"]["visionProfilePhotoList"]:
                        Common.logger("kuaishou").info(
                            f'数据为空{response.json()["data"]["visionProfilePhotoList"]}')
                        break
                    elif len(response.json()["data"]["visionProfilePhotoList"]["feeds"]) == 0:
                        Common.logger("kuaishou").info(
                            f'数据为空{response.json()["data"]["visionProfilePhotoList"]["feeds"]}')
                        break
                    pcursor = response.json()['data']['visionProfilePhotoList']['pcursor']
                    feeds = response.json()['data']['visionProfilePhotoList']['feeds']
                    for j in range(len(feeds)):
                        try:
                            try:
                                video_id = feeds[j].get("photo", {}).get("videoResource").get("h264", {}).get("videoId", "")
                            except KeyError:
                                video_id = feeds[j].get("photo", {}).get("videoResource").get("hevc", {}).get("videoId", "")
                            video_url = feeds[j].get('photo', {}).get('photoUrl', "")

                            id = cls.select_videoUrl_id(video_id)
                            if id:
                                if count > 5:
                                    count += 1
                                    Common.logger("kuaishou").info(
                                        f"重复视频不在抓取该用户，用户主页id：{account_id}")
                                    break
                                continue
                            channel_name = mark+'/kuaishou'
                            oss_object_key = Oss.video_sync_upload_oss(video_url, video_id, account_id, channel_name)
                            status = oss_object_key.get("status")
                            # 发送 oss
                            oss_object_key = oss_object_key.get("oss_object_key")
                            Common.logger("kuaishou").info(f"抖音视频链接oss发送成功，oss地址：{oss_object_key}")
                            if status == 200:
                                cls.insert_videoUrl(video_id, account_id, oss_object_key, mark)
                                Common.logger("kuaishou").info(
                                    f"视频地址插入数据库成功，视频id：{video_id},用户主页id：{account_id},视频储存地址：{oss_object_key}")
                        except Exception as e:
                            Common.logger("kuaishou").warning(f"抓取单条视频异常:{e}\n")
                            continue
        except Exception as e:
            Common.logger("kuaishou").warning(f"抓取异常:{e}\n")
            return

if __name__ == '__main__':
    kuaishouAuthor.get_kuaishou_videoList()