# -*- coding: utf-8 -*- # @Time: 2024/01/18 import datetime import os import random import sys import time from datetime import datetime import requests import json import urllib3 sys.path.append(os.getcwd()) from common.aliyun_oss_uploading import Oss from common.common import Common from common.material import Material from common.feishu import Feishu from common.db import MysqlHelper from requests.adapters import HTTPAdapter class kuaishouAuthor(): """ oss视频地址 存入数据库 """ @classmethod def insert_videoUrl(cls, video_id, account_id, oss_object_key, mark): current_time = datetime.now() formatted_time = current_time.strftime("%Y-%m-%d %H:%M") insert_sql = f"""INSERT INTO agc_video_url (video_id, account_id, oss_object_key, time, status, mark) values ("{video_id}", "{account_id}", "{oss_object_key}", "{formatted_time}", 1, "{mark}")""" MysqlHelper.update_values( sql=insert_sql, env="prod", machine="", ) """ 获取快手用户主页id """ @classmethod def get_kuaishou_videoUserId(cls, mark): select_user_sql = f"""select user_id, channel from agc_channel_data where mark = '{mark}' and channel = '快手' ORDER BY id DESC;""" user_list = MysqlHelper.get_values(select_user_sql, "prod") return user_list """ 查询该video_id是否在数据库存在 """ @classmethod def select_videoUrl_id(cls, video_id): select_user_sql = f"""select video_id from agc_video_url where video_id='{video_id}' ;""" user_list = MysqlHelper.get_values(select_user_sql, "prod") if user_list: return True else: return False """快手读取数据 将数据存储到oss上""" @classmethod def get_kuaishou_videoList(cls, data): try: mark = data['mark'] token = data['token'] feishu_id = data['feishu_id'] channel_id = data['channel'][0] channel = data['channel'][1] Material.insert_user(feishu_id, channel_id, mark, channel) cookie = Material.get_cookie(feishu_id, token, channel) # 获取 用户主页id user_list = cls.get_kuaishou_videoUserId(mark) if len(user_list) == 0: return for i in user_list: account_id = i[0].replace('(', '').replace(')', '').replace(',', '') Common.logger("kuaishou").info(f"用户主页ID：{account_id}") pcursor = "" count = 0 while True: if count > 5: break time.sleep(random.randint(10, 50)) url = "https://www.kuaishou.com/graphql" payload = json.dumps({ "operationName": "visionProfilePhotoList", "variables": { "userId": account_id, "pcursor": pcursor, "page": "profile" }, "query": "fragment photoContent on PhotoEntity {\n id\n duration\n caption\n originCaption\n likeCount\n viewCount\n commentCount\n realLikeCount\n coverUrl\n photoUrl\n photoH265Url\n manifest\n manifestH265\n videoResource\n coverUrls {\n url\n __typename\n }\n timestamp\n expTag\n animatedCoverUrl\n distance\n videoRatio\n liked\n stereoType\n profileUserTopPhoto\n musicBlocked\n __typename\n}\n\nfragment feedContent on Feed {\n type\n author {\n id\n name\n headerUrl\n following\n headerUrls {\n url\n __typename\n }\n __typename\n }\n photo {\n ...photoContent\n __typename\n }\n canAddComment\n llsid\n status\n currentPcursor\n tags {\n type\n name\n __typename\n }\n __typename\n}\n\nquery visionProfilePhotoList($pcursor: String, $userId: String, $page: String, $webPageArea: String) {\n visionProfilePhotoList(pcursor: $pcursor, userId: $userId, page: $page, webPageArea: $webPageArea) {\n result\n llsid\n webPageArea\n feeds {\n ...feedContent\n __typename\n }\n hostName\n pcursor\n __typename\n }\n}\n" }) headers = { 'Accept': '*/*', 'Content-Type': 'application/json', 'Origin': 'https://www.kuaishou.com', 'Cookie': cookie, 'Content-Length': '1260', 'Accept-Language': 'zh-CN,zh-Hans;q=0.9', 'Host': 'www.kuaishou.com', 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.6.1 Safari/605.1.15', 'Referer': f'https://www.kuaishou.com/profile/{account_id}', 'Accept-Encoding': 'gzip, deflate, br', 'Connection': 'keep-alive' } urllib3.disable_warnings() s = requests.session() # max_retries=3 重试3次 s.mount('http://', HTTPAdapter(max_retries=3)) s.mount('https://', HTTPAdapter(max_retries=3)) response = s.post(url=url, headers=headers, data=payload, verify=False, timeout=10) response.close() if response.status_code != 200: Common.logger("kuaishou").info( f"接口请求失败，请更换cookie，{response.status_code}") Feishu.bot('recommend', '快手', f'{mark}:快手cookie失效，请及时更换～', mark) return elif "feeds" not in response.json()["data"]["visionProfilePhotoList"]: Common.logger("kuaishou").info( f'数据为空{response.json()["data"]["visionProfilePhotoList"]}') break elif len(response.json()["data"]["visionProfilePhotoList"]["feeds"]) == 0: Common.logger("kuaishou").info( f'数据为空{response.json()["data"]["visionProfilePhotoList"]["feeds"]}') break pcursor = response.json()['data']['visionProfilePhotoList']['pcursor'] feeds = response.json()['data']['visionProfilePhotoList']['feeds'] for j in range(len(feeds)): try: try: video_id = feeds[j].get("photo", {}).get("videoResource").get("h264", {}).get("videoId", "") except KeyError: video_id = feeds[j].get("photo", {}).get("videoResource").get("hevc", {}).get("videoId", "") video_url = feeds[j].get('photo', {}).get('photoUrl', "") id = cls.select_videoUrl_id(video_id) if id: if count > 5: count += 1 Common.logger("kuaishou").info( f"重复视频不在抓取该用户，用户主页id：{account_id}") break continue channel_name = mark+'/kuaishou' oss_object_key = Oss.video_sync_upload_oss(video_url, video_id, account_id, channel_name) status = oss_object_key.get("status") # 发送 oss oss_object_key = oss_object_key.get("oss_object_key") Common.logger("kuaishou").info(f"抖音视频链接oss发送成功，oss地址：{oss_object_key}") if status == 200: cls.insert_videoUrl(video_id, account_id, oss_object_key, mark) Common.logger("kuaishou").info( f"视频地址插入数据库成功，视频id：{video_id},用户主页id：{account_id},视频储存地址：{oss_object_key}") except Exception as e: Common.logger("kuaishou").warning(f"抓取单条视频异常:{e}\n") continue except Exception as e: Common.logger("kuaishou").warning(f"抓取异常:{e}\n") return if __name__ == '__main__': kuaishouAuthor.get_kuaishou_videoList()