|
|
@@ -0,0 +1,531 @@
|
|
|
+# -*- coding: utf-8 -*-
|
|
|
+# @Author: wangkun
|
|
|
+# @Time: 2023/2/24
|
|
|
+import os
|
|
|
+import random
|
|
|
+import shutil
|
|
|
+import sys
|
|
|
+import time
|
|
|
+from hashlib import md5
|
|
|
+
|
|
|
+import requests
|
|
|
+import json
|
|
|
+
|
|
|
+import urllib3
|
|
|
+from requests.adapters import HTTPAdapter
|
|
|
+
|
|
|
+sys.path.append(os.getcwd())
|
|
|
+from common.common import Common
|
|
|
+from common.feishu import Feishu
|
|
|
+from common.getuser import getUser
|
|
|
+from common.db import MysqlHelper
|
|
|
+from common.publish import Publish
|
|
|
+from common.public import get_user_from_mysql
|
|
|
+from common.userAgent import get_random_user_agent
|
|
|
+
|
|
|
+
|
|
|
+class KuaiShouRecommend:
|
|
|
+ platform = "快手"
|
|
|
+ tag = "快手爬虫,推荐爬虫策略"
|
|
|
+
|
|
|
+ @classmethod
|
|
|
+ def get_rule(cls, log_type, crawler):
|
|
|
+ try:
|
|
|
+ rule_sheet = Feishu.get_values_batch(log_type, crawler, "NQ6CZN")
|
|
|
+ rule_dict = {
|
|
|
+ # "play_cnt": int(rule_sheet[0][2]),
|
|
|
+ # "comment_cnt": int(rule_sheet[1][2]),
|
|
|
+ "video_width": int(rule_sheet[0][2]),
|
|
|
+ "video_height": int(rule_sheet[1][2]),
|
|
|
+ "like_cnt": int(rule_sheet[2][2]),
|
|
|
+ "duration": int(rule_sheet[3][2]),
|
|
|
+ "publish_time": int(rule_sheet[4][2]),
|
|
|
+
|
|
|
+ }
|
|
|
+ return rule_dict
|
|
|
+ except Exception as e:
|
|
|
+ Common.logger(log_type, crawler).error(f"get_rule:{e}\n")
|
|
|
+
|
|
|
+ @classmethod
|
|
|
+ def download_rule(cls, video_dict, rule_dict):
|
|
|
+ if eval(f"{video_dict['video_width']}{rule_dict['video_width']}") is True \
|
|
|
+ and eval(f"{video_dict['video_height']}{rule_dict['video_height']}") is True \
|
|
|
+ and eval(f"{video_dict['like_cnt']}{rule_dict['like_cnt']}") is True \
|
|
|
+ and eval(f"{video_dict['duration']}{rule_dict['duration']}") is True \
|
|
|
+ and eval(f"{video_dict['publish_time']}{rule_dict['publish_time']}") is True:
|
|
|
+ return True
|
|
|
+ else:
|
|
|
+ return False
|
|
|
+
|
|
|
+ # 过滤词库
|
|
|
+ @classmethod
|
|
|
+ def filter_words(cls, log_type, crawler):
|
|
|
+ try:
|
|
|
+ while True:
|
|
|
+ filter_words_sheet = Feishu.get_values_batch(log_type, crawler, 'HIKVvs')
|
|
|
+ if filter_words_sheet is None:
|
|
|
+ Common.logger(log_type, crawler).warning(f"filter_words_sheet:{filter_words_sheet} 10秒钟后重试")
|
|
|
+ continue
|
|
|
+ filter_words_list = []
|
|
|
+ for x in filter_words_sheet:
|
|
|
+ for y in x:
|
|
|
+ if y is None:
|
|
|
+ pass
|
|
|
+ else:
|
|
|
+ filter_words_list.append(y)
|
|
|
+ return filter_words_list
|
|
|
+ except Exception as e:
|
|
|
+ Common.logger(log_type, crawler).error(f'filter_words异常:{e}\n')
|
|
|
+
|
|
|
+ # 万能标题
|
|
|
+ @classmethod
|
|
|
+ def random_title(cls, log_type, crawler):
|
|
|
+ try:
|
|
|
+ while True:
|
|
|
+ random_title_sheet = Feishu.get_values_batch(log_type, crawler, '0DiyXe')
|
|
|
+ if random_title_sheet is None:
|
|
|
+ Common.logger(log_type, crawler).warning(f"filter_words_sheet:{random_title_sheet} 10秒钟后重试")
|
|
|
+ continue
|
|
|
+ random_title_list = []
|
|
|
+ for x in random_title_sheet:
|
|
|
+ for y in x:
|
|
|
+ if y is None:
|
|
|
+ pass
|
|
|
+ else:
|
|
|
+ random_title_list.append(y)
|
|
|
+ return random.choice(random_title_list)
|
|
|
+ except Exception as e:
|
|
|
+ Common.logger(log_type, crawler).error(f'random_title:{e}\n')
|
|
|
+
|
|
|
+ # 获取用户信息列表
|
|
|
+ @classmethod
|
|
|
+ def get_user_list(cls, log_type, crawler, sheetid, env, machine):
|
|
|
+ try:
|
|
|
+ while True:
|
|
|
+ user_sheet = Feishu.get_values_batch(log_type, crawler, sheetid)
|
|
|
+ if user_sheet is None:
|
|
|
+ Common.logger(log_type, crawler).warning(f"user_sheet:{user_sheet} 10秒钟后重试")
|
|
|
+ continue
|
|
|
+ our_user_list = []
|
|
|
+ for i in range(1, len(user_sheet)):
|
|
|
+ # for i in range(1, 2):
|
|
|
+ out_uid = user_sheet[i][2]
|
|
|
+ user_name = user_sheet[i][3]
|
|
|
+ our_uid = user_sheet[i][6]
|
|
|
+ our_user_link = user_sheet[i][7]
|
|
|
+ if out_uid is None or user_name is None:
|
|
|
+ Common.logger(log_type, crawler).info("空行\n")
|
|
|
+ else:
|
|
|
+ Common.logger(log_type, crawler).info(f"正在更新 {user_name} 用户信息\n")
|
|
|
+ if our_uid is None:
|
|
|
+ out_user_info = cls.get_out_user_info(log_type, crawler, out_uid)
|
|
|
+ out_user_dict = {
|
|
|
+ "out_uid": out_uid,
|
|
|
+ "user_name": user_name,
|
|
|
+ "out_avatar_url": out_user_info["out_avatar_url"],
|
|
|
+ "out_create_time": '',
|
|
|
+ "out_tag": '',
|
|
|
+ "out_play_cnt": 0,
|
|
|
+ "out_fans": out_user_info["out_fans"],
|
|
|
+ "out_follow": out_user_info["out_follow"],
|
|
|
+ "out_friend": 0,
|
|
|
+ "out_like": 0,
|
|
|
+ "platform": cls.platform,
|
|
|
+ "tag": cls.tag,
|
|
|
+ }
|
|
|
+ our_user_dict = getUser.create_user(log_type=log_type, crawler=crawler,
|
|
|
+ out_user_dict=out_user_dict, env=env, machine=machine)
|
|
|
+ our_uid = our_user_dict['our_uid']
|
|
|
+ our_user_link = our_user_dict['our_user_link']
|
|
|
+ Feishu.update_values(log_type, crawler, sheetid, f'G{i + 1}:H{i + 1}',
|
|
|
+ [[our_uid, our_user_link]])
|
|
|
+ Common.logger(log_type, crawler).info(f'站内用户信息写入飞书成功！\n')
|
|
|
+ our_user_list.append(our_user_dict)
|
|
|
+ else:
|
|
|
+ our_user_dict = {
|
|
|
+ 'out_uid': out_uid,
|
|
|
+ 'user_name': user_name,
|
|
|
+ 'our_uid': our_uid,
|
|
|
+ 'our_user_link': our_user_link,
|
|
|
+ }
|
|
|
+ our_user_list.append(our_user_dict)
|
|
|
+ return our_user_list
|
|
|
+ except Exception as e:
|
|
|
+ Common.logger(log_type, crawler).error(f'get_user_list:{e}\n')
|
|
|
+
|
|
|
+ # 处理视频标题
|
|
|
+ @classmethod
|
|
|
+ def video_title(cls, log_type, crawler, title):
|
|
|
+ title_split1 = title.split(" #")
|
|
|
+ if title_split1[0] != "":
|
|
|
+ title1 = title_split1[0]
|
|
|
+ else:
|
|
|
+ title1 = title_split1[-1]
|
|
|
+
|
|
|
+ title_split2 = title1.split(" #")
|
|
|
+ if title_split2[0] != "":
|
|
|
+ title2 = title_split2[0]
|
|
|
+ else:
|
|
|
+ title2 = title_split2[-1]
|
|
|
+
|
|
|
+ title_split3 = title2.split("@")
|
|
|
+ if title_split3[0] != "":
|
|
|
+ title3 = title_split3[0]
|
|
|
+ else:
|
|
|
+ title3 = title_split3[-1]
|
|
|
+
|
|
|
+ video_title = title3.strip().replace("\n", "") \
|
|
|
+ .replace("/", "").replace("快手", "").replace(" ", "") \
|
|
|
+ .replace(" ", "").replace("&NBSP", "").replace("\r", "") \
|
|
|
+ .replace("#", "").replace(".", "。").replace("\\", "") \
|
|
|
+ .replace(":", "").replace("*", "").replace("？", "") \
|
|
|
+ .replace("?", "").replace('"', "").replace("<", "") \
|
|
|
+ .replace(">", "").replace("|", "").replace("@", "")[:40]
|
|
|
+ if video_title.replace(" ", "") == "" or video_title == "。。。" or video_title == "...":
|
|
|
+ return cls.random_title(log_type, crawler)
|
|
|
+ else:
|
|
|
+ return video_title
|
|
|
+
|
|
|
+ @classmethod
|
|
|
+ def get_videoList(cls, log_type, crawler, strategy, our_uid, oss_endpoint, env, machine):
|
|
|
+ # rule_dict_1 = cls.get_rule(log_type, crawler)
|
|
|
+ # if rule_dict_1 is None:
|
|
|
+ # Common.logger(log_type, crawler).warning(f"rule_dict is None")
|
|
|
+ # return
|
|
|
+
|
|
|
+ for i in range(100):
|
|
|
+ url = "https://www.kuaishou.com/graphql"
|
|
|
+
|
|
|
+ payload = json.dumps({
|
|
|
+ "operationName": "visionNewRecoFeed",
|
|
|
+ "variables": {
|
|
|
+ "dailyFirstPage": False
|
|
|
+ },
|
|
|
+ "query": "fragment photoContent on PhotoEntity {\n id\n duration\n caption\n originCaption\n likeCount\n viewCount\n realLikeCount\n coverUrl\n photoUrl\n photoH265Url\n manifest\n manifestH265\n videoResource\n coverUrls {\n url\n __typename\n }\n timestamp\n expTag\n animatedCoverUrl\n distance\n videoRatio\n liked\n stereoType\n profileUserTopPhoto\n musicBlocked\n __typename\n}\n\nfragment feedContent on Feed {\n type\n author {\n id\n name\n headerUrl\n following\n headerUrls {\n url\n __typename\n }\n __typename\n }\n photo {\n ...photoContent\n __typename\n }\n canAddComment\n llsid\n status\n currentPcursor\n tags {\n type\n name\n __typename\n }\n __typename\n}\n\nfragment photoResult on PhotoResult {\n result\n llsid\n expTag\n serverExpTag\n pcursor\n feeds {\n ...feedContent\n __typename\n }\n webPageArea\n __typename\n}\n\nquery visionNewRecoFeed($semKeyword: String, $semCrowd: String, $utmSource: String, $utmMedium: String, $utmCampaign: String, $dailyFirstPage: Boolean) {\n visionNewRecoFeed(semKeyword: $semKeyword, semCrowd: $semCrowd, utmSource: $utmSource, utmMedium: $utmMedium, utmCampaign: $utmCampaign, dailyFirstPage: $dailyFirstPage) {\n ...photoResult\n __typename\n }\n}\n"
|
|
|
+ })
|
|
|
+ headers = {
|
|
|
+ 'Accept-Language': 'zh-CN,zh;q=0.9',
|
|
|
+ 'Connection': 'keep-alive',
|
|
|
+ 'Cookie': 'kpf=PC_WEB; clientid=3; did=web_aba004b1780f4d7174d0a2ff42da1fe7; kpn=KUAISHOU_VISION;',
|
|
|
+ 'Origin': 'https://www.kuaishou.com',
|
|
|
+ 'Referer': 'https://www.kuaishou.com/new-reco',
|
|
|
+ 'Sec-Fetch-Dest': 'empty',
|
|
|
+ 'Sec-Fetch-Mode': 'cors',
|
|
|
+ 'Sec-Fetch-Site': 'same-origin',
|
|
|
+ 'User-Agent': get_random_user_agent('pc'),
|
|
|
+ 'accept': '*/*',
|
|
|
+ 'content-type': 'application/json',
|
|
|
+ 'sec-ch-ua': '"Google Chrome";v="111", "Not(A:Brand";v="8", "Chromium";v="111"',
|
|
|
+ 'sec-ch-ua-mobile': '?0',
|
|
|
+ 'sec-ch-ua-platform': '"macOS"'
|
|
|
+ }
|
|
|
+
|
|
|
+ try:
|
|
|
+ urllib3.disable_warnings()
|
|
|
+ s = requests.session()
|
|
|
+ # max_retries=3 重试3次
|
|
|
+ s.mount('http://', HTTPAdapter(max_retries=3))
|
|
|
+ s.mount('https://', HTTPAdapter(max_retries=3))
|
|
|
+ response = s.post(url=url, headers=headers, data=payload, proxies=Common.tunnel_proxies(), verify=False,
|
|
|
+ timeout=10)
|
|
|
+ response.close()
|
|
|
+ except Exception as e:
|
|
|
+ Common.logger(log_type, crawler).error(f"get_videoList:{e}\n")
|
|
|
+ return
|
|
|
+ # Common.logger(log_type, crawler).info(f"get_videoList:{response.text}\n")
|
|
|
+ if response.status_code != 200:
|
|
|
+ Common.logger(log_type, crawler).warning(f"get_videoList_response:{response.text}\n")
|
|
|
+ return
|
|
|
+ elif 'data' not in response.json():
|
|
|
+ Common.logger(log_type, crawler).warning(f"get_videoList_response:{response.json()}\n")
|
|
|
+ return
|
|
|
+ elif 'visionNewRecoFeed' not in response.json()['data']:
|
|
|
+ Common.logger(log_type, crawler).warning(f"get_videoList_response:{response.json()['data']}\n")
|
|
|
+ return
|
|
|
+ elif 'feeds' not in response.json()['data']['visionNewRecoFeed']:
|
|
|
+ Common.logger(log_type, crawler).warning(
|
|
|
+ f"get_videoList_response:{response.json()['data']['visionNewRecoFeed']}\n")
|
|
|
+ return
|
|
|
+ elif len(response.json()['data']['visionNewRecoFeed']['feeds']) == 0:
|
|
|
+ Common.logger(log_type, crawler).info("没有更多视频啦 ~\n")
|
|
|
+ return
|
|
|
+ else:
|
|
|
+ feeds = response.json()['data']['visionNewRecoFeed']['feeds']
|
|
|
+ # pcursor = response.json()['data']['visionNewRecoFeed']['pcursor']
|
|
|
+ for i in range(len(feeds)):
|
|
|
+ if 'photo' not in feeds[i]:
|
|
|
+ Common.logger(log_type, crawler).warning(f"get_videoList:{feeds[i]}\n")
|
|
|
+ break
|
|
|
+
|
|
|
+ # video_title
|
|
|
+ if 'caption' not in feeds[i]['photo']:
|
|
|
+ video_title = cls.random_title(log_type, crawler)
|
|
|
+ elif feeds[i]['photo']['caption'].strip() == "":
|
|
|
+ video_title = cls.random_title(log_type, crawler)
|
|
|
+ else:
|
|
|
+ video_title = cls.video_title(log_type, crawler, feeds[i]['photo']['caption'])
|
|
|
+
|
|
|
+ if 'videoResource' not in feeds[i]['photo'] \
|
|
|
+ and 'manifest' not in feeds[i]['photo'] \
|
|
|
+ and 'manifestH265' not in feeds[i]['photo']:
|
|
|
+ Common.logger(log_type, crawler).warning(f"get_videoList:{feeds[i]['photo']}\n")
|
|
|
+ break
|
|
|
+ videoResource = feeds[i]['photo']['videoResource']
|
|
|
+
|
|
|
+ if 'h264' not in videoResource and 'hevc' not in videoResource:
|
|
|
+ Common.logger(log_type, crawler).warning(f"get_videoList:{videoResource}\n")
|
|
|
+ break
|
|
|
+
|
|
|
+ # video_id
|
|
|
+ if 'h264' in videoResource and 'videoId' in videoResource['h264']:
|
|
|
+ video_id = videoResource['h264']['videoId']
|
|
|
+ elif 'hevc' in videoResource and 'videoId' in videoResource['hevc']:
|
|
|
+ video_id = videoResource['hevc']['videoId']
|
|
|
+ else:
|
|
|
+ video_id = ""
|
|
|
+
|
|
|
+ # play_cnt
|
|
|
+ if 'viewCount' not in feeds[i]['photo']:
|
|
|
+ play_cnt = 0
|
|
|
+ else:
|
|
|
+ play_cnt = int(feeds[i]['photo']['viewCount'])
|
|
|
+
|
|
|
+ # like_cnt
|
|
|
+ if 'realLikeCount' not in feeds[i]['photo']:
|
|
|
+ like_cnt = 0
|
|
|
+ else:
|
|
|
+ like_cnt = feeds[i]['photo']['realLikeCount']
|
|
|
+
|
|
|
+ # publish_time
|
|
|
+ if 'timestamp' not in feeds[i]['photo']:
|
|
|
+ publish_time_stamp = 0
|
|
|
+ publish_time_str = ''
|
|
|
+ publish_time = 0
|
|
|
+ else:
|
|
|
+ publish_time_stamp = int(int(feeds[i]['photo']['timestamp']) / 1000)
|
|
|
+ publish_time_str = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(publish_time_stamp))
|
|
|
+ publish_time = int((int(time.time()) - publish_time_stamp) / (3600 * 24))
|
|
|
+
|
|
|
+ # duration
|
|
|
+ if 'duration' not in feeds[i]['photo']:
|
|
|
+ duration = 0
|
|
|
+ else:
|
|
|
+ duration = int(int(feeds[i]['photo']['duration']) / 1000)
|
|
|
+
|
|
|
+ # video_width / video_height / video_url
|
|
|
+ mapping = {}
|
|
|
+ for item in ['width', 'height']:
|
|
|
+ try:
|
|
|
+ val = str(videoResource['h264']['adaptationSet'][0]['representation'][0][item])
|
|
|
+ except Exception:
|
|
|
+ val = str(videoResource['hevc']['adaptationSet'][0]['representation'][0][item])
|
|
|
+ except:
|
|
|
+ val = ''
|
|
|
+ mapping[item] = val
|
|
|
+ video_width = int(mapping['width']) if mapping['width'] != '' else 0
|
|
|
+ video_height = int(mapping['height']) if mapping['height'] != '' else 0
|
|
|
+ # cover_url
|
|
|
+ if 'coverUrl' not in feeds[i]['photo']:
|
|
|
+ cover_url = ""
|
|
|
+ else:
|
|
|
+ cover_url = feeds[i]['photo']['coverUrl']
|
|
|
+
|
|
|
+ # user_name / avatar_url
|
|
|
+ try:
|
|
|
+ user_name = feeds[i]['author']['name']
|
|
|
+ avatar_url = feeds[i]['author']['headerUrl']
|
|
|
+ user_id = feeds[i]['author']['id']
|
|
|
+ except Exception:
|
|
|
+ user_name = ''
|
|
|
+ avatar_url = ''
|
|
|
+ user_id = ''
|
|
|
+ video_url = feeds[i]['photo']['photoUrl']
|
|
|
+ video_dict = {'video_title': video_title,
|
|
|
+ 'video_id': video_id,
|
|
|
+ 'play_cnt': play_cnt,
|
|
|
+ 'comment_cnt': 0,
|
|
|
+ 'like_cnt': like_cnt,
|
|
|
+ 'share_cnt': 0,
|
|
|
+ 'video_width': video_width,
|
|
|
+ 'video_height': video_height,
|
|
|
+ 'duration': duration,
|
|
|
+ 'publish_time': publish_time,
|
|
|
+ 'publish_time_stamp': publish_time_stamp,
|
|
|
+ 'publish_time_str': publish_time_str,
|
|
|
+ 'user_name': user_name,
|
|
|
+ 'user_id': user_id,
|
|
|
+ 'avatar_url': avatar_url,
|
|
|
+ 'cover_url': cover_url,
|
|
|
+ 'video_url': video_url,
|
|
|
+ 'session': f"kuaishou{int(time.time())}"}
|
|
|
+
|
|
|
+ # rule_1 = cls.download_rule(video_dict, rule_dict_1)
|
|
|
+ # Common.logger(log_type, crawler).info(f"video_title:{video_title}")
|
|
|
+ # Common.logger(log_type, crawler).info(f"video_id:{video_id}\n")
|
|
|
+ #
|
|
|
+ # Common.logger(log_type, crawler).info(
|
|
|
+ # f"like_cnt:{video_dict['like_cnt']}{rule_dict_1['like_cnt']}, {eval(str(video_dict['like_cnt']) + str(rule_dict_1['like_cnt']))}")
|
|
|
+ # Common.logger(log_type, crawler).info(
|
|
|
+ # f"video_width:{video_dict['video_width']}{rule_dict_1['video_width']}, {eval(str(video_dict['video_width']) + str(rule_dict_1['video_width']))}")
|
|
|
+ # Common.logger(log_type, crawler).info(
|
|
|
+ # f"video_height:{video_dict['video_height']}{rule_dict_1['video_height']}, {eval(str(video_dict['video_height']) + str(rule_dict_1['video_height']))}")
|
|
|
+ # Common.logger(log_type, crawler).info(
|
|
|
+ # f"duration:{video_dict['duration']}{rule_dict_1['duration']}, {eval(str(video_dict['duration']) + str(rule_dict_1['duration']))}")
|
|
|
+ # Common.logger(log_type, crawler).info(
|
|
|
+ # f"publish_time:{video_dict['publish_time']}{rule_dict_1['publish_time']}, {eval(str(video_dict['publish_time']) + str(rule_dict_1['publish_time']))}")
|
|
|
+ # Common.logger(log_type, crawler).info(f"rule_1:{rule_1}\n")
|
|
|
+
|
|
|
+ if video_title == "" or video_url == "":
|
|
|
+ Common.logger(log_type, crawler).info("无效视频\n")
|
|
|
+ continue
|
|
|
+ cls.download_publish(log_type=log_type,
|
|
|
+ crawler=crawler,
|
|
|
+ strategy=strategy,
|
|
|
+ video_dict=video_dict,
|
|
|
+ rule_dict={},
|
|
|
+ our_uid=our_uid,
|
|
|
+ oss_endpoint=oss_endpoint,
|
|
|
+ env=env,
|
|
|
+ machine=machine)
|
|
|
+ # elif rule_1 is True:
|
|
|
+ # cls.download_publish(log_type=log_type,
|
|
|
+ # crawler=crawler,
|
|
|
+ # strategy=strategy,
|
|
|
+ # video_dict=video_dict,
|
|
|
+ # rule_dict=rule_dict_1,
|
|
|
+ # our_uid=our_uid,
|
|
|
+ # oss_endpoint=oss_endpoint,
|
|
|
+ # env=env,
|
|
|
+ # machine=machine)
|
|
|
+
|
|
|
+
|
|
|
+ # else:
|
|
|
+ # Common.logger(log_type, crawler).info("不满足下载规则\n")
|
|
|
+
|
|
|
+ # if pcursor == "no_more":
|
|
|
+ # Common.logger(log_type, crawler).info(f"作者,{out_uid},已经到底了，没有更多内容了\n")
|
|
|
+ # return
|
|
|
+ # cls.get_videoList(log_type, crawler, strategy, our_uid, out_uid, oss_endpoint, env, machine,
|
|
|
+ # pcursor=pcursor)
|
|
|
+ # time.sleep(random.randint(1, 3))
|
|
|
+
|
|
|
+ @classmethod
|
|
|
+ def repeat_video(cls, log_type, crawler, video_id, video_title, publish_time, env, machine):
|
|
|
+ sql = f""" select * from crawler_video where platform="{cls.platform}" and out_video_id="{video_id}" or (platform="{cls.platform}" and video_title="{video_title}" and publish_time="{publish_time}") """
|
|
|
+ repeat_video = MysqlHelper.get_values(log_type, crawler, sql, env, machine)
|
|
|
+ return len(repeat_video)
|
|
|
+
|
|
|
+ @classmethod
|
|
|
+ def download_publish(cls, log_type, crawler, strategy, video_dict, rule_dict, our_uid, oss_endpoint, env, machine):
|
|
|
+ try:
|
|
|
+ download_finished = False
|
|
|
+ if cls.repeat_video(log_type, crawler, video_dict['video_id'], video_dict['video_title'],
|
|
|
+ video_dict['publish_time_str'], env, machine) != 0:
|
|
|
+ Common.logger(log_type, crawler).info('视频已下载\n')
|
|
|
+ elif any(word if word in video_dict['video_title'] else False for word in
|
|
|
+ cls.filter_words(log_type, crawler)) is True:
|
|
|
+ Common.logger(log_type, crawler).info('标题已中过滤词\n')
|
|
|
+ else:
|
|
|
+ # 下载视频
|
|
|
+ Common.download_method(log_type=log_type, crawler=crawler, text='video',
|
|
|
+ title=video_dict['video_title'], url=video_dict['video_url'])
|
|
|
+ md_title = md5(video_dict['video_title'].encode('utf8')).hexdigest()
|
|
|
+ if os.path.getsize(f"./{crawler}/videos/{md_title}/video.mp4") == 0:
|
|
|
+ # 删除视频文件夹
|
|
|
+ shutil.rmtree(f"./{crawler}/videos/{md_title}")
|
|
|
+ Common.logger(log_type, crawler).info("视频size=0，删除成功\n")
|
|
|
+ return
|
|
|
+ # 下载封面
|
|
|
+ Common.download_method(log_type=log_type, crawler=crawler, text='cover',
|
|
|
+ title=video_dict['video_title'], url=video_dict['cover_url'])
|
|
|
+ # 保存视频信息至txt
|
|
|
+ Common.save_video_info(log_type=log_type, crawler=crawler, video_dict=video_dict)
|
|
|
+
|
|
|
+ # 上传视频
|
|
|
+ Common.logger(log_type, crawler).info("开始上传视频...")
|
|
|
+ our_video_id = Publish.upload_and_publish(log_type=log_type,
|
|
|
+ crawler=crawler,
|
|
|
+ strategy=strategy,
|
|
|
+ our_uid=our_uid,
|
|
|
+ env=env,
|
|
|
+ oss_endpoint=oss_endpoint)
|
|
|
+ if env == 'dev':
|
|
|
+ our_video_link = f"https://testadmin.piaoquantv.com/cms/post-detail/{our_video_id}/info"
|
|
|
+ else:
|
|
|
+ our_video_link = f"https://admin.piaoquantv.com/cms/post-detail/{our_video_id}/info"
|
|
|
+ Common.logger(log_type, crawler).info("视频上传完成")
|
|
|
+
|
|
|
+ if our_video_id is None:
|
|
|
+ Common.logger(log_type, crawler).warning(f"our_video_id:{our_video_id} 删除该视频文件夹")
|
|
|
+ # 删除视频文件夹
|
|
|
+ shutil.rmtree(f"./{crawler}/videos/{video_dict['video_title']}")
|
|
|
+ return download_finished
|
|
|
+
|
|
|
+ # 视频信息保存数据库
|
|
|
+ insert_sql = f""" insert into crawler_video(video_id,
|
|
|
+ user_id,
|
|
|
+ out_user_id,
|
|
|
+ platform,
|
|
|
+ strategy,
|
|
|
+ out_video_id,
|
|
|
+ video_title,
|
|
|
+ cover_url,
|
|
|
+ video_url,
|
|
|
+ duration,
|
|
|
+ publish_time,
|
|
|
+ play_cnt,
|
|
|
+ crawler_rule,
|
|
|
+ width,
|
|
|
+ height)
|
|
|
+ values({our_video_id},
|
|
|
+ {our_uid},
|
|
|
+ "{video_dict['user_id']}",
|
|
|
+ "{cls.platform}",
|
|
|
+ "定向爬虫策略",
|
|
|
+ "{video_dict['video_id']}",
|
|
|
+ "{video_dict['video_title']}",
|
|
|
+ "{video_dict['cover_url']}",
|
|
|
+ "{video_dict['video_url']}",
|
|
|
+ {int(video_dict['duration'])},
|
|
|
+ "{video_dict['publish_time_str']}",
|
|
|
+ {int(video_dict['play_cnt'])},
|
|
|
+ '{json.dumps(rule_dict)}',
|
|
|
+ {int(video_dict['video_width'])},
|
|
|
+ {int(video_dict['video_height'])}) """
|
|
|
+ Common.logger(log_type, crawler).info(f"insert_sql:{insert_sql}")
|
|
|
+ MysqlHelper.update_values(log_type, crawler, insert_sql, env, machine)
|
|
|
+ Common.logger(log_type, crawler).info('视频信息插入数据库成功！\n')
|
|
|
+
|
|
|
+ # 视频写入飞书
|
|
|
+ Feishu.insert_columns(log_type, 'kuaishou', "Aps2BI", "ROWS", 1, 2)
|
|
|
+ upload_time = int(time.time())
|
|
|
+ values = [[our_video_id,
|
|
|
+ time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(upload_time)),
|
|
|
+ "定向榜",
|
|
|
+ str(video_dict['video_id']),
|
|
|
+ video_dict['video_title'],
|
|
|
+ our_video_link,
|
|
|
+ video_dict['play_cnt'],
|
|
|
+ video_dict['comment_cnt'],
|
|
|
+ video_dict['like_cnt'],
|
|
|
+ video_dict['share_cnt'],
|
|
|
+ video_dict['duration'],
|
|
|
+ f"{video_dict['video_width']}*{video_dict['video_height']}",
|
|
|
+ video_dict['publish_time_str'],
|
|
|
+ video_dict['user_name'],
|
|
|
+ video_dict['user_id'],
|
|
|
+ video_dict['avatar_url'],
|
|
|
+ video_dict['cover_url'],
|
|
|
+ video_dict['video_url']]]
|
|
|
+ time.sleep(1)
|
|
|
+ Feishu.update_values(log_type, 'kuaishou', "Aps2BI", "E2:Z2", values)
|
|
|
+ Common.logger(log_type, crawler).info(f"视频已保存至云文档\n")
|
|
|
+ download_finished = True
|
|
|
+ return download_finished
|
|
|
+ except Exception as e:
|
|
|
+ Common.logger(log_type, crawler).error(f"download_publish:{e}\n")
|
|
|
+
|
|
|
+
|
|
|
+if __name__ == "__main__":
|
|
|
+ KuaiShouRecommend.get_videoList('recommend', 'kuaishou', '推荐抓取策略', 55440319, 'outer', 'prod', 'aliyun')
|
|
|
+
|
