# -*- coding: utf-8 -*- # @Author: wangkun # @Time: 2022/3/29 """ 从 微信小程序-快手短视频 中，下载符合规则的视频 """ import json import time import requests import urllib3 from main.common import Common from main.publish import Publish class KuaiShou: # 已下载视频列表 download_video_list = [] @classmethod def kuaishou_sensitive_words(cls): sensitive_words = [ "集结吧光合创作者", "电影解说", "快来露两手", "分享家常美食教程", "光合作者助手", "创作者中心", "创作者学院", "娱乐星熠计划", "解说电影", "电影剪辑", "放映室", "老剧", "影视剪辑", "精彩片段", "冬日影娱大作战", "春日追剧计划单", "影视解说", "中视频影视混剪计划", "众志成城共抗疫情", "我在追好剧", "娱乐星灿计划", "电影", "电视剧", "毛泽东", "毛主席", "周恩来", "林彪", "习近平", "习大大", "彭丽媛", "怀旧经典影视", ] return sensitive_words @staticmethod def kuaishou_download_rule(d_duration, d_width, d_height, d_play_cnt, d_like_cnt, d_share_cnt): """ 下载视频的基本规则 :param d_duration: 时长 :param d_width: 宽 :param d_height: 高 :param d_play_cnt: 播放量 :param d_like_cnt: 点赞量 :param d_share_cnt: 分享量 :return: 满足规则，返回 True；反之，返回 False """ if 600 >= int(float(d_duration)) >= 60: if int(d_width) >= 720 or int(d_height) >= 720: if int(d_play_cnt) >= 50000: if int(d_like_cnt) >= 50000: if int(d_share_cnt) >= 2000: return True else: return False else: return False else: return False return False return False @classmethod def kuaishou_get_recommend(cls): """ 从快手小程序首页推荐获取视频list: 1.在 kuaishou_videoid.txt 中去重 2.在 kuaishou_feeds.txt 中去重 3.添加视频信息到 kuaishou_feeds.txt """ url = "https://wxmini-api.uyouqu.com/rest/wd/wechatApp/feed/recommend" params = { "__NS_sig3": "e6f6b281ea31e3d7d1bbb8b91f662576fc25f7c3a7a7a5a5aaaba8b2", "__NS_sig3_origin": "3sCt3iAAAAAAAAAAAAAAAwEQBv2b8ewCwkZKaiAAAAAPg0soi" "e7GiOlU vF4zPrG1Nl6xvaoBgFd3MwTzOed9w==" } cookies = { "did": "wxo_05f915ac6b1deca87db36cea1a0fd18fae6c", "preMinaVersion": "v3.109.0", "sid": "kuaishou.wechat.app", "appId": "ks_wechat_small_app_2", "clientid": "13", "client_key": "f60ac815", "kpn": "WECHAT_SMALL_APP", "kpf": "OUTSIDE_ANDROID_H5", "language": "zh_CN", "smallAppVersion": "v3.109.0", "session_key": "123005bcc551a92aac29cdb96190251c9f492c29d4ba6c502dc" "0d2f8b8d18df356a2f7a22d6924d1dd34b8554a64af49b1bb1a" "1236cd2f69c25d4ac2a2531ebcd28c179da14b222023f9e111c" "c4d3b064ac7b0915d8c9fdaccb59e4048e96a5c38a32b2ce9f4abf628053001", "unionid": "V2:1230b56c8337908c3eecba63142a58daca05535c1f14bf67d3d8" "85cace91a7db335c5572d204762d075f24aa84412e2955711a12bb9" "2bd9c2290489ba7a733708a4a446de83822205ab727650489dda0db" "9d2a226c5ddb66d88a1f1373283a3d3b959611d816660028053001", "eUserStableOpenId": "12303325e8710eb802137c70fd1fb65997a4e5e33d82" "cddd409d335d096e20873e07ee472090133bc7a67e5c" "749da045d9a31a12da4c4c26181d432b873ec39432f4" "10196c6c2220323d0e6b562d1b3786aefb352b4e509c" "d96f3466b7b2e5e74b904a94c40792d928053001", "openId": "o5otV45DcV1EUsWw4fAUk_iq0YSA", "eOpenUserId": "124074b7726c996283f25044a42e2c7427e929cd6d968c5342" "330e61fc8939e57b0da4ffe21887f3abc8784175f73e1a267d" "671247273806f293f64c9c8c2adc00a21a12bb92bd9c229048" "9ba7a733708a4a446de8382220534aa79c69b74866bb09187e" "eceec880fa1e0fa421b7df8b3289dab603b17c4828053001", "kuaishou.wechat.app_st": "ChZrdWFpc2hvdS53ZWNoYXQuYXBwLnN0ErAB8aO" "EcB6jh4CMSJ-p_4BJFCId0PKNa_5IeFfeV_tj7q" "CjdXK0y13CSte6-KHbNK9BPo6Rjy3OGny0sh4Zb" "5AUl3Q_zqVXe2TunW7_F3nlTdJOdZ6iVIhPrHa1" "CM0Y-cG9gS4FDDzTvejfWaTI0CbjfNN0RZXzYVE" "AUVT_BNgUVDtYBbEY792gPylMfXxwxKMSzkhaDe" "eaHkGCWUj62FGCFYQ9Fw2W3d7suCXFsNylqT4aE" "s8oNwmycUiygfvfKuoXlHkbeSIgOhEFMZ3ArImS" "vFY_OwLJDHak1iXRO8g5TwzHTvBT3WcoBTAB", "passToken": "ChNwYXNzcG9ydC5wYXNzLXRva2VuEpABI42IhPCJHfFngXC3i-vF" "3daRTB-EtnAYyE6HpfWcPoZ6VSRDvKrom_RvltQ2zKk1T3_FJteb" "mv7ZzQLD7IicnTypaGoeflb7KQVrAv50Mp_JL4ObfBu_xTiwI53t" "bTlM6iML0G7DFd16K5z0jZZ1xECKVQQbk_vIqnseUujFIWAsKcDz" "BqqfnQNbUU5DzDUkGhKgKyzmNjRDxLfpDU5SPFhJmG0iIGBZ_Vd-" "7eT8i_Xit9ZPM-zdFpnRZFveFE9iplMg8Z06KAUwAQ", "userId": "2845397958" } json_data = { "thirdPartyUserId": 2845397958, "photoId": "5250352807040393911", "forwardUserId": 2845397958, "count": 10, "portal": 2, "pageType": 2, "needLivestream": "true", "extraRequestInfo": "{\"scene\":1074,\"fid\":\"2845397958\"," "\"sharerUserId\":\"2845397958\",\"curPhotoIndex\":0," "\"adShow\":true,\"weChatAd\":{},\"page\":0}", "pcursor": 0, "sourceFrom": 2, } try: urllib3.disable_warnings() r = requests.post(url=url, params=params, cookies=cookies, json=json_data, verify=False) response = json.loads(r.content.decode("utf8")) if "feeds" not in response: Common.crawler_log().info("获取快手视频 list 出错:{}，休眠 10s".format(response)) time.sleep(10) else: feeds = response["feeds"] for i in range(len(feeds)): if "photoId" not in feeds[i]: photo_id = "0" Common.crawler_log().info("photo_id:{}".format(photo_id)) else: photo_id = feeds[i]["photoId"] Common.crawler_log().info("photo_id:{}".format(photo_id)) if "viewCount" not in feeds[i]: video_play_cnt = "0" Common.crawler_log().info("video_play_cnt:0") else: video_play_cnt = feeds[i]["viewCount"] Common.crawler_log().info("video_play_cnt:{}".format(video_play_cnt)) if "likeCount" not in feeds[i]: video_like_cnt = "0" Common.crawler_log().info("video_like_cnt:0") else: video_like_cnt = feeds[i]["likeCount"] Common.crawler_log().info("video_like_cnt:{}".format(video_like_cnt)) if "headUrl" not in feeds[i]: head_url = "0" Common.crawler_log().info("head_url:不存在") else: head_url = feeds[i]["headUrl"] Common.crawler_log().info("head_url:{}".format(head_url)) if len(feeds[i]["coverUrls"]) == 0: cover_url = "0" Common.crawler_log().info("cover_url:不存在") else: cover_url = feeds[i]["coverUrls"][0]["url"] Common.crawler_log().info("cover_url:{}".format(cover_url)) if len(feeds[i]["mainMvUrls"]) == 0: video_url = "0" Common.crawler_log().info("video_url:不存在") else: video_url = feeds[i]["mainMvUrls"][0]["url"] Common.crawler_log().info("video_url:{}".format(video_url)) if "shareCount" not in feeds[i]: video_share_cnt = "0" Common.crawler_log().info("video_share_cnt:0") else: video_share_cnt = feeds[i]["shareCount"] Common.crawler_log().info("video_share_cnt:{}".format(video_share_cnt)) if "width" not in feeds[i] or "height"not in feeds[i]: video_width = "0" video_height = "0" video_resolution = str(video_width) + "*" + str(video_height) Common.crawler_log().info("无分辨率") else: video_width = feeds[i]["width"] video_height = feeds[i]["height"] video_resolution = str(video_width) + "*" + str(video_height) Common.crawler_log().info("video_resolution:{}".format(video_resolution)) if "commentCount" not in feeds[i]: video_comment_cnt = "0" Common.crawler_log().info("video_comment_cnt:0") else: video_comment_cnt = feeds[i]["commentCount"] Common.crawler_log().info("video_comment_cnt:{}".format(video_comment_cnt)) if "duration" not in feeds[i]: video_duration = "0" Common.crawler_log().info("video_duration:不存在") else: video_duration = int(int(feeds[i]["duration"])/1000) Common.crawler_log().info("video_duration:{}秒".format(video_duration)) if "timestamp" not in feeds[i]: video_send_time = "0" Common.crawler_log().info("video_send_time:不存在") else: video_send_time = feeds[i]["timestamp"] Common.crawler_log().info("video_send_time:{}".format( time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(int(video_send_time)/1000)))) user_name = feeds[i]["userName"].strip().replace("\n", "")\ .replace("/", "").replace("快手", "").replace(" ", "")\ .replace(" ", "").replace("&NBSP", "").replace("\r", "") Common.crawler_log().info("user_name:{}".format(user_name)) user_id = feeds[i]["userId"] Common.crawler_log().info("user_id:{}".format(user_id)) # 视频标题过滤话题及处理特殊字符 kuaishou_title = feeds[i]["caption"] title_split1 = kuaishou_title.split(" #") if title_split1[0] != "": title1 = title_split1[0] else: title1 = title_split1[-1] title_split2 = title1.split(" #") if title_split2[0] != "": title2 = title_split2[0] else: title2 = title_split2[-1] title_split3 = title2.split("@") if title_split3[0] != "": title3 = title_split3[0] else: title3 = title_split3[-1] video_title = title3.strip().replace("\n", "")\ .replace("/", "").replace("快手", "").replace(" ", "")\ .replace(" ", "").replace("&NBSP", "").replace("\r", "")\ .replace("#", "").replace(".", "。") Common.crawler_log().info("video_title:{}".format(video_title)) # 从 kuaishou_videoid.txt 中去重 photo_ids = Common.read_txt("kuaishou_videoid.txt") if photo_id in [p_id.strip() for p_id in photo_ids]: Common.crawler_log().info("该视频已下载:{}".format(video_title)) pass else: Common.crawler_log().info("该视频未下载:{}".format(video_title)) # 从 kuaishou_feeds.txt 中去重 contents = Common.read_txt("kuaishou_feeds.txt") # kuaishou_feeds.txt 为空时，直接保存 if len(contents) == 0 and head_url != "0" \ and cover_url != "0" and video_url != "0" \ and video_duration != "0" and photo_id != "0": # 判断敏感词 if any(word if word in kuaishou_title else False for word in cls.kuaishou_sensitive_words()) is True: Common.crawler_log().info("视频已中敏感词：{}".format(kuaishou_title)) else: basic_time = int(time.time()) Common.crawler_log().info("添加视频信息至kuaishou_feeds.txt:{}".format(video_title)) with open("./txt/kuaishou_feeds.txt", "a", encoding="utf8") as f_a: f_a.write(str(basic_time) + " + " + str(photo_id) + " + " + str(video_play_cnt) + " + " + str(video_title) + " + " + str(video_duration) + " + " + str(video_comment_cnt) + " + " + str(video_like_cnt) + " + " + str(video_share_cnt) + " + " + str(video_resolution) + " + " + str(video_send_time) + " + " + str(user_name) + " + " + str(head_url) + " + " + str(cover_url) + " + " + str(video_url) + " + " + str(user_id) + " + " + str("wxo_b07ba02ad4340205d89b47c76030bb090977") + "\n") else: if photo_id in [content.split(" + ")[1] for content in contents]: Common.crawler_log().info("该视频已在 kuaishou_feeds.txt 中:{}".format(video_title)) elif head_url == "0" or cover_url == "0" \ or video_url == "0" or video_duration == "0" or photo_id == "0": Common.crawler_log().info("视频封面/播放地址/播放时长/用户头像不存在") else: # 判断敏感词 if any(word if word in kuaishou_title else False for word in cls.kuaishou_sensitive_words()) is True: Common.crawler_log().info("视频已中敏感词：{}".format(kuaishou_title)) else: basic_time = int(time.time()) Common.crawler_log().info("添加视频信息至kuaishou_feeds.txt:{}".format(video_title)) with open("./txt/kuaishou_feeds.txt", "a", encoding="utf8") as f_a: f_a.write(str(basic_time) + " + " + str(photo_id) + " + " + str(video_play_cnt) + " + " + str(video_title) + " + " + str(video_duration) + " + " + str(video_comment_cnt) + " + " + str(video_like_cnt) + " + " + str(video_share_cnt) + " + " + str(video_resolution) + " + " + str(video_send_time) + " + " + str(user_name) + " + " + str(head_url) + " + " + str(cover_url) + " + " + str(video_url) + " + " + str(user_id) + " + " + str("wxo_b07ba02ad4340205d89b47c76030bb090977") + "\n") except Exception as e: Common.crawler_log().error("获取视频 list 异常:{}".format(e)) @classmethod def kuaishou_download_play_video(cls, env): """ 下载播放量视频 测试环境:env == dev 正式环境:env == prod """ videos = Common.read_txt("kuaishou_feeds.txt") for video in videos: download_photo_id = video.strip().split(" + ")[1] download_video_title = video.strip().split(" + ")[3] download_video_duration = video.strip().split(" + ")[4] download_video_play_cnt = video.strip().split(" + ")[2] download_video_comment_cnt = video.strip().split(" + ")[5] download_video_like_cnt = video.strip().split(" + ")[6] download_video_share_cnt = video.strip().split(" + ")[7] download_video_resolution = video.strip().split(" + ")[8] download_video_width = download_video_resolution.split("*")[0] download_video_height = download_video_resolution.split("*")[-1] download_video_send_time = video.strip().split(" + ")[9] download_user_name = video.strip().split(" + ")[10] download_head_url = video.strip().split(" + ")[11] download_cover_url = video.strip().split(" + ")[12] download_video_url = video.strip().split(" + ")[13] download_video_session = video.strip().split(" + ")[-1] if cls.kuaishou_download_rule(download_video_duration, download_video_width, download_video_height, download_video_play_cnt, download_video_like_cnt, download_video_share_cnt) is True: Common.crawler_log().info("开始下载快手视频:{}".format(download_video_title)) # 下载封面 Common.download_method(text="cover", d_name=download_video_title, d_url=download_cover_url) # 下载视频 Common.download_method(text="video", d_name=download_video_title, d_url=download_video_url) # 保存视频信息至 kuaishou_videoid.txt with open("./txt/kuaishou_videoid.txt", "a", encoding="utf8") as fa: fa.write(download_photo_id + "\n") # 添加视频 ID 到 list，用于统计当次下载总数 cls.download_video_list.append(download_photo_id) # # 保存视频信息至 {today}_kuaishou_videoid.txt # with open("./txt/" + str(Common.today) + "_kuaishou_videoid.txt", "a", encoding="utf8") as fc: # fc.write(download_photo_id + "\n") # 保存视频信息至 "./videos/{download_video_title}/info.txt" with open("./videos/" + download_video_title + "/info.txt", "a", encoding="utf8") as f_a: f_a.write(str(download_photo_id) + "\n" + str(download_video_title) + "\n" + str(download_video_duration) + "\n" + str(download_video_play_cnt) + "\n" + str(download_video_comment_cnt) + "\n" + str(download_video_like_cnt) + "\n" + str(download_video_share_cnt) + "\n" + str(download_video_resolution) + "\n" + str(download_video_send_time) + "\n" + str(download_user_name) + "\n" + str(download_head_url) + "\n" + str(download_video_url) + "\n" + str(download_cover_url) + "\n" + str(download_video_session)) # 上传视频 if env == "dev": Common.crawler_log().info("开始上传视频:{}".format(download_video_title)) Publish.upload_and_publish("dev", "play") elif env == "prod": Common.crawler_log().info("开始上传视频:{}".format(download_video_title)) Publish.upload_and_publish("prod", "play") # 删除该视频在kuaishou_feeds.txt中的信息 Common.crawler_log().info("删除该视频在kuaishou_feeds.txt中的信息:{}".format(download_video_title)) with open("./txt/kuaishou_feeds.txt", "r", encoding="utf8") as f_r: lines = f_r.readlines() with open("./txt/kuaishou_feeds.txt", "w", encoding="utf-8") as f_w: for line in lines: if download_photo_id in line.split(" + ")[1]: continue f_w.write(line) else: # 删除该视频在 recommend.txt中的信息 Common.crawler_log().info("该视频不满足下载规则，删除在kuaishou_feeds.txt中的信息:{}".format(download_video_title)) with open("./txt/kuaishou_feeds.txt", "r", encoding="utf8") as f_r: lines = f_r.readlines() with open("./txt/kuaishou_feeds.txt", "w", encoding="utf-8") as f_w: for line in lines: if download_photo_id in line.split(" + ")[1]: continue f_w.write(line) if __name__ == "__main__": kuaishou = KuaiShou() kuaishou.kuaishou_get_recommend()