xigua_follow.py 60 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043
  1. # -*- coding: utf-8 -*-
  2. # @Author: wangkun
  3. # @Time: 2023/2/17
  4. import base64
  5. import json
  6. import os
  7. import random
  8. import shutil
  9. import string
  10. import sys
  11. import time
  12. from hashlib import md5
  13. import requests
  14. import urllib3
  15. from requests.adapters import HTTPAdapter
  16. # from selenium.webdriver import DesiredCapabilities
  17. # from selenium.webdriver.chrome.service import Service
  18. # from selenium.webdriver.common.by import By
  19. # from selenium import webdriver
  20. from lxml import etree
  21. sys.path.append(os.getcwd())
  22. from common.db import MysqlHelper
  23. from common.getuser import getUser
  24. from common.common import Common
  25. from common.feishu import Feishu
  26. from common.publish import Publish
  27. from common.public import get_user_from_mysql
  28. class Follow:
  29. # 个人主页视频翻页参数
  30. offset = 0
  31. platform = "西瓜视频"
  32. tag = "西瓜视频爬虫,定向爬虫策略"
  33. @classmethod
  34. def get_rule(cls, log_type, crawler):
  35. try:
  36. while True:
  37. rule_sheet = Feishu.get_values_batch(log_type, crawler, "4kxd31")
  38. if rule_sheet is None:
  39. Common.logger(log_type, crawler).warning("rule_sheet is None! 10秒后重新获取")
  40. time.sleep(10)
  41. continue
  42. rule_dict = {
  43. "play_cnt": int(rule_sheet[1][2]),
  44. "comment_cnt": int(rule_sheet[2][2]),
  45. "like_cnt": int(rule_sheet[3][2]),
  46. "duration": int(rule_sheet[4][2]),
  47. "publish_time": int(rule_sheet[5][2]),
  48. "video_width": int(rule_sheet[6][2]),
  49. "video_height": int(rule_sheet[7][2]),
  50. }
  51. return rule_dict
  52. except Exception as e:
  53. Common.logger(log_type, crawler).error(f"get_rule:{e}\n")
  54. # 下载规则
  55. @classmethod
  56. def download_rule(cls, video_info_dict, rule_dict):
  57. if video_info_dict['play_cnt'] >= rule_dict['play_cnt']:
  58. if video_info_dict['comment_cnt'] >= rule_dict['comment_cnt']:
  59. if video_info_dict['like_cnt'] >= rule_dict['like_cnt']:
  60. if video_info_dict['duration'] >= rule_dict['duration']:
  61. if video_info_dict['video_width'] >= rule_dict['video_width'] \
  62. or video_info_dict['video_height'] >= rule_dict['video_height']:
  63. return True
  64. else:
  65. return False
  66. else:
  67. return False
  68. else:
  69. return False
  70. else:
  71. return False
  72. else:
  73. return False
  74. # 过滤词库
  75. @classmethod
  76. def filter_words(cls, log_type, crawler):
  77. try:
  78. while True:
  79. filter_words_sheet = Feishu.get_values_batch(log_type, crawler, 'KGB4Hc')
  80. if filter_words_sheet is None:
  81. Common.logger(log_type, crawler).warning(f"filter_words_sheet:{filter_words_sheet} 10秒钟后重试")
  82. continue
  83. filter_words_list = []
  84. for x in filter_words_sheet:
  85. for y in x:
  86. if y is None:
  87. pass
  88. else:
  89. filter_words_list.append(y)
  90. return filter_words_list
  91. except Exception as e:
  92. Common.logger(log_type, crawler).error(f'filter_words异常:{e}\n')
  93. @classmethod
  94. def get_out_user_info(cls, log_type, crawler, out_uid):
  95. try:
  96. headers = {
  97. 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36 Edg/110.0.1587.41',
  98. 'referer': f'https://www.ixigua.com/home/{out_uid}',
  99. 'Cookie': f'ixigua-a-s=1; support_webp=true; support_avif=false; csrf_session_id=a5355d954d3c63ed1ba35faada452b4d; __ac_signature={cls.random_signature()}; MONITOR_WEB_ID=67cb5099-a022-4ec3-bb8e-c4de6ba51dd0; s_v_web_id=verify_lef4i99x_32SosrdH_Qrtk_4LJn_8S7q_fhu16xe3s8ZV; tt_scid=QLJjPuHf6wxVqu6IIq6gHiJXQpVrCwrdhjH2zpm7-E3ZniE1RXBcP6M8b41FJOdo41e1; ttwid=1%7CHHtv2QqpSGuSu8r-zXF1QoWsvjmNi1SJrqOrZzg-UCY%7C1677047013%7C5866a444e5ae10a9df8c11551db75010fb77b657f214ccf84e503fae8d313d09; msToken=PerXJcDdIsZ6zXkGITsftXX4mDaVaW21GuqtzSVdctH46oXXT2GcELIs9f0XW2hunRzP6KVHLZaYElRvNYflLKUXih7lC27XKxs3HjdZiXPK9NQaoKbLfA==; ixigua-a-s=1', }
  100. url = f"https://www.ixigua.com/home/{out_uid}"
  101. urllib3.disable_warnings()
  102. s = requests.session()
  103. # max_retries=3 重试3次
  104. s.mount('http://', HTTPAdapter(max_retries=3))
  105. s.mount('https://', HTTPAdapter(max_retries=3))
  106. response = s.get(url=url, headers=headers, proxies=Common.tunnel_proxies(), verify=False, timeout=5).text
  107. html = etree.HTML(response)
  108. out_follow_str = html.xpath('//div[@class="userDetailV3__header__detail2"]/*[1]/span')[0].text.encode(
  109. 'raw_unicode_escape').decode()
  110. out_fans_str = html.xpath('//div[@class="userDetailV3__header__detail2"]/*[2]/span')[0].text.encode(
  111. 'raw_unicode_escape').decode()
  112. out_like_str = html.xpath('//div[@class="userDetailV3__header__detail2"]/*[3]/span')[0].text.encode(
  113. 'raw_unicode_escape').decode()
  114. out_avatar_url = f"""https:{html.xpath('//span[@class="component-avatar__inner"]//img/@src')[0]}"""
  115. if "万" in out_follow_str:
  116. out_follow = int(float(out_follow_str.split("万")[0]) * 10000)
  117. else:
  118. out_follow = int(out_follow_str.replace(",", ""))
  119. if "万" in out_fans_str:
  120. out_fans = int(float(out_fans_str.split("万")[0]) * 10000)
  121. else:
  122. out_fans = int(out_fans_str.replace(",", ""))
  123. if "万" in out_like_str:
  124. out_like = int(float(out_like_str.split("万")[0]) * 10000)
  125. else:
  126. out_like = int(out_like_str.replace(",", ""))
  127. out_user_dict = {
  128. "out_follow": out_follow,
  129. "out_fans": out_fans,
  130. "out_like": out_like,
  131. "out_avatar_url": out_avatar_url,
  132. }
  133. # for k, v in out_user_dict.items():
  134. # print(f"{k}:{v}")
  135. return out_user_dict
  136. except Exception as e:
  137. Common.logger(log_type, crawler).error(f"get_out_user_info:{e}\n")
  138. # 获取用户信息(字典格式). 注意:部分 user_id 字符类型是 int / str
  139. @classmethod
  140. def get_user_list(cls, log_type, crawler, sheetid, env, machine):
  141. try:
  142. while True:
  143. user_sheet = Feishu.get_values_batch(log_type, crawler, sheetid)
  144. if user_sheet is None:
  145. Common.logger(log_type, crawler).warning(f"user_sheet:{user_sheet} 10秒钟后重试")
  146. continue
  147. our_user_list = []
  148. for i in range(1, len(user_sheet)):
  149. # for i in range(428, len(user_sheet)):
  150. out_uid = user_sheet[i][2]
  151. user_name = user_sheet[i][3]
  152. our_uid = user_sheet[i][6]
  153. our_user_link = user_sheet[i][7]
  154. if out_uid is None or user_name is None:
  155. Common.logger(log_type, crawler).info("空行\n")
  156. else:
  157. Common.logger(log_type, crawler).info(f"正在更新 {user_name} 用户信息\n")
  158. if our_uid is None:
  159. try:
  160. out_user_info = cls.get_out_user_info(log_type, crawler, out_uid)
  161. except Exception as e:
  162. continue
  163. out_user_dict = {
  164. "out_uid": out_uid,
  165. "user_name": user_name,
  166. "out_avatar_url": out_user_info["out_avatar_url"],
  167. "out_create_time": '',
  168. "out_tag": '',
  169. "out_play_cnt": 0,
  170. "out_fans": out_user_info["out_fans"],
  171. "out_follow": out_user_info["out_follow"],
  172. "out_friend": 0,
  173. "out_like": out_user_info["out_like"],
  174. "platform": cls.platform,
  175. "tag": cls.tag,
  176. }
  177. our_user_dict = getUser.create_user(log_type=log_type, crawler=crawler,
  178. out_user_dict=out_user_dict, env=env, machine=machine)
  179. our_uid = our_user_dict['our_uid']
  180. our_user_link = our_user_dict['our_user_link']
  181. Feishu.update_values(log_type, crawler, sheetid, f'G{i + 1}:H{i + 1}',
  182. [[our_uid, our_user_link]])
  183. Common.logger(log_type, crawler).info(f'站内用户信息写入飞书成功!\n')
  184. our_user_list.append(our_user_dict)
  185. else:
  186. our_user_dict = {
  187. 'out_uid': out_uid,
  188. 'user_name': user_name,
  189. 'our_uid': our_uid,
  190. 'our_user_link': our_user_link,
  191. }
  192. our_user_list.append(our_user_dict)
  193. return our_user_list
  194. except Exception as e:
  195. Common.logger(log_type, crawler).error(f'get_user_id_from_feishu异常:{e}\n')
  196. @classmethod
  197. def random_signature(cls):
  198. src_digits = string.digits # string_数字
  199. src_uppercase = string.ascii_uppercase # string_大写字母
  200. src_lowercase = string.ascii_lowercase # string_小写字母
  201. digits_num = random.randint(1, 6)
  202. uppercase_num = random.randint(1, 26 - digits_num - 1)
  203. lowercase_num = 26 - (digits_num + uppercase_num)
  204. password = random.sample(src_digits, digits_num) + random.sample(src_uppercase, uppercase_num) + random.sample(
  205. src_lowercase, lowercase_num)
  206. random.shuffle(password)
  207. new_password = 'AAAAAAAAAA' + ''.join(password)[10:-4] + 'AAAB'
  208. new_password_start = new_password[0:18]
  209. new_password_end = new_password[-7:]
  210. if new_password[18] == '8':
  211. new_password = new_password_start + 'w' + new_password_end
  212. elif new_password[18] == '9':
  213. new_password = new_password_start + 'x' + new_password_end
  214. elif new_password[18] == '-':
  215. new_password = new_password_start + 'y' + new_password_end
  216. elif new_password[18] == '.':
  217. new_password = new_password_start + 'z' + new_password_end
  218. else:
  219. new_password = new_password_start + 'y' + new_password_end
  220. return new_password
  221. # @classmethod
  222. # def get_signature(cls, log_type, crawler, out_uid, machine):
  223. # try:
  224. # # 打印请求配置
  225. # ca = DesiredCapabilities.CHROME
  226. # ca["goog:loggingPrefs"] = {"performance": "ALL"}
  227. #
  228. # # 不打开浏览器运行
  229. # chrome_options = webdriver.ChromeOptions()
  230. # chrome_options.add_argument("--headless")
  231. # chrome_options.add_argument('--user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.79 Safari/537.36')
  232. # chrome_options.add_argument("--no-sandbox")
  233. #
  234. # # driver初始化
  235. # if machine == 'aliyun' or machine == 'aliyun_hk':
  236. # driver = webdriver.Chrome(desired_capabilities=ca, options=chrome_options)
  237. # elif machine == 'macpro':
  238. # driver = webdriver.Chrome(desired_capabilities=ca, options=chrome_options,
  239. # service=Service('/Users/lieyunye/Downloads/chromedriver_v86/chromedriver'))
  240. # elif machine == 'macair':
  241. # driver = webdriver.Chrome(desired_capabilities=ca, options=chrome_options,
  242. # service=Service('/Users/piaoquan/Downloads/chromedriver'))
  243. # else:
  244. # driver = webdriver.Chrome(desired_capabilities=ca, options=chrome_options, service=Service('/Users/wangkun/Downloads/chromedriver/chromedriver_v110/chromedriver'))
  245. # driver.implicitly_wait(10)
  246. # driver.get(f'https://www.ixigua.com/home/{out_uid}/')
  247. # time.sleep(3)
  248. # data_src = driver.find_elements(By.XPATH, '//img[@class="tt-img BU-MagicImage tt-img-loaded"]')[1].get_attribute("data-src")
  249. # signature = data_src.split("x-signature=")[-1]
  250. # return signature
  251. # except Exception as e:
  252. # Common.logger(log_type, crawler).error(f'get_signature异常:{e}\n')
  253. # 获取视频详情
  254. @classmethod
  255. def get_video_url(cls, log_type, crawler, gid):
  256. try:
  257. url = 'https://www.ixigua.com/api/mixVideo/information?'
  258. headers = {
  259. "accept-encoding": "gzip, deflate",
  260. "accept-language": "zh-CN,zh-Hans;q=0.9",
  261. "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
  262. "AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.5 Safari/605.1.15",
  263. "referer": "https://www.ixigua.com/7102614741050196520?logTag=0531c88ac04f38ab2c62",
  264. }
  265. params = {
  266. 'mixId': gid,
  267. 'msToken': 'IlG0wd0Pylyw9ghcYiB2YseUmTwrsrqqhXrbIcsSaTcLTJyVlbYJzk20zw3UO-CfrfC'
  268. 'NVVIOBNjIl7vfBoxnVUwO9ZyzAI3umSKsT5-pef_RRfQCJwmA',
  269. 'X-Bogus': 'DFSzswVupYTANCJOSBk0P53WxM-r',
  270. '_signature': '_02B4Z6wo0000119LvEwAAIDCuktNZ0y5wkdfS7jAALThuOR8D9yWNZ.EmWHKV0WSn6Px'
  271. 'fPsH9-BldyxVje0f49ryXgmn7Tzk-swEHNb15TiGqa6YF.cX0jW8Eds1TtJOIZyfc9s5emH7gdWN94',
  272. }
  273. cookies = {
  274. 'ixigua-a-s': '1',
  275. 'msToken': 'IlG0wd0Pylyw9ghcYiB2YseUmTwrsrqqhXrbIcsSaTcLTJyVlbYJzk20zw3UO-CfrfCNVVIOB'
  276. 'NjIl7vfBoxnVUwO9ZyzAI3umSKsT5-pef_RRfQCJwmA',
  277. 'ttwid': '1%7C_yXQeHWwLZgCsgHClOwTCdYSOt_MjdOkgnPIkpi-Sr8%7C1661241238%7Cf57d0c5ef3f1d7'
  278. '6e049fccdca1ac54887c34d1f8731c8e51a49780ff0ceab9f8',
  279. 'tt_scid': 'QZ4l8KXDG0YAEaMCSbADdcybdKbUfG4BC6S4OBv9lpRS5VyqYLX2bIR8CTeZeGHR9ee3',
  280. 'MONITOR_WEB_ID': '0a49204a-7af5-4e96-95f0-f4bafb7450ad',
  281. '__ac_nonce': '06304878000964fdad287',
  282. '__ac_signature': '_02B4Z6wo00f017Rcr3AAAIDCUVxeW1tOKEu0fKvAAI4cvoYzV-wBhq7B6D8k0no7lb'
  283. 'FlvYoinmtK6UXjRIYPXnahUlFTvmWVtb77jsMkKAXzAEsLE56m36RlvL7ky.M3Xn52r9t1IEb7IR3ke8',
  284. 'ttcid': 'e56fabf6e85d4adf9e4d91902496a0e882',
  285. '_tea_utm_cache_1300': 'undefined',
  286. 'support_avif': 'false',
  287. 'support_webp': 'false',
  288. 'xiguavideopcwebid': '7134967546256016900',
  289. 'xiguavideopcwebid.sig': 'xxRww5R1VEMJN_dQepHorEu_eAc',
  290. }
  291. urllib3.disable_warnings()
  292. s = requests.session()
  293. # max_retries=3 重试3次
  294. s.mount('http://', HTTPAdapter(max_retries=3))
  295. s.mount('https://', HTTPAdapter(max_retries=3))
  296. response = s.get(url=url, headers=headers, params=params, cookies=cookies, verify=False,
  297. proxies=Common.tunnel_proxies(), timeout=5)
  298. response.close()
  299. if 'data' not in response.json() or response.json()['data'] == '':
  300. Common.logger(log_type, crawler).warning('get_video_info: response: {}', response)
  301. else:
  302. video_info = response.json()['data']['gidInformation']['packerData']['video']
  303. video_url_dict = {}
  304. # video_url
  305. if 'videoResource' not in video_info:
  306. video_url_dict["video_url"] = ''
  307. video_url_dict["audio_url"] = ''
  308. video_url_dict["video_width"] = 0
  309. video_url_dict["video_height"] = 0
  310. elif 'dash_120fps' in video_info['videoResource']:
  311. if "video_list" in video_info['videoResource']['dash_120fps'] and 'video_4' in \
  312. video_info['videoResource']['dash_120fps']['video_list']:
  313. video_url = video_info['videoResource']['dash_120fps']['video_list']['video_4']['backup_url_1']
  314. audio_url = video_info['videoResource']['dash_120fps']['video_list']['video_4']['backup_url_1']
  315. if len(video_url) % 3 == 1:
  316. video_url += '=='
  317. elif len(video_url) % 3 == 2:
  318. video_url += '='
  319. elif len(audio_url) % 3 == 1:
  320. audio_url += '=='
  321. elif len(audio_url) % 3 == 2:
  322. audio_url += '='
  323. video_url = base64.b64decode(video_url).decode('utf8')
  324. audio_url = base64.b64decode(audio_url).decode('utf8')
  325. video_width = video_info['videoResource']['dash_120fps']['video_list']['video_4']['vwidth']
  326. video_height = video_info['videoResource']['dash_120fps']['video_list']['video_4']['vheight']
  327. video_url_dict["video_url"] = video_url
  328. video_url_dict["audio_url"] = audio_url
  329. video_url_dict["video_width"] = video_width
  330. video_url_dict["video_height"] = video_height
  331. elif "video_list" in video_info['videoResource']['dash_120fps'] and 'video_3' in \
  332. video_info['videoResource']['dash_120fps']['video_list']:
  333. video_url = video_info['videoResource']['dash_120fps']['video_list']['video_3']['backup_url_1']
  334. audio_url = video_info['videoResource']['dash_120fps']['video_list']['video_3']['backup_url_1']
  335. if len(video_url) % 3 == 1:
  336. video_url += '=='
  337. elif len(video_url) % 3 == 2:
  338. video_url += '='
  339. elif len(audio_url) % 3 == 1:
  340. audio_url += '=='
  341. elif len(audio_url) % 3 == 2:
  342. audio_url += '='
  343. video_url = base64.b64decode(video_url).decode('utf8')
  344. audio_url = base64.b64decode(audio_url).decode('utf8')
  345. video_width = video_info['videoResource']['dash_120fps']['video_list']['video_3']['vwidth']
  346. video_height = video_info['videoResource']['dash_120fps']['video_list']['video_3']['vheight']
  347. video_url_dict["video_url"] = video_url
  348. video_url_dict["audio_url"] = audio_url
  349. video_url_dict["video_width"] = video_width
  350. video_url_dict["video_height"] = video_height
  351. elif "video_list" in video_info['videoResource']['dash_120fps'] and 'video_2' in \
  352. video_info['videoResource']['dash_120fps']['video_list']:
  353. video_url = video_info['videoResource']['dash_120fps']['video_list']['video_2']['backup_url_1']
  354. audio_url = video_info['videoResource']['dash_120fps']['video_list']['video_2']['backup_url_1']
  355. if len(video_url) % 3 == 1:
  356. video_url += '=='
  357. elif len(video_url) % 3 == 2:
  358. video_url += '='
  359. elif len(audio_url) % 3 == 1:
  360. audio_url += '=='
  361. elif len(audio_url) % 3 == 2:
  362. audio_url += '='
  363. video_url = base64.b64decode(video_url).decode('utf8')
  364. audio_url = base64.b64decode(audio_url).decode('utf8')
  365. video_width = video_info['videoResource']['dash_120fps']['video_list']['video_2']['vwidth']
  366. video_height = video_info['videoResource']['dash_120fps']['video_list']['video_2']['vheight']
  367. video_url_dict["video_url"] = video_url
  368. video_url_dict["audio_url"] = audio_url
  369. video_url_dict["video_width"] = video_width
  370. video_url_dict["video_height"] = video_height
  371. elif "video_list" in video_info['videoResource']['dash_120fps'] and 'video_1' in \
  372. video_info['videoResource']['dash_120fps']['video_list']:
  373. video_url = video_info['videoResource']['dash_120fps']['video_list']['video_1']['backup_url_1']
  374. audio_url = video_info['videoResource']['dash_120fps']['video_list']['video_1']['backup_url_1']
  375. if len(video_url) % 3 == 1:
  376. video_url += '=='
  377. elif len(video_url) % 3 == 2:
  378. video_url += '='
  379. elif len(audio_url) % 3 == 1:
  380. audio_url += '=='
  381. elif len(audio_url) % 3 == 2:
  382. audio_url += '='
  383. video_url = base64.b64decode(video_url).decode('utf8')
  384. audio_url = base64.b64decode(audio_url).decode('utf8')
  385. video_width = video_info['videoResource']['dash_120fps']['video_list']['video_1']['vwidth']
  386. video_height = video_info['videoResource']['dash_120fps']['video_list']['video_1']['vheight']
  387. video_url_dict["video_url"] = video_url
  388. video_url_dict["audio_url"] = audio_url
  389. video_url_dict["video_width"] = video_width
  390. video_url_dict["video_height"] = video_height
  391. elif 'dynamic_video' in video_info['videoResource']['dash_120fps'] \
  392. and 'dynamic_video_list' in video_info['videoResource']['dash_120fps']['dynamic_video'] \
  393. and 'dynamic_audio_list' in video_info['videoResource']['dash_120fps']['dynamic_video'] \
  394. and len(
  395. video_info['videoResource']['dash_120fps']['dynamic_video']['dynamic_video_list']) != 0 \
  396. and len(
  397. video_info['videoResource']['dash_120fps']['dynamic_video']['dynamic_audio_list']) != 0:
  398. video_url = \
  399. video_info['videoResource']['dash_120fps']['dynamic_video']['dynamic_video_list'][-1][
  400. 'backup_url_1']
  401. audio_url = \
  402. video_info['videoResource']['dash_120fps']['dynamic_video']['dynamic_audio_list'][-1][
  403. 'backup_url_1']
  404. if len(video_url) % 3 == 1:
  405. video_url += '=='
  406. elif len(video_url) % 3 == 2:
  407. video_url += '='
  408. elif len(audio_url) % 3 == 1:
  409. audio_url += '=='
  410. elif len(audio_url) % 3 == 2:
  411. audio_url += '='
  412. video_url = base64.b64decode(video_url).decode('utf8')
  413. audio_url = base64.b64decode(audio_url).decode('utf8')
  414. video_width = \
  415. video_info['videoResource']['dash_120fps']['dynamic_video']['dynamic_video_list'][-1]['vwidth']
  416. video_height = \
  417. video_info['videoResource']['dash_120fps']['dynamic_video']['dynamic_video_list'][-1]['vheight']
  418. video_url_dict["video_url"] = video_url
  419. video_url_dict["audio_url"] = audio_url
  420. video_url_dict["video_width"] = video_width
  421. video_url_dict["video_height"] = video_height
  422. else:
  423. video_url_dict["video_url"] = ''
  424. video_url_dict["audio_url"] = ''
  425. video_url_dict["video_width"] = 0
  426. video_url_dict["video_height"] = 0
  427. elif 'dash' in video_info['videoResource']:
  428. if "video_list" in video_info['videoResource']['dash'] and 'video_4' in \
  429. video_info['videoResource']['dash']['video_list']:
  430. video_url = video_info['videoResource']['dash']['video_list']['video_4']['backup_url_1']
  431. audio_url = video_info['videoResource']['dash']['video_list']['video_4']['backup_url_1']
  432. if len(video_url) % 3 == 1:
  433. video_url += '=='
  434. elif len(video_url) % 3 == 2:
  435. video_url += '='
  436. elif len(audio_url) % 3 == 1:
  437. audio_url += '=='
  438. elif len(audio_url) % 3 == 2:
  439. audio_url += '='
  440. video_url = base64.b64decode(video_url).decode('utf8')
  441. audio_url = base64.b64decode(audio_url).decode('utf8')
  442. video_width = video_info['videoResource']['dash']['video_list']['video_4']['vwidth']
  443. video_height = video_info['videoResource']['dash']['video_list']['video_4']['vheight']
  444. video_url_dict["video_url"] = video_url
  445. video_url_dict["audio_url"] = audio_url
  446. video_url_dict["video_width"] = video_width
  447. video_url_dict["video_height"] = video_height
  448. elif "video_list" in video_info['videoResource']['dash'] and 'video_3' in \
  449. video_info['videoResource']['dash']['video_list']:
  450. video_url = video_info['videoResource']['dash']['video_list']['video_3']['backup_url_1']
  451. audio_url = video_info['videoResource']['dash']['video_list']['video_3']['backup_url_1']
  452. if len(video_url) % 3 == 1:
  453. video_url += '=='
  454. elif len(video_url) % 3 == 2:
  455. video_url += '='
  456. elif len(audio_url) % 3 == 1:
  457. audio_url += '=='
  458. elif len(audio_url) % 3 == 2:
  459. audio_url += '='
  460. video_url = base64.b64decode(video_url).decode('utf8')
  461. audio_url = base64.b64decode(audio_url).decode('utf8')
  462. video_width = video_info['videoResource']['dash']['video_list']['video_3']['vwidth']
  463. video_height = video_info['videoResource']['dash']['video_list']['video_3']['vheight']
  464. video_url_dict["video_url"] = video_url
  465. video_url_dict["audio_url"] = audio_url
  466. video_url_dict["video_width"] = video_width
  467. video_url_dict["video_height"] = video_height
  468. elif "video_list" in video_info['videoResource']['dash'] and 'video_2' in \
  469. video_info['videoResource']['dash']['video_list']:
  470. video_url = video_info['videoResource']['dash']['video_list']['video_2']['backup_url_1']
  471. audio_url = video_info['videoResource']['dash']['video_list']['video_2']['backup_url_1']
  472. if len(video_url) % 3 == 1:
  473. video_url += '=='
  474. elif len(video_url) % 3 == 2:
  475. video_url += '='
  476. elif len(audio_url) % 3 == 1:
  477. audio_url += '=='
  478. elif len(audio_url) % 3 == 2:
  479. audio_url += '='
  480. video_url = base64.b64decode(video_url).decode('utf8')
  481. audio_url = base64.b64decode(audio_url).decode('utf8')
  482. video_width = video_info['videoResource']['dash']['video_list']['video_2']['vwidth']
  483. video_height = video_info['videoResource']['dash']['video_list']['video_2']['vheight']
  484. video_url_dict["video_url"] = video_url
  485. video_url_dict["audio_url"] = audio_url
  486. video_url_dict["video_width"] = video_width
  487. video_url_dict["video_height"] = video_height
  488. elif "video_list" in video_info['videoResource']['dash'] and 'video_1' in \
  489. video_info['videoResource']['dash']['video_list']:
  490. video_url = video_info['videoResource']['dash']['video_list']['video_1']['backup_url_1']
  491. audio_url = video_info['videoResource']['dash']['video_list']['video_1']['backup_url_1']
  492. if len(video_url) % 3 == 1:
  493. video_url += '=='
  494. elif len(video_url) % 3 == 2:
  495. video_url += '='
  496. elif len(audio_url) % 3 == 1:
  497. audio_url += '=='
  498. elif len(audio_url) % 3 == 2:
  499. audio_url += '='
  500. video_url = base64.b64decode(video_url).decode('utf8')
  501. audio_url = base64.b64decode(audio_url).decode('utf8')
  502. video_width = video_info['videoResource']['dash']['video_list']['video_1']['vwidth']
  503. video_height = video_info['videoResource']['dash']['video_list']['video_1']['vheight']
  504. video_url_dict["video_url"] = video_url
  505. video_url_dict["audio_url"] = audio_url
  506. video_url_dict["video_width"] = video_width
  507. video_url_dict["video_height"] = video_height
  508. elif 'dynamic_video' in video_info['videoResource']['dash'] \
  509. and 'dynamic_video_list' in video_info['videoResource']['dash']['dynamic_video'] \
  510. and 'dynamic_audio_list' in video_info['videoResource']['dash']['dynamic_video'] \
  511. and len(video_info['videoResource']['dash']['dynamic_video']['dynamic_video_list']) != 0 \
  512. and len(video_info['videoResource']['dash']['dynamic_video']['dynamic_audio_list']) != 0:
  513. video_url = video_info['videoResource']['dash']['dynamic_video']['dynamic_video_list'][-1][
  514. 'backup_url_1']
  515. audio_url = video_info['videoResource']['dash']['dynamic_video']['dynamic_audio_list'][-1][
  516. 'backup_url_1']
  517. if len(video_url) % 3 == 1:
  518. video_url += '=='
  519. elif len(video_url) % 3 == 2:
  520. video_url += '='
  521. elif len(audio_url) % 3 == 1:
  522. audio_url += '=='
  523. elif len(audio_url) % 3 == 2:
  524. audio_url += '='
  525. video_url = base64.b64decode(video_url).decode('utf8')
  526. audio_url = base64.b64decode(audio_url).decode('utf8')
  527. video_width = video_info['videoResource']['dash']['dynamic_video']['dynamic_video_list'][-1][
  528. 'vwidth']
  529. video_height = video_info['videoResource']['dash']['dynamic_video']['dynamic_video_list'][-1][
  530. 'vheight']
  531. video_url_dict["video_url"] = video_url
  532. video_url_dict["audio_url"] = audio_url
  533. video_url_dict["video_width"] = video_width
  534. video_url_dict["video_height"] = video_height
  535. else:
  536. video_url_dict["video_url"] = ''
  537. video_url_dict["audio_url"] = ''
  538. video_url_dict["video_width"] = 0
  539. video_url_dict["video_height"] = 0
  540. elif 'normal' in video_info['videoResource']:
  541. if "video_list" in video_info['videoResource']['normal'] and 'video_4' in \
  542. video_info['videoResource']['normal']['video_list']:
  543. video_url = video_info['videoResource']['normal']['video_list']['video_4']['backup_url_1']
  544. audio_url = video_info['videoResource']['normal']['video_list']['video_4']['backup_url_1']
  545. if len(video_url) % 3 == 1:
  546. video_url += '=='
  547. elif len(video_url) % 3 == 2:
  548. video_url += '='
  549. elif len(audio_url) % 3 == 1:
  550. audio_url += '=='
  551. elif len(audio_url) % 3 == 2:
  552. audio_url += '='
  553. video_url = base64.b64decode(video_url).decode('utf8')
  554. audio_url = base64.b64decode(audio_url).decode('utf8')
  555. video_width = video_info['videoResource']['normal']['video_list']['video_4']['vwidth']
  556. video_height = video_info['videoResource']['normal']['video_list']['video_4']['vheight']
  557. video_url_dict["video_url"] = video_url
  558. video_url_dict["audio_url"] = audio_url
  559. video_url_dict["video_width"] = video_width
  560. video_url_dict["video_height"] = video_height
  561. elif "video_list" in video_info['videoResource']['normal'] and 'video_3' in \
  562. video_info['videoResource']['normal']['video_list']:
  563. video_url = video_info['videoResource']['normal']['video_list']['video_3']['backup_url_1']
  564. audio_url = video_info['videoResource']['normal']['video_list']['video_3']['backup_url_1']
  565. if len(video_url) % 3 == 1:
  566. video_url += '=='
  567. elif len(video_url) % 3 == 2:
  568. video_url += '='
  569. elif len(audio_url) % 3 == 1:
  570. audio_url += '=='
  571. elif len(audio_url) % 3 == 2:
  572. audio_url += '='
  573. video_url = base64.b64decode(video_url).decode('utf8')
  574. audio_url = base64.b64decode(audio_url).decode('utf8')
  575. video_width = video_info['videoResource']['normal']['video_list']['video_3']['vwidth']
  576. video_height = video_info['videoResource']['normal']['video_list']['video_3']['vheight']
  577. video_url_dict["video_url"] = video_url
  578. video_url_dict["audio_url"] = audio_url
  579. video_url_dict["video_width"] = video_width
  580. video_url_dict["video_height"] = video_height
  581. elif "video_list" in video_info['videoResource']['normal'] and 'video_2' in \
  582. video_info['videoResource']['normal']['video_list']:
  583. video_url = video_info['videoResource']['normal']['video_list']['video_2']['backup_url_1']
  584. audio_url = video_info['videoResource']['normal']['video_list']['video_2']['backup_url_1']
  585. if len(video_url) % 3 == 1:
  586. video_url += '=='
  587. elif len(video_url) % 3 == 2:
  588. video_url += '='
  589. elif len(audio_url) % 3 == 1:
  590. audio_url += '=='
  591. elif len(audio_url) % 3 == 2:
  592. audio_url += '='
  593. video_url = base64.b64decode(video_url).decode('utf8')
  594. audio_url = base64.b64decode(audio_url).decode('utf8')
  595. video_width = video_info['videoResource']['normal']['video_list']['video_2']['vwidth']
  596. video_height = video_info['videoResource']['normal']['video_list']['video_2']['vheight']
  597. video_url_dict["video_url"] = video_url
  598. video_url_dict["audio_url"] = audio_url
  599. video_url_dict["video_width"] = video_width
  600. video_url_dict["video_height"] = video_height
  601. elif "video_list" in video_info['videoResource']['normal'] and 'video_1' in \
  602. video_info['videoResource']['normal']['video_list']:
  603. video_url = video_info['videoResource']['normal']['video_list']['video_1']['backup_url_1']
  604. audio_url = video_info['videoResource']['normal']['video_list']['video_1']['backup_url_1']
  605. if len(video_url) % 3 == 1:
  606. video_url += '=='
  607. elif len(video_url) % 3 == 2:
  608. video_url += '='
  609. elif len(audio_url) % 3 == 1:
  610. audio_url += '=='
  611. elif len(audio_url) % 3 == 2:
  612. audio_url += '='
  613. video_url = base64.b64decode(video_url).decode('utf8')
  614. audio_url = base64.b64decode(audio_url).decode('utf8')
  615. video_width = video_info['videoResource']['normal']['video_list']['video_1']['vwidth']
  616. video_height = video_info['videoResource']['normal']['video_list']['video_1']['vheight']
  617. video_url_dict["video_url"] = video_url
  618. video_url_dict["audio_url"] = audio_url
  619. video_url_dict["video_width"] = video_width
  620. video_url_dict["video_height"] = video_height
  621. elif 'dynamic_video' in video_info['videoResource']['normal'] \
  622. and 'dynamic_video_list' in video_info['videoResource']['normal']['dynamic_video'] \
  623. and 'dynamic_audio_list' in video_info['videoResource']['normal']['dynamic_video'] \
  624. and len(video_info['videoResource']['normal']['dynamic_video']['dynamic_video_list']) != 0 \
  625. and len(video_info['videoResource']['normal']['dynamic_video']['dynamic_audio_list']) != 0:
  626. video_url = video_info['videoResource']['normal']['dynamic_video']['dynamic_video_list'][-1][
  627. 'backup_url_1']
  628. audio_url = video_info['videoResource']['normal']['dynamic_video']['dynamic_audio_list'][-1][
  629. 'backup_url_1']
  630. if len(video_url) % 3 == 1:
  631. video_url += '=='
  632. elif len(video_url) % 3 == 2:
  633. video_url += '='
  634. elif len(audio_url) % 3 == 1:
  635. audio_url += '=='
  636. elif len(audio_url) % 3 == 2:
  637. audio_url += '='
  638. video_url = base64.b64decode(video_url).decode('utf8')
  639. audio_url = base64.b64decode(audio_url).decode('utf8')
  640. video_width = video_info['videoResource']['normal']['dynamic_video']['dynamic_video_list'][-1][
  641. 'vwidth']
  642. video_height = video_info['videoResource']['normal']['dynamic_video']['dynamic_video_list'][-1][
  643. 'vheight']
  644. video_url_dict["video_url"] = video_url
  645. video_url_dict["audio_url"] = audio_url
  646. video_url_dict["video_width"] = video_width
  647. video_url_dict["video_height"] = video_height
  648. else:
  649. video_url_dict["video_url"] = ''
  650. video_url_dict["audio_url"] = ''
  651. video_url_dict["video_width"] = 0
  652. video_url_dict["video_height"] = 0
  653. else:
  654. video_url_dict["video_url"] = ''
  655. video_url_dict["audio_url"] = ''
  656. video_url_dict["video_width"] = 0
  657. video_url_dict["video_height"] = 0
  658. return video_url_dict
  659. except Exception as e:
  660. Common.logger(log_type, crawler).error(f'get_video_url:{e}\n')
  661. @classmethod
  662. def get_videolist(cls, log_type, crawler, strategy, our_uid, out_uid, oss_endpoint, env, machine):
  663. try:
  664. signature = cls.random_signature()
  665. while True:
  666. url = "https://www.ixigua.com/api/videov2/author/new_video_list?"
  667. params = {
  668. 'to_user_id': str(out_uid),
  669. 'offset': str(cls.offset),
  670. 'limit': '30',
  671. 'maxBehotTime': '0',
  672. 'order': 'new',
  673. 'isHome': '0',
  674. # 'msToken': 'G0eRzNkw189a8TLaXjc6nTHVMQwh9XcxVAqTbGKi7iPJdQcLwS3-XRrJ3MZ7QBfqErpxp3EX1WtvWOIcZ3NIgr41hgcd-v64so_RRj3YCRw1UsKW8mIssNLlIMspsg==',
  675. # 'X-Bogus': 'DFSzswVuEkUANjW9ShFTgR/F6qHt',
  676. '_signature': signature,
  677. }
  678. headers = {
  679. 'referer': f'https://www.ixigua.com/home/{out_uid}/video/?preActiveKey=hotsoon&list_entrance=userdetail',
  680. 'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36 Edg/110.0.1587.41',
  681. }
  682. urllib3.disable_warnings()
  683. s = requests.session()
  684. # max_retries=3 重试3次
  685. s.mount('http://', HTTPAdapter(max_retries=3))
  686. s.mount('https://', HTTPAdapter(max_retries=3))
  687. response = s.get(url=url, headers=headers, params=params, proxies=Common.tunnel_proxies(), verify=False,
  688. timeout=5)
  689. response.close()
  690. cls.offset += 30
  691. if response.status_code != 200:
  692. Common.logger(log_type, crawler).warning(f"get_videolist_response:{response.text}\n")
  693. cls.offset = 0
  694. return
  695. elif 'data' not in response.text:
  696. Common.logger(log_type, crawler).warning(f"get_videolist_response:{response.text}\n")
  697. cls.offset = 0
  698. return
  699. elif not response.json()["data"]['videoList']:
  700. Common.logger(log_type, crawler).warning(f"get_videolist_response:{response.json()}\n")
  701. cls.offset = 0
  702. return
  703. else:
  704. videoList = response.json()['data']['videoList']
  705. for i in range(len(videoList)):
  706. # video_title
  707. if 'title' not in videoList[i]:
  708. video_title = 0
  709. else:
  710. video_title = videoList[i]['title'].strip().replace('手游', '') \
  711. .replace('/', '').replace('\/', '').replace('\n', '').replace('"', '').replace("'", '')
  712. # video_id
  713. if 'video_id' not in videoList[i]:
  714. video_id = 0
  715. else:
  716. video_id = videoList[i]['video_id']
  717. # gid
  718. if 'gid' not in videoList[i]:
  719. gid = 0
  720. else:
  721. gid = videoList[i]['gid']
  722. # play_cnt
  723. if 'video_detail_info' not in videoList[i]:
  724. play_cnt = 0
  725. elif 'video_watch_count' not in videoList[i]['video_detail_info']:
  726. play_cnt = 0
  727. else:
  728. play_cnt = videoList[i]['video_detail_info']['video_watch_count']
  729. # comment_cnt
  730. if 'comment_count' not in videoList[i]:
  731. comment_cnt = 0
  732. else:
  733. comment_cnt = videoList[i]['comment_count']
  734. # like_cnt
  735. if 'digg_count' not in videoList[i]:
  736. like_cnt = 0
  737. else:
  738. like_cnt = videoList[i]['digg_count']
  739. # share_cnt
  740. share_cnt = 0
  741. # video_duration
  742. if 'video_duration' not in videoList[i]:
  743. video_duration = 0
  744. else:
  745. video_duration = int(videoList[i]['video_duration'])
  746. # send_time
  747. if 'publish_time' not in videoList[i]:
  748. publish_time = 0
  749. else:
  750. publish_time = videoList[i]['publish_time']
  751. publish_time_str = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(publish_time))
  752. # is_top
  753. if 'is_top' not in videoList[i]:
  754. is_top = 0
  755. else:
  756. is_top = videoList[i]['is_top']
  757. # user_name
  758. if 'user_info' not in videoList[i]:
  759. user_name = 0
  760. elif 'name' not in videoList[i]['user_info']:
  761. user_name = 0
  762. else:
  763. user_name = videoList[i]['user_info']['name']
  764. # user_id
  765. if 'user_info' not in videoList[i]:
  766. user_id = 0
  767. elif 'user_id' not in videoList[i]['user_info']:
  768. user_id = 0
  769. else:
  770. user_id = videoList[i]['user_info']['user_id']
  771. # avatar_url
  772. if 'user_info' not in videoList[i]:
  773. avatar_url = 0
  774. elif 'avatar_url' not in videoList[i]['user_info']:
  775. avatar_url = 0
  776. else:
  777. avatar_url = videoList[i]['user_info']['avatar_url']
  778. # cover_url
  779. if 'video_detail_info' not in videoList[i]:
  780. cover_url = 0
  781. elif 'detail_video_large_image' not in videoList[i]['video_detail_info']:
  782. cover_url = 0
  783. elif 'url' in videoList[i]['video_detail_info']['detail_video_large_image']:
  784. cover_url = videoList[i]['video_detail_info']['detail_video_large_image']['url']
  785. else:
  786. cover_url = videoList[i]['video_detail_info']['detail_video_large_image']['url_list'][0][
  787. 'url']
  788. Common.logger(log_type, crawler).info(f'---开始读取规则---')
  789. rule_dict = cls.get_rule(log_type, crawler)
  790. Common.logger(log_type, crawler).info(f'---读取规则完成---')
  791. if gid == 0 or video_id == 0 or cover_url == 0:
  792. Common.logger(log_type, crawler).info('无效视频\n')
  793. elif is_top is True and int(time.time()) - int(publish_time) > 3600 * 24 * rule_dict[
  794. 'publish_time']:
  795. Common.logger(log_type, crawler).info(
  796. f'置顶视频,且发布时间:{publish_time_str} 超过{rule_dict["publish_time"]}天\n')
  797. elif int(time.time()) - int(publish_time) > 3600 * 24 * rule_dict['publish_time']:
  798. Common.logger(log_type, crawler).info(
  799. f'发布时间:{publish_time_str}超过{rule_dict["publish_time"]}天\n')
  800. cls.offset = 0
  801. return
  802. else:
  803. video_url_dict = cls.get_video_url(log_type, crawler, gid)
  804. video_url = video_url_dict["video_url"]
  805. audio_url = video_url_dict["audio_url"]
  806. video_width = video_url_dict["video_width"]
  807. video_height = video_url_dict["video_height"]
  808. video_dict = {'video_title': video_title,
  809. 'video_id': video_id,
  810. 'gid': gid,
  811. 'play_cnt': play_cnt,
  812. 'comment_cnt': comment_cnt,
  813. 'like_cnt': like_cnt,
  814. 'share_cnt': share_cnt,
  815. 'video_width': video_width,
  816. 'video_height': video_height,
  817. 'duration': video_duration,
  818. 'publish_time_stamp': publish_time,
  819. 'publish_time_str': publish_time_str,
  820. 'is_top': is_top,
  821. 'user_name': user_name,
  822. 'user_id': user_id,
  823. 'avatar_url': avatar_url,
  824. 'cover_url': cover_url,
  825. 'audio_url': audio_url,
  826. 'video_url': video_url,
  827. 'session': signature}
  828. for k, v in video_dict.items():
  829. Common.logger(log_type, crawler).info(f"{k}:{v}")
  830. cls.download_publish(log_type=log_type,
  831. crawler=crawler,
  832. video_dict=video_dict,
  833. rule_dict=rule_dict,
  834. strategy=strategy,
  835. our_uid=our_uid,
  836. oss_endpoint=oss_endpoint,
  837. env=env,
  838. machine=machine)
  839. except Exception as e:
  840. Common.logger(log_type, crawler).error(f"get_videolist:{e}\n")
  841. @classmethod
  842. def repeat_video(cls, log_type, crawler, video_id, env, machine):
  843. sql = f""" select * from crawler_video where platform="{cls.platform}" and out_video_id="{video_id}"; """
  844. repeat_video = MysqlHelper.get_values(log_type, crawler, sql, env, machine)
  845. return len(repeat_video)
  846. # 下载 / 上传
  847. @classmethod
  848. def download_publish(cls, log_type, crawler, strategy, video_dict, rule_dict, our_uid, oss_endpoint, env, machine):
  849. try:
  850. if cls.download_rule(video_dict, rule_dict) is False:
  851. Common.logger(log_type, crawler).info('不满足抓取规则\n')
  852. elif any(word if word in video_dict['video_title'] else False for word in
  853. cls.filter_words(log_type, crawler)) is True:
  854. Common.logger(log_type, crawler).info('标题已中过滤词:{}\n', video_dict['video_title'])
  855. elif cls.repeat_video(log_type, crawler, video_dict['video_id'], env, machine) != 0:
  856. Common.logger(log_type, crawler).info('视频已下载\n')
  857. # elif str(video_dict['video_id']) in [x for y in Feishu.get_values_batch(log_type, 'xigua', 'e075e9') for x in y]:
  858. # Common.logger(log_type, crawler).info('视频已下载\n')
  859. # elif str(video_dict['video_id']) in [x for y in Feishu.get_values_batch(log_type, 'xigua', '3Ul6wZ') for x in y]:
  860. # Common.logger(log_type, crawler).info('视频已下载\n')
  861. # elif str(video_dict['video_id']) in [x for y in Feishu.get_values_batch(log_type, 'xigua', 'QOWqMo') for x in y]:
  862. # Common.logger(log_type, crawler).info('视频已下载\n')
  863. # elif str(video_dict['video_id']) in [x for y in Feishu.get_values_batch(log_type, 'xigua', 'wjhpDs') for x in y]:
  864. # Common.logger(log_type, crawler).info('视频已存在\n')
  865. else:
  866. # 下载视频
  867. Common.download_method(log_type=log_type, crawler=crawler, text='xigua_video',
  868. title=video_dict['video_title'], url=video_dict['video_url'])
  869. # 下载音频
  870. Common.download_method(log_type=log_type, crawler=crawler, text='xigua_audio',
  871. title=video_dict['video_title'], url=video_dict['audio_url'])
  872. # 合成音视频
  873. Common.video_compose(log_type=log_type, crawler=crawler,
  874. video_dir=f"./{crawler}/videos/{video_dict['video_title']}")
  875. md_title = md5(video_dict['video_title'].encode('utf8')).hexdigest()
  876. if os.path.getsize(f"./{crawler}/videos/{md_title}/video.mp4") == 0:
  877. # 删除视频文件夹
  878. shutil.rmtree(f"./{crawler}/videos/{md_title}")
  879. Common.logger(log_type, crawler).info("视频size=0,删除成功\n")
  880. return
  881. # ffmpeg_dict = Common.ffmpeg(log_type, crawler, f"./{crawler}/videos/{video_dict['video_title']}/video.mp4")
  882. # if ffmpeg_dict is None or ffmpeg_dict['size'] == 0:
  883. # Common.logger(log_type, crawler).warning(f"下载的视频无效,已删除\n")
  884. # # 删除视频文件夹
  885. # shutil.rmtree(f"./{crawler}/videos/{video_dict['video_title']}")
  886. # return
  887. # 下载封面
  888. Common.download_method(log_type=log_type, crawler=crawler, text='cover',
  889. title=video_dict['video_title'], url=video_dict['cover_url'])
  890. # 保存视频信息至txt
  891. Common.save_video_info(log_type=log_type, crawler=crawler, video_dict=video_dict)
  892. # 上传视频
  893. Common.logger(log_type, crawler).info("开始上传视频...")
  894. our_video_id = Publish.upload_and_publish(log_type=log_type,
  895. crawler=crawler,
  896. strategy=strategy,
  897. our_uid=our_uid,
  898. env=env,
  899. oss_endpoint=oss_endpoint)
  900. if env == 'dev':
  901. our_video_link = f"https://testadmin.piaoquantv.com/cms/post-detail/{our_video_id}/info"
  902. else:
  903. our_video_link = f"https://admin.piaoquantv.com/cms/post-detail/{our_video_id}/info"
  904. Common.logger(log_type, crawler).info("视频上传完成")
  905. if our_video_id is None:
  906. # 删除视频文件夹
  907. shutil.rmtree(f"./{crawler}/videos/{video_dict['video_title']}")
  908. return
  909. # 视频写入飞书
  910. Feishu.insert_columns(log_type, 'xigua', "e075e9", "ROWS", 1, 2)
  911. upload_time = int(time.time())
  912. values = [[time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(upload_time)),
  913. "定向榜",
  914. video_dict['video_title'],
  915. str(video_dict['video_id']),
  916. our_video_link,
  917. video_dict['gid'],
  918. video_dict['play_cnt'],
  919. video_dict['comment_cnt'],
  920. video_dict['like_cnt'],
  921. video_dict['share_cnt'],
  922. video_dict['duration'],
  923. str(video_dict['video_width']) + '*' + str(video_dict['video_height']),
  924. video_dict['publish_time_str'],
  925. video_dict['user_name'],
  926. video_dict['user_id'],
  927. video_dict['avatar_url'],
  928. video_dict['cover_url'],
  929. video_dict['video_url'],
  930. video_dict['audio_url']]]
  931. time.sleep(1)
  932. Feishu.update_values(log_type, 'xigua', "e075e9", "F2:Z2", values)
  933. Common.logger(log_type, crawler).info(f"视频已保存至云文档\n")
  934. # 视频信息保存数据库
  935. insert_sql = f""" insert into crawler_video(video_id,
  936. user_id,
  937. out_user_id,
  938. platform,
  939. strategy,
  940. out_video_id,
  941. video_title,
  942. cover_url,
  943. video_url,
  944. duration,
  945. publish_time,
  946. play_cnt,
  947. crawler_rule,
  948. width,
  949. height)
  950. values({our_video_id},
  951. {our_uid},
  952. "{video_dict['user_id']}",
  953. "{cls.platform}",
  954. "定向爬虫策略",
  955. "{video_dict['video_id']}",
  956. "{video_dict['video_title']}",
  957. "{video_dict['cover_url']}",
  958. "{video_dict['video_url']}",
  959. {int(video_dict['duration'])},
  960. "{video_dict['publish_time_str']}",
  961. {int(video_dict['play_cnt'])},
  962. '{json.dumps(rule_dict)}',
  963. {int(video_dict['video_width'])},
  964. {int(video_dict['video_height'])}) """
  965. Common.logger(log_type, crawler).info(f"insert_sql:{insert_sql}")
  966. MysqlHelper.update_values(log_type, crawler, insert_sql, env, machine)
  967. Common.logger(log_type, crawler).info('视频信息插入数据库成功!\n')
  968. except Exception as e:
  969. Common.logger(log_type, crawler).error(f'download_publish异常:{e}\n')
  970. @classmethod
  971. def get_follow_videos(cls, log_type, crawler, strategy, oss_endpoint, env, machine):
  972. user_list = get_user_from_mysql(log_type, crawler, crawler, env)
  973. for user in user_list:
  974. spider_link = user["spider_link"]
  975. out_uid = spider_link.split('/')[-1]
  976. user_name = user["nick_name"]
  977. our_uid = user["media_id"]
  978. Common.logger(log_type, crawler).info(f"开始抓取 {user_name} 用户主页视频\n")
  979. cls.get_videolist(log_type=log_type,
  980. crawler=crawler,
  981. strategy=strategy,
  982. our_uid=our_uid,
  983. out_uid=out_uid,
  984. oss_endpoint=oss_endpoint,
  985. env=env,
  986. machine=machine)
  987. cls.offset = 0
  988. if __name__ == '__main__':
  989. Follow.get_follow_videos('follow', 'xigua', '定向抓取策略', 'inner', 'prod', 'aliyun')