xigua_search.py 50 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883
  1. # -*- coding: utf-8 -*-
  2. # @Author: wangkun
  3. # @Time: 2023/2/17
  4. import base64
  5. import json
  6. import os
  7. import random
  8. import re
  9. import shutil
  10. import string
  11. import sys
  12. import time
  13. import requests
  14. import urllib3
  15. from urllib.parse import quote
  16. from requests.adapters import HTTPAdapter
  17. sys.path.append(os.getcwd())
  18. from common.db import MysqlHelper
  19. from common.users import Users
  20. from common.common import Common
  21. from common.feishu import Feishu
  22. from common.publish import Publish
  23. from common.userAgent import get_random_user_agent, random_user_agent
  24. class Search:
  25. platform = "西瓜视频"
  26. tag = "西瓜视频爬虫,搜索爬虫策略"
  27. @classmethod
  28. def get_rule(cls, log_type, crawler):
  29. try:
  30. while True:
  31. rule_sheet = Feishu.get_values_batch(log_type, crawler, "shxOl7")
  32. if rule_sheet is None:
  33. Common.logger(log_type, crawler).warning("rule_sheet is None! 10秒后重新获取")
  34. time.sleep(10)
  35. continue
  36. rule_dict = {
  37. "play_cnt": int(rule_sheet[1][2]),
  38. "min_duration": int(rule_sheet[2][2]),
  39. "max_duration": int(rule_sheet[3][2]),
  40. "publish_time": int(rule_sheet[4][2]),
  41. }
  42. return rule_dict
  43. except Exception as e:
  44. Common.logger(log_type, crawler).error(f"get_rule:{e}\n")
  45. # 下载规则
  46. @classmethod
  47. def download_rule(cls, video_info_dict, rule_dict):
  48. if video_info_dict['play_cnt'] >= rule_dict['play_cnt']:
  49. if video_info_dict['comment_cnt'] >= rule_dict['comment_cnt']:
  50. if video_info_dict['like_cnt'] >= rule_dict['like_cnt']:
  51. if video_info_dict['duration'] >= rule_dict['duration']:
  52. if video_info_dict['video_width'] >= rule_dict['video_width'] \
  53. or video_info_dict['video_height'] >= rule_dict['video_height']:
  54. return True
  55. else:
  56. return False
  57. else:
  58. return False
  59. else:
  60. return False
  61. else:
  62. return False
  63. else:
  64. return False
  65. # 过滤词库
  66. @classmethod
  67. def filter_words(cls, log_type, crawler):
  68. try:
  69. while True:
  70. filter_words_sheet = Feishu.get_values_batch(log_type, crawler, 'KGB4Hc')
  71. if filter_words_sheet is None:
  72. Common.logger(log_type, crawler).warning(f"filter_words_sheet:{filter_words_sheet} 10秒钟后重试")
  73. continue
  74. filter_words_list = []
  75. for x in filter_words_sheet:
  76. for y in x:
  77. if y is None:
  78. pass
  79. else:
  80. filter_words_list.append(y)
  81. return filter_words_list
  82. except Exception as e:
  83. Common.logger(log_type, crawler).error(f'filter_words异常:{e}\n')
  84. # 获取用户信息(字典格式). 注意:部分 user_id 字符类型是 int / str
  85. @classmethod
  86. def get_user_list(cls, log_type, crawler, sheetid, env, machine):
  87. try:
  88. while True:
  89. user_sheet = Feishu.get_values_batch(log_type, crawler, sheetid)
  90. if user_sheet is None:
  91. Common.logger(log_type, crawler).warning(f"user_sheet:{user_sheet} 10秒钟后重试")
  92. continue
  93. our_user_list = []
  94. for i in range(1, len(user_sheet)):
  95. our_uid = user_sheet[i][6]
  96. search_word = user_sheet[i][4]
  97. storage = user_sheet[i][5]
  98. tag = user_sheet[i][8]
  99. Common.logger(log_type, crawler).info(f"正在更新 {search_word} 关键词信息\n")
  100. if our_uid is None:
  101. default_user = Users.get_default_user()
  102. # 用来创建our_id的信息
  103. user_dict = {
  104. 'nickName': default_user['nickName'],
  105. 'avatarUrl': default_user['avatarUrl'],
  106. 'tagName': f'{storage},{tag}',
  107. }
  108. Common.logger(log_type, crawler).info(f'新创建的站内UID:{our_uid}')
  109. our_uid = Users.create_uid(log_type, crawler, user_dict, env)
  110. if env == 'prod':
  111. our_user_link = f'https://admin.piaoquantv.com/ums/user/{our_uid}/post'
  112. else:
  113. our_user_link = f'https://testadmin.piaoquantv.com/ums/user/{our_uid}/post'
  114. Feishu.update_values(log_type, crawler, sheetid, f'G{i + 1}:H{i + 1}',
  115. [[our_uid, our_user_link]])
  116. Common.logger(log_type, crawler).info(f'站内用户信息写入飞书成功!\n')
  117. our_user_dict = {
  118. 'out_uid': '',
  119. 'search_word': search_word,
  120. 'our_uid': our_uid,
  121. 'our_user_link': f'https://admin.piaoquantv.com/ums/user/{our_uid}/post',
  122. }
  123. our_user_list.append(our_user_dict)
  124. return our_user_list
  125. except Exception as e:
  126. Common.logger(log_type, crawler).error(f'get_user_id_from_feishu异常:{e}\n')
  127. @classmethod
  128. def random_signature(cls):
  129. src_digits = string.digits # string_数字
  130. src_uppercase = string.ascii_uppercase # string_大写字母
  131. src_lowercase = string.ascii_lowercase # string_小写字母
  132. digits_num = random.randint(1, 6)
  133. uppercase_num = random.randint(1, 26 - digits_num - 1)
  134. lowercase_num = 26 - (digits_num + uppercase_num)
  135. password = random.sample(src_digits, digits_num) + random.sample(src_uppercase, uppercase_num) + random.sample(
  136. src_lowercase, lowercase_num)
  137. random.shuffle(password)
  138. new_password = 'AAAAAAAAAA' + ''.join(password)[10:-4] + 'AAAB'
  139. new_password_start = new_password[0:18]
  140. new_password_end = new_password[-7:]
  141. if new_password[18] == '8':
  142. new_password = new_password_start + 'w' + new_password_end
  143. elif new_password[18] == '9':
  144. new_password = new_password_start + 'x' + new_password_end
  145. elif new_password[18] == '-':
  146. new_password = new_password_start + 'y' + new_password_end
  147. elif new_password[18] == '.':
  148. new_password = new_password_start + 'z' + new_password_end
  149. else:
  150. new_password = new_password_start + 'y' + new_password_end
  151. return new_password
  152. # 获取视频详情
  153. @classmethod
  154. def get_video_url(cls, log_type, crawler, gid):
  155. try:
  156. url = 'https://www.ixigua.com/api/mixVideo/information?'
  157. headers = {
  158. "accept-encoding": "gzip, deflate",
  159. "accept-language": "zh-CN,zh-Hans;q=0.9",
  160. "user-agent": get_random_user_agent('pc'),
  161. "referer": "https://www.ixigua.com/7102614741050196520?logTag=0531c88ac04f38ab2c62",
  162. }
  163. params = {
  164. 'mixId': gid,
  165. 'msToken': 'IlG0wd0Pylyw9ghcYiB2YseUmTwrsrqqhXrbIcsSaTcLTJyVlbYJzk20zw3UO-CfrfC'
  166. 'NVVIOBNjIl7vfBoxnVUwO9ZyzAI3umSKsT5-pef_RRfQCJwmA',
  167. 'X-Bogus': 'DFSzswVupYTANCJOSBk0P53WxM-r',
  168. '_signature': '_02B4Z6wo0000119LvEwAAIDCuktNZ0y5wkdfS7jAALThuOR8D9yWNZ.EmWHKV0WSn6Px'
  169. 'fPsH9-BldyxVje0f49ryXgmn7Tzk-swEHNb15TiGqa6YF.cX0jW8Eds1TtJOIZyfc9s5emH7gdWN94',
  170. }
  171. cookies = {
  172. 'ixigua-a-s': '1',
  173. 'msToken': 'IlG0wd0Pylyw9ghcYiB2YseUmTwrsrqqhXrbIcsSaTcLTJyVlbYJzk20zw3UO-CfrfCNVVIOB'
  174. 'NjIl7vfBoxnVUwO9ZyzAI3umSKsT5-pef_RRfQCJwmA',
  175. 'ttwid': '1%7C_yXQeHWwLZgCsgHClOwTCdYSOt_MjdOkgnPIkpi-Sr8%7C1661241238%7Cf57d0c5ef3f1d7'
  176. '6e049fccdca1ac54887c34d1f8731c8e51a49780ff0ceab9f8',
  177. 'tt_scid': 'QZ4l8KXDG0YAEaMCSbADdcybdKbUfG4BC6S4OBv9lpRS5VyqYLX2bIR8CTeZeGHR9ee3',
  178. 'MONITOR_WEB_ID': '0a49204a-7af5-4e96-95f0-f4bafb7450ad',
  179. '__ac_nonce': '06304878000964fdad287',
  180. '__ac_signature': '_02B4Z6wo00f017Rcr3AAAIDCUVxeW1tOKEu0fKvAAI4cvoYzV-wBhq7B6D8k0no7lb'
  181. 'FlvYoinmtK6UXjRIYPXnahUlFTvmWVtb77jsMkKAXzAEsLE56m36RlvL7ky.M3Xn52r9t1IEb7IR3ke8',
  182. 'ttcid': 'e56fabf6e85d4adf9e4d91902496a0e882',
  183. '_tea_utm_cache_1300': 'undefined',
  184. 'support_avif': 'false',
  185. 'support_webp': 'false',
  186. 'xiguavideopcwebid': '7134967546256016900',
  187. 'xiguavideopcwebid.sig': 'xxRww5R1VEMJN_dQepHorEu_eAc',
  188. }
  189. urllib3.disable_warnings()
  190. s = requests.session()
  191. # max_retries=3 重试3次
  192. s.mount('http://', HTTPAdapter(max_retries=3))
  193. s.mount('https://', HTTPAdapter(max_retries=3))
  194. response = s.get(url=url, headers=headers, params=params, cookies=cookies, verify=False,
  195. proxies=Common.tunnel_proxies(), timeout=5)
  196. response.close()
  197. if 'data' not in response.json() or response.json()['data'] == '':
  198. Common.logger(log_type, crawler).warning('get_video_info: response: {}', response)
  199. else:
  200. video_info = response.json()['data']['gidInformation']['packerData']['video']
  201. video_url_dict = {}
  202. # video_url
  203. if 'videoResource' not in video_info:
  204. video_url_dict["video_url"] = ''
  205. video_url_dict["audio_url"] = ''
  206. video_url_dict["video_width"] = 0
  207. video_url_dict["video_height"] = 0
  208. elif 'dash_120fps' in video_info['videoResource']:
  209. if "video_list" in video_info['videoResource']['dash_120fps'] and 'video_4' in \
  210. video_info['videoResource']['dash_120fps']['video_list']:
  211. video_url = video_info['videoResource']['dash_120fps']['video_list']['video_4']['backup_url_1']
  212. audio_url = video_info['videoResource']['dash_120fps']['video_list']['video_4']['backup_url_1']
  213. if len(video_url) % 3 == 1:
  214. video_url += '=='
  215. elif len(video_url) % 3 == 2:
  216. video_url += '='
  217. elif len(audio_url) % 3 == 1:
  218. audio_url += '=='
  219. elif len(audio_url) % 3 == 2:
  220. audio_url += '='
  221. video_url = base64.b64decode(video_url).decode('utf8')
  222. audio_url = base64.b64decode(audio_url).decode('utf8')
  223. video_width = video_info['videoResource']['dash_120fps']['video_list']['video_4']['vwidth']
  224. video_height = video_info['videoResource']['dash_120fps']['video_list']['video_4']['vheight']
  225. video_url_dict["video_url"] = video_url
  226. video_url_dict["audio_url"] = audio_url
  227. video_url_dict["video_width"] = video_width
  228. video_url_dict["video_height"] = video_height
  229. elif "video_list" in video_info['videoResource']['dash_120fps'] and 'video_3' in \
  230. video_info['videoResource']['dash_120fps']['video_list']:
  231. video_url = video_info['videoResource']['dash_120fps']['video_list']['video_3']['backup_url_1']
  232. audio_url = video_info['videoResource']['dash_120fps']['video_list']['video_3']['backup_url_1']
  233. if len(video_url) % 3 == 1:
  234. video_url += '=='
  235. elif len(video_url) % 3 == 2:
  236. video_url += '='
  237. elif len(audio_url) % 3 == 1:
  238. audio_url += '=='
  239. elif len(audio_url) % 3 == 2:
  240. audio_url += '='
  241. video_url = base64.b64decode(video_url).decode('utf8')
  242. audio_url = base64.b64decode(audio_url).decode('utf8')
  243. video_width = video_info['videoResource']['dash_120fps']['video_list']['video_3']['vwidth']
  244. video_height = video_info['videoResource']['dash_120fps']['video_list']['video_3']['vheight']
  245. video_url_dict["video_url"] = video_url
  246. video_url_dict["audio_url"] = audio_url
  247. video_url_dict["video_width"] = video_width
  248. video_url_dict["video_height"] = video_height
  249. elif "video_list" in video_info['videoResource']['dash_120fps'] and 'video_2' in \
  250. video_info['videoResource']['dash_120fps']['video_list']:
  251. video_url = video_info['videoResource']['dash_120fps']['video_list']['video_2']['backup_url_1']
  252. audio_url = video_info['videoResource']['dash_120fps']['video_list']['video_2']['backup_url_1']
  253. if len(video_url) % 3 == 1:
  254. video_url += '=='
  255. elif len(video_url) % 3 == 2:
  256. video_url += '='
  257. elif len(audio_url) % 3 == 1:
  258. audio_url += '=='
  259. elif len(audio_url) % 3 == 2:
  260. audio_url += '='
  261. video_url = base64.b64decode(video_url).decode('utf8')
  262. audio_url = base64.b64decode(audio_url).decode('utf8')
  263. video_width = video_info['videoResource']['dash_120fps']['video_list']['video_2']['vwidth']
  264. video_height = video_info['videoResource']['dash_120fps']['video_list']['video_2']['vheight']
  265. video_url_dict["video_url"] = video_url
  266. video_url_dict["audio_url"] = audio_url
  267. video_url_dict["video_width"] = video_width
  268. video_url_dict["video_height"] = video_height
  269. elif "video_list" in video_info['videoResource']['dash_120fps'] and 'video_1' in \
  270. video_info['videoResource']['dash_120fps']['video_list']:
  271. video_url = video_info['videoResource']['dash_120fps']['video_list']['video_1']['backup_url_1']
  272. audio_url = video_info['videoResource']['dash_120fps']['video_list']['video_1']['backup_url_1']
  273. if len(video_url) % 3 == 1:
  274. video_url += '=='
  275. elif len(video_url) % 3 == 2:
  276. video_url += '='
  277. elif len(audio_url) % 3 == 1:
  278. audio_url += '=='
  279. elif len(audio_url) % 3 == 2:
  280. audio_url += '='
  281. video_url = base64.b64decode(video_url).decode('utf8')
  282. audio_url = base64.b64decode(audio_url).decode('utf8')
  283. video_width = video_info['videoResource']['dash_120fps']['video_list']['video_1']['vwidth']
  284. video_height = video_info['videoResource']['dash_120fps']['video_list']['video_1']['vheight']
  285. video_url_dict["video_url"] = video_url
  286. video_url_dict["audio_url"] = audio_url
  287. video_url_dict["video_width"] = video_width
  288. video_url_dict["video_height"] = video_height
  289. elif 'dynamic_video' in video_info['videoResource']['dash_120fps'] \
  290. and 'dynamic_video_list' in video_info['videoResource']['dash_120fps']['dynamic_video'] \
  291. and 'dynamic_audio_list' in video_info['videoResource']['dash_120fps']['dynamic_video'] \
  292. and len(
  293. video_info['videoResource']['dash_120fps']['dynamic_video']['dynamic_video_list']) != 0 \
  294. and len(
  295. video_info['videoResource']['dash_120fps']['dynamic_video']['dynamic_audio_list']) != 0:
  296. video_url = \
  297. video_info['videoResource']['dash_120fps']['dynamic_video']['dynamic_video_list'][-1][
  298. 'backup_url_1']
  299. audio_url = \
  300. video_info['videoResource']['dash_120fps']['dynamic_video']['dynamic_audio_list'][-1][
  301. 'backup_url_1']
  302. if len(video_url) % 3 == 1:
  303. video_url += '=='
  304. elif len(video_url) % 3 == 2:
  305. video_url += '='
  306. elif len(audio_url) % 3 == 1:
  307. audio_url += '=='
  308. elif len(audio_url) % 3 == 2:
  309. audio_url += '='
  310. video_url = base64.b64decode(video_url).decode('utf8')
  311. audio_url = base64.b64decode(audio_url).decode('utf8')
  312. video_width = \
  313. video_info['videoResource']['dash_120fps']['dynamic_video']['dynamic_video_list'][-1][
  314. 'vwidth']
  315. video_height = \
  316. video_info['videoResource']['dash_120fps']['dynamic_video']['dynamic_video_list'][-1][
  317. 'vheight']
  318. video_url_dict["video_url"] = video_url
  319. video_url_dict["audio_url"] = audio_url
  320. video_url_dict["video_width"] = video_width
  321. video_url_dict["video_height"] = video_height
  322. else:
  323. video_url_dict["video_url"] = ''
  324. video_url_dict["audio_url"] = ''
  325. video_url_dict["video_width"] = 0
  326. video_url_dict["video_height"] = 0
  327. elif 'dash' in video_info['videoResource']:
  328. if "video_list" in video_info['videoResource']['dash'] and 'video_4' in \
  329. video_info['videoResource']['dash']['video_list']:
  330. video_url = video_info['videoResource']['dash']['video_list']['video_4']['backup_url_1']
  331. audio_url = video_info['videoResource']['dash']['video_list']['video_4']['backup_url_1']
  332. if len(video_url) % 3 == 1:
  333. video_url += '=='
  334. elif len(video_url) % 3 == 2:
  335. video_url += '='
  336. elif len(audio_url) % 3 == 1:
  337. audio_url += '=='
  338. elif len(audio_url) % 3 == 2:
  339. audio_url += '='
  340. video_url = base64.b64decode(video_url).decode('utf8')
  341. audio_url = base64.b64decode(audio_url).decode('utf8')
  342. video_width = video_info['videoResource']['dash']['video_list']['video_4']['vwidth']
  343. video_height = video_info['videoResource']['dash']['video_list']['video_4']['vheight']
  344. video_url_dict["video_url"] = video_url
  345. video_url_dict["audio_url"] = audio_url
  346. video_url_dict["video_width"] = video_width
  347. video_url_dict["video_height"] = video_height
  348. elif "video_list" in video_info['videoResource']['dash'] and 'video_3' in \
  349. video_info['videoResource']['dash']['video_list']:
  350. video_url = video_info['videoResource']['dash']['video_list']['video_3']['backup_url_1']
  351. audio_url = video_info['videoResource']['dash']['video_list']['video_3']['backup_url_1']
  352. if len(video_url) % 3 == 1:
  353. video_url += '=='
  354. elif len(video_url) % 3 == 2:
  355. video_url += '='
  356. elif len(audio_url) % 3 == 1:
  357. audio_url += '=='
  358. elif len(audio_url) % 3 == 2:
  359. audio_url += '='
  360. video_url = base64.b64decode(video_url).decode('utf8')
  361. audio_url = base64.b64decode(audio_url).decode('utf8')
  362. video_width = video_info['videoResource']['dash']['video_list']['video_3']['vwidth']
  363. video_height = video_info['videoResource']['dash']['video_list']['video_3']['vheight']
  364. video_url_dict["video_url"] = video_url
  365. video_url_dict["audio_url"] = audio_url
  366. video_url_dict["video_width"] = video_width
  367. video_url_dict["video_height"] = video_height
  368. elif "video_list" in video_info['videoResource']['dash'] and 'video_2' in \
  369. video_info['videoResource']['dash']['video_list']:
  370. video_url = video_info['videoResource']['dash']['video_list']['video_2']['backup_url_1']
  371. audio_url = video_info['videoResource']['dash']['video_list']['video_2']['backup_url_1']
  372. if len(video_url) % 3 == 1:
  373. video_url += '=='
  374. elif len(video_url) % 3 == 2:
  375. video_url += '='
  376. elif len(audio_url) % 3 == 1:
  377. audio_url += '=='
  378. elif len(audio_url) % 3 == 2:
  379. audio_url += '='
  380. video_url = base64.b64decode(video_url).decode('utf8')
  381. audio_url = base64.b64decode(audio_url).decode('utf8')
  382. video_width = video_info['videoResource']['dash']['video_list']['video_2']['vwidth']
  383. video_height = video_info['videoResource']['dash']['video_list']['video_2']['vheight']
  384. video_url_dict["video_url"] = video_url
  385. video_url_dict["audio_url"] = audio_url
  386. video_url_dict["video_width"] = video_width
  387. video_url_dict["video_height"] = video_height
  388. elif "video_list" in video_info['videoResource']['dash'] and 'video_1' in \
  389. video_info['videoResource']['dash']['video_list']:
  390. video_url = video_info['videoResource']['dash']['video_list']['video_1']['backup_url_1']
  391. audio_url = video_info['videoResource']['dash']['video_list']['video_1']['backup_url_1']
  392. if len(video_url) % 3 == 1:
  393. video_url += '=='
  394. elif len(video_url) % 3 == 2:
  395. video_url += '='
  396. elif len(audio_url) % 3 == 1:
  397. audio_url += '=='
  398. elif len(audio_url) % 3 == 2:
  399. audio_url += '='
  400. video_url = base64.b64decode(video_url).decode('utf8')
  401. audio_url = base64.b64decode(audio_url).decode('utf8')
  402. video_width = video_info['videoResource']['dash']['video_list']['video_1']['vwidth']
  403. video_height = video_info['videoResource']['dash']['video_list']['video_1']['vheight']
  404. video_url_dict["video_url"] = video_url
  405. video_url_dict["audio_url"] = audio_url
  406. video_url_dict["video_width"] = video_width
  407. video_url_dict["video_height"] = video_height
  408. elif 'dynamic_video' in video_info['videoResource']['dash'] \
  409. and 'dynamic_video_list' in video_info['videoResource']['dash']['dynamic_video'] \
  410. and 'dynamic_audio_list' in video_info['videoResource']['dash']['dynamic_video'] \
  411. and len(video_info['videoResource']['dash']['dynamic_video']['dynamic_video_list']) != 0 \
  412. and len(video_info['videoResource']['dash']['dynamic_video']['dynamic_audio_list']) != 0:
  413. video_url = video_info['videoResource']['dash']['dynamic_video']['dynamic_video_list'][-1][
  414. 'backup_url_1']
  415. audio_url = video_info['videoResource']['dash']['dynamic_video']['dynamic_audio_list'][-1][
  416. 'backup_url_1']
  417. if len(video_url) % 3 == 1:
  418. video_url += '=='
  419. elif len(video_url) % 3 == 2:
  420. video_url += '='
  421. elif len(audio_url) % 3 == 1:
  422. audio_url += '=='
  423. elif len(audio_url) % 3 == 2:
  424. audio_url += '='
  425. video_url = base64.b64decode(video_url).decode('utf8')
  426. audio_url = base64.b64decode(audio_url).decode('utf8')
  427. video_width = video_info['videoResource']['dash']['dynamic_video']['dynamic_video_list'][-1][
  428. 'vwidth']
  429. video_height = video_info['videoResource']['dash']['dynamic_video']['dynamic_video_list'][-1][
  430. 'vheight']
  431. video_url_dict["video_url"] = video_url
  432. video_url_dict["audio_url"] = audio_url
  433. video_url_dict["video_width"] = video_width
  434. video_url_dict["video_height"] = video_height
  435. else:
  436. video_url_dict["video_url"] = ''
  437. video_url_dict["audio_url"] = ''
  438. video_url_dict["video_width"] = 0
  439. video_url_dict["video_height"] = 0
  440. elif 'normal' in video_info['videoResource']:
  441. if "video_list" in video_info['videoResource']['normal'] and 'video_4' in \
  442. video_info['videoResource']['normal']['video_list']:
  443. video_url = video_info['videoResource']['normal']['video_list']['video_4']['backup_url_1']
  444. audio_url = video_info['videoResource']['normal']['video_list']['video_4']['backup_url_1']
  445. if len(video_url) % 3 == 1:
  446. video_url += '=='
  447. elif len(video_url) % 3 == 2:
  448. video_url += '='
  449. elif len(audio_url) % 3 == 1:
  450. audio_url += '=='
  451. elif len(audio_url) % 3 == 2:
  452. audio_url += '='
  453. video_url = base64.b64decode(video_url).decode('utf8')
  454. audio_url = base64.b64decode(audio_url).decode('utf8')
  455. video_width = video_info['videoResource']['normal']['video_list']['video_4']['vwidth']
  456. video_height = video_info['videoResource']['normal']['video_list']['video_4']['vheight']
  457. video_url_dict["video_url"] = video_url
  458. video_url_dict["audio_url"] = audio_url
  459. video_url_dict["video_width"] = video_width
  460. video_url_dict["video_height"] = video_height
  461. elif "video_list" in video_info['videoResource']['normal'] and 'video_3' in \
  462. video_info['videoResource']['normal']['video_list']:
  463. video_url = video_info['videoResource']['normal']['video_list']['video_3']['backup_url_1']
  464. audio_url = video_info['videoResource']['normal']['video_list']['video_3']['backup_url_1']
  465. if len(video_url) % 3 == 1:
  466. video_url += '=='
  467. elif len(video_url) % 3 == 2:
  468. video_url += '='
  469. elif len(audio_url) % 3 == 1:
  470. audio_url += '=='
  471. elif len(audio_url) % 3 == 2:
  472. audio_url += '='
  473. video_url = base64.b64decode(video_url).decode('utf8')
  474. audio_url = base64.b64decode(audio_url).decode('utf8')
  475. video_width = video_info['videoResource']['normal']['video_list']['video_3']['vwidth']
  476. video_height = video_info['videoResource']['normal']['video_list']['video_3']['vheight']
  477. video_url_dict["video_url"] = video_url
  478. video_url_dict["audio_url"] = audio_url
  479. video_url_dict["video_width"] = video_width
  480. video_url_dict["video_height"] = video_height
  481. elif "video_list" in video_info['videoResource']['normal'] and 'video_2' in \
  482. video_info['videoResource']['normal']['video_list']:
  483. video_url = video_info['videoResource']['normal']['video_list']['video_2']['backup_url_1']
  484. audio_url = video_info['videoResource']['normal']['video_list']['video_2']['backup_url_1']
  485. if len(video_url) % 3 == 1:
  486. video_url += '=='
  487. elif len(video_url) % 3 == 2:
  488. video_url += '='
  489. elif len(audio_url) % 3 == 1:
  490. audio_url += '=='
  491. elif len(audio_url) % 3 == 2:
  492. audio_url += '='
  493. video_url = base64.b64decode(video_url).decode('utf8')
  494. audio_url = base64.b64decode(audio_url).decode('utf8')
  495. video_width = video_info['videoResource']['normal']['video_list']['video_2']['vwidth']
  496. video_height = video_info['videoResource']['normal']['video_list']['video_2']['vheight']
  497. video_url_dict["video_url"] = video_url
  498. video_url_dict["audio_url"] = audio_url
  499. video_url_dict["video_width"] = video_width
  500. video_url_dict["video_height"] = video_height
  501. elif "video_list" in video_info['videoResource']['normal'] and 'video_1' in \
  502. video_info['videoResource']['normal']['video_list']:
  503. video_url = video_info['videoResource']['normal']['video_list']['video_1']['backup_url_1']
  504. audio_url = video_info['videoResource']['normal']['video_list']['video_1']['backup_url_1']
  505. if len(video_url) % 3 == 1:
  506. video_url += '=='
  507. elif len(video_url) % 3 == 2:
  508. video_url += '='
  509. elif len(audio_url) % 3 == 1:
  510. audio_url += '=='
  511. elif len(audio_url) % 3 == 2:
  512. audio_url += '='
  513. video_url = base64.b64decode(video_url).decode('utf8')
  514. audio_url = base64.b64decode(audio_url).decode('utf8')
  515. video_width = video_info['videoResource']['normal']['video_list']['video_1']['vwidth']
  516. video_height = video_info['videoResource']['normal']['video_list']['video_1']['vheight']
  517. video_url_dict["video_url"] = video_url
  518. video_url_dict["audio_url"] = audio_url
  519. video_url_dict["video_width"] = video_width
  520. video_url_dict["video_height"] = video_height
  521. elif 'dynamic_video' in video_info['videoResource']['normal'] \
  522. and 'dynamic_video_list' in video_info['videoResource']['normal']['dynamic_video'] \
  523. and 'dynamic_audio_list' in video_info['videoResource']['normal']['dynamic_video'] \
  524. and len(video_info['videoResource']['normal']['dynamic_video']['dynamic_video_list']) != 0 \
  525. and len(video_info['videoResource']['normal']['dynamic_video']['dynamic_audio_list']) != 0:
  526. video_url = video_info['videoResource']['normal']['dynamic_video']['dynamic_video_list'][-1][
  527. 'backup_url_1']
  528. audio_url = video_info['videoResource']['normal']['dynamic_video']['dynamic_audio_list'][-1][
  529. 'backup_url_1']
  530. if len(video_url) % 3 == 1:
  531. video_url += '=='
  532. elif len(video_url) % 3 == 2:
  533. video_url += '='
  534. elif len(audio_url) % 3 == 1:
  535. audio_url += '=='
  536. elif len(audio_url) % 3 == 2:
  537. audio_url += '='
  538. video_url = base64.b64decode(video_url).decode('utf8')
  539. audio_url = base64.b64decode(audio_url).decode('utf8')
  540. video_width = video_info['videoResource']['normal']['dynamic_video']['dynamic_video_list'][-1][
  541. 'vwidth']
  542. video_height = video_info['videoResource']['normal']['dynamic_video']['dynamic_video_list'][-1][
  543. 'vheight']
  544. video_url_dict["video_url"] = video_url
  545. video_url_dict["audio_url"] = audio_url
  546. video_url_dict["video_width"] = video_width
  547. video_url_dict["video_height"] = video_height
  548. else:
  549. video_url_dict["video_url"] = ''
  550. video_url_dict["audio_url"] = ''
  551. video_url_dict["video_width"] = 0
  552. video_url_dict["video_height"] = 0
  553. else:
  554. video_url_dict["video_url"] = ''
  555. video_url_dict["audio_url"] = ''
  556. video_url_dict["video_width"] = 0
  557. video_url_dict["video_height"] = 0
  558. return video_url_dict
  559. except Exception as e:
  560. Common.logger(log_type, crawler).error(f'get_video_url:{e}\n')
  561. @classmethod
  562. def get_video_info(cls, item_id):
  563. d_url = "http://a6.pstatp.com/article/full/11/1/{video_id}/{video_id}/1/0/?iid=3636030325&device_id=5787057242" \
  564. "&ac=wifi&channel=wandoujia&aid=13&app_name=news_article&version_code=532&version_name=5.3.2&device_platform" \
  565. "=android&ab_client=a1%2Cc2%2Ce1%2Cf2%2Cg2%2Cb3%2Cf4&abflag=3&ssmix=a&device_type=SM705" \
  566. "&device_brand=smartisan&os_api=19&os_version=4.4.2&uuid=864593021012562&openudid=e23a5ff037ef2d1a" \
  567. "&manifest_version_code=532&resolution=1080*1920&dpi=480&update_version_code=5320".format(
  568. video_id=item_id)
  569. res = requests.get(url=d_url, headers=random_user_agent('pc'), proxies=Common.tunnel_proxies())
  570. data = json.loads(res.text)['data']
  571. item_counter = data['h5_extra']['itemCell']['itemCounter']
  572. user_info = data['user_info']
  573. detail_info = data['video_detail_info']
  574. video_dict = {'video_title': data['title'],
  575. 'video_id': detail_info['video_id'],
  576. 'gid': data['group_id'],
  577. 'play_cnt': item_counter['videoWatchCount'],
  578. 'comment_cnt': item_counter['commentCount'],
  579. 'like_cnt': item_counter['diggCount'],
  580. 'share_cnt': item_counter['shareCount'],
  581. 'duration': data['video_duration'],
  582. 'publish_time_stamp': data['publish_time'],
  583. 'publish_time_str': time.strftime("%Y-%m-%d %H:%M:%S",
  584. time.localtime(data['publish_time'])),
  585. 'user_name': user_info['name'],
  586. 'user_id': user_info['user_id'],
  587. 'avatar_url': user_info['avatar_url'],
  588. 'cover_url': data['large_image']['url'].replace('\u0026', '&'),
  589. }
  590. return video_dict
  591. @classmethod
  592. def is_ruled(cls, log_type, crawler, video_dict, rule_dict):
  593. old_time = int(time.time()) - (3600 * 24 * rule_dict['publish_time'])
  594. if video_dict['publish_time_stamp'] <= old_time:
  595. return False
  596. elif video_dict['play_cnt'] <= rule_dict['play_cnt']:
  597. return False
  598. elif video_dict['duration'] < rule_dict['min_duration'] or video_dict['duration'] > rule_dict['max_duration']:
  599. return False
  600. else:
  601. return True
  602. @classmethod
  603. def get_videolist(cls, log_type, crawler, strategy, our_uid, search_word, oss_endpoint, env, machine):
  604. total_count = 0
  605. offset = 0
  606. while True:
  607. signature = cls.random_signature()
  608. url = "https://www.ixigua.com/api/searchv2/complex/{}/{}".format(
  609. quote(search_word), offset, signature)
  610. headers = {
  611. 'referer': 'https://www.ixigua.com/search/{}/?logTag=594535e3690f17a88cdb&tab_name=search'.format(
  612. quote(search_word)),
  613. 'cookie': 'ttwid=1|x_4RDmVTqp6BQ5Xy5AnuCZCQdDyDxv-fnMVWzj19VU0|1678693556|883092b75667cbcc48cbcc4b3b44d060aa205ef26c7640dc672d0cc50ddf0be9',
  614. 'user-agent': get_random_user_agent('pc'),
  615. }
  616. try:
  617. res = requests.request("GET", url, headers=headers, proxies=Common.tunnel_proxies())
  618. search_list = res.json()['data']['data']
  619. except Exception as e:
  620. search_list = []
  621. if not search_list:
  622. return
  623. for video_info in search_list:
  624. v_type = video_info['type']
  625. item_id = video_info['data']['group_id']
  626. rule_dict = cls.get_rule(log_type, crawler)
  627. video_dict = {}
  628. if v_type == 'video':
  629. try:
  630. video_dict = cls.get_video_info(item_id)
  631. video_url_dict = cls.get_video_url(log_type, crawler, video_dict['gid'])
  632. video_dict['video_width'] = video_url_dict["video_width"]
  633. video_dict['video_height'] = video_url_dict["video_height"]
  634. video_dict['audio_url'] = video_url_dict["audio_url"]
  635. video_dict['video_url'] = video_url_dict["video_url"]
  636. video_dict['session'] = signature
  637. except Exception as e:
  638. Common.logger(log_type, crawler).error(f'视频:{item_id},没有获取到视频详情,原因:{e}')
  639. continue
  640. elif v_type == 'pseries':
  641. p_url = "https://www.ixigua.com/api/videov2/pseries_more_v2?pSeriesId={}&rank=0&tailCount=30&aid=1768&msToken=wHEafKFLx0k3hihOPbhXYNsfMBxWiq2AB0K5R-34kEFixyq3ATi_DuXbL4Q47J9C2uK2zgWItMa1g2yc4FyDxM4dMijmSdwF4c4T8sSmOkoOI0wGzeEcPw==&X-Bogus=DFSzswVOzdUANG3ItaVHYr7TlqCv&_signature=_02B4Z6wo00001vB6l3QAAIDBZKzMeTihTmbwepPAANgh1Ai3JgFFo4e6anoezmBEpHfEMEYlWISGhXI-QKfev4N-2bwgXsHOuNGLnOsGqMbANIjFPh7Yj6OakQWrkbACenlv0P-arswtB6Zn45".format(
  642. item_id)
  643. p_headers = {
  644. 'referer': 'https://www.ixigua.com/{}?series_flow=1&logTag=cfec9d927da968feff89'.format(
  645. item_id),
  646. 'user-agent': get_random_user_agent('pc'),
  647. }
  648. try:
  649. p_res = requests.request("GET", p_url, headers=p_headers,
  650. proxies=Common.tunnel_proxies()).json()
  651. except Exception as e:
  652. Common.logger(log_type, crawler).error(f'合集:{item_id},没有获取到合集详情,原因:{e}')
  653. continue
  654. for video in p_res['data']:
  655. item_id = video['item_id']
  656. try:
  657. video_dict = cls.get_video_info(item_id)
  658. if cls.is_ruled(log_type, crawler, video_dict, rule_dict):
  659. video_url_dict = cls.get_video_url(log_type, crawler, video_dict['gid'])
  660. video_dict['video_width'] = video_url_dict["video_width"]
  661. video_dict['video_height'] = video_url_dict["video_height"]
  662. video_dict['audio_url'] = video_url_dict["audio_url"]
  663. video_dict['video_url'] = video_url_dict["video_url"]
  664. video_dict['session'] = signature
  665. break
  666. else:
  667. continue
  668. except Exception as e:
  669. Common.logger(log_type, crawler).error(f'视频:{item_id},没有获取到视频详情,原因:{e}')
  670. continue
  671. if not cls.is_ruled(log_type, crawler, video_dict, rule_dict):
  672. continue
  673. for k, v in video_dict.items():
  674. Common.logger(log_type, crawler).info(f"{k}:{v}")
  675. # print(f'title:{video_dict["video_title"]},gid:{video_dict["gid"]},offset:{offset}, total:{total_count}')
  676. cls.download_publish(
  677. search_word=search_word,
  678. log_type=log_type,
  679. crawler=crawler,
  680. video_dict=video_dict,
  681. rule_dict=rule_dict,
  682. strategy=strategy,
  683. our_uid=our_uid,
  684. oss_endpoint=oss_endpoint,
  685. env=env,
  686. machine=machine
  687. )
  688. total_count += 1
  689. if total_count >= 30:
  690. return
  691. offset += 10
  692. @classmethod
  693. def repeat_video(cls, log_type, crawler, video_id, env, machine):
  694. sql = f""" select * from crawler_video where platform="{cls.platform}" and out_video_id="{video_id}"; """
  695. repeat_video = MysqlHelper.get_values(log_type, crawler, sql, env, machine)
  696. return len(repeat_video)
  697. # 下载 / 上传
  698. @classmethod
  699. def download_publish(cls, log_type, crawler, search_word, strategy, video_dict, rule_dict, our_uid, oss_endpoint,
  700. env, machine):
  701. try:
  702. if cls.repeat_video(log_type, crawler, video_dict['video_id'], env, machine) != 0:
  703. Common.logger(log_type, crawler).info('视频已下载\n')
  704. return
  705. Common.logger(log_type, crawler).info('视频已下载\n')
  706. Common.download_method(log_type=log_type, crawler=crawler, text='xigua_video',
  707. title=video_dict['video_title'], url=video_dict['video_url'])
  708. # 下载音频
  709. Common.download_method(log_type=log_type, crawler=crawler, text='xigua_audio',
  710. title=video_dict['video_title'], url=video_dict['audio_url'])
  711. # 合成音视频
  712. Common.video_compose(log_type=log_type, crawler=crawler,
  713. video_dir=f"./{crawler}/videos/{video_dict['video_title']}")
  714. ffmpeg_dict = Common.ffmpeg(log_type, crawler,
  715. f"./{crawler}/videos/{video_dict['video_title']}/video.mp4")
  716. if ffmpeg_dict is None or ffmpeg_dict['size'] == 0:
  717. Common.logger(log_type, crawler).warning(f"下载的视频无效,已删除\n")
  718. # 删除视频文件夹
  719. shutil.rmtree(f"./{crawler}/videos/{video_dict['video_title']}")
  720. return
  721. # 下载封面
  722. Common.download_method(log_type=log_type, crawler=crawler, text='cover',
  723. title=video_dict['video_title'], url=video_dict['cover_url'])
  724. # 保存视频信息至txt
  725. Common.save_video_info(log_type=log_type, crawler=crawler, video_dict=video_dict)
  726. # 上传视频
  727. Common.logger(log_type, crawler).info("开始上传视频...")
  728. our_video_id = Publish.upload_and_publish(log_type=log_type,
  729. crawler=crawler,
  730. strategy=strategy,
  731. our_uid=our_uid,
  732. env=env,
  733. oss_endpoint=oss_endpoint)
  734. if env == 'dev':
  735. our_video_link = f"https://testadmin.piaoquantv.com/cms/post-detail/{our_video_id}/info"
  736. else:
  737. our_video_link = f"https://admin.piaoquantv.com/cms/post-detail/{our_video_id}/info"
  738. Common.logger(log_type, crawler).info("视频上传完成")
  739. if our_video_id is None:
  740. # 删除视频文件夹
  741. shutil.rmtree(f"./{crawler}/videos/{video_dict['video_title']}")
  742. return
  743. # 视频写入飞书
  744. Feishu.insert_columns(log_type, 'xigua', "BUNvGC", "ROWS", 1, 2)
  745. upload_time = int(time.time())
  746. values = [[
  747. search_word,
  748. time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(upload_time)),
  749. "关键词搜索",
  750. video_dict['video_title'],
  751. str(video_dict['video_id']),
  752. our_video_link,
  753. video_dict['gid'],
  754. video_dict['play_cnt'],
  755. video_dict['comment_cnt'],
  756. video_dict['like_cnt'],
  757. video_dict['share_cnt'],
  758. video_dict['duration'],
  759. str(video_dict['video_width']) + '*' + str(video_dict['video_height']),
  760. video_dict['publish_time_str'],
  761. video_dict['user_name'],
  762. video_dict['user_id'],
  763. video_dict['avatar_url'],
  764. video_dict['cover_url'],
  765. video_dict['video_url'],
  766. video_dict['audio_url']]]
  767. time.sleep(1)
  768. Feishu.update_values(log_type, 'xigua', "BUNvGC", "E2:Z2", values)
  769. Common.logger(log_type, crawler).info(f"视频已保存至云文档\n")
  770. # 视频信息保存数据库
  771. insert_sql = f""" insert into crawler_video(video_id,
  772. user_id,
  773. out_user_id,
  774. platform,
  775. strategy,
  776. out_video_id,
  777. video_title,
  778. cover_url,
  779. video_url,
  780. duration,
  781. publish_time,
  782. play_cnt,
  783. crawler_rule,
  784. width,
  785. height)
  786. values({our_video_id},
  787. {our_uid},
  788. "{video_dict['user_id']}",
  789. "{cls.platform}",
  790. "定向爬虫策略",
  791. "{video_dict['video_id']}",
  792. "{video_dict['video_title']}",
  793. "{video_dict['cover_url']}",
  794. "{video_dict['video_url']}",
  795. {int(video_dict['duration'])},
  796. "{video_dict['publish_time_str']}",
  797. {int(video_dict['play_cnt'])},
  798. '{json.dumps(rule_dict)}',
  799. {int(video_dict['video_width'])},
  800. {int(video_dict['video_height'])}) """
  801. Common.logger(log_type, crawler).info(f"insert_sql:{insert_sql}")
  802. MysqlHelper.update_values(log_type, crawler, insert_sql, env, machine)
  803. Common.logger(log_type, crawler).info('视频信息插入数据库成功!\n')
  804. except Exception as e:
  805. Common.logger(log_type, crawler).error(f'download_publish异常:{e}\n')
  806. @classmethod
  807. def get_search_videos(cls, log_type, crawler, strategy, oss_endpoint, env, machine):
  808. try:
  809. user_list = cls.get_user_list(log_type=log_type, crawler=crawler, sheetid="SSPNPW", env=env,
  810. machine=machine)
  811. for user in user_list:
  812. search_word = user["search_word"]
  813. our_uid = user["our_uid"]
  814. Common.logger(log_type, crawler).info(f"开始抓取 {search_word} 用户主页视频\n")
  815. cls.get_videolist(log_type=log_type,
  816. crawler=crawler,
  817. strategy=strategy,
  818. our_uid=our_uid,
  819. search_word=search_word,
  820. oss_endpoint=oss_endpoint,
  821. env=env,
  822. machine=machine)
  823. except Exception as e:
  824. Common.logger(log_type, crawler).error(f"get_search_videos:{e}\n")
  825. if __name__ == '__main__':
  826. # print(Follow.get_signature("follow", "xigua", "95420624045", "local"))
  827. Search.get_search_videos('search', 'xigua', 'xigua_search', 'out', 'dev', 'local')
  828. # Follow.get_videolist(log_type="follow",
  829. # crawler="xigua",
  830. # strategy="定向爬虫策略",
  831. # our_uid="6267141",
  832. # out_uid="95420624045",
  833. # oss_endpoint="out",
  834. # env="dev",
  835. # machine="local")
  836. # print(Follow.random_signature())
  837. # rule = Follow.get_rule("follow", "xigua")
  838. # print(type(rule))
  839. # print(type(json.dumps(rule)))
  840. # print(json.dumps(rule))
  841. pass