gongzhonghao_follow.py 28 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522
  1. # -*- coding: utf-8 -*-
  2. # @Author: wangkun
  3. # @Time: 2023/3/28
  4. import datetime
  5. import difflib
  6. import json
  7. import os
  8. import random
  9. import shutil
  10. import sys
  11. import time
  12. from hashlib import md5
  13. import requests
  14. import urllib3
  15. from selenium.webdriver import DesiredCapabilities
  16. from selenium.webdriver.chrome.service import Service
  17. from selenium.webdriver.common.by import By
  18. from selenium import webdriver
  19. sys.path.append(os.getcwd())
  20. from common.common import Common
  21. from common.feishu import Feishu
  22. from common.public import filter_word
  23. from common.publish import Publish
  24. from common.scheduling_db import MysqlHelper
  25. class GongzhonghaoFollow:
  26. # 翻页参数
  27. begin = 0
  28. platform = "公众号"
  29. # 基础门槛规则
  30. @staticmethod
  31. def download_rule(video_dict):
  32. """
  33. 下载视频的基本规则
  34. :param video_dict: 视频信息,字典格式
  35. :return: 满足规则,返回 True;反之,返回 False
  36. """
  37. # 视频时长 20秒 - 45 分钟
  38. if 60 * 45 >= int(float(video_dict['duration'])) >= 20:
  39. # 宽或高
  40. if int(video_dict['video_width']) >= 0 or int(video_dict['video_height']) >= 0:
  41. return True
  42. else:
  43. return False
  44. else:
  45. return False
  46. @classmethod
  47. def title_like(cls, log_type, crawler, title, env):
  48. select_sql = f""" select * from crawler_video where platform="公众号" """
  49. video_list = MysqlHelper.get_values(log_type, crawler, select_sql, env, action="")
  50. if len(video_list) == 0:
  51. return None
  52. for video_dict in video_list:
  53. video_title = video_dict["video_title"]
  54. if difflib.SequenceMatcher(None, title, video_title).quick_ratio() >= 0.8:
  55. return True
  56. else:
  57. pass
  58. # 获取 token
  59. @classmethod
  60. def get_token(cls, log_type, crawler):
  61. while True:
  62. try:
  63. sheet = Feishu.get_values_batch(log_type, "gongzhonghao", "OjyJqs")
  64. if sheet is None:
  65. time.sleep(1)
  66. continue
  67. token = sheet[0][1]
  68. cookie = sheet[1][1]
  69. gzh_name = sheet[2][1]
  70. gzh_time = sheet[3][1]
  71. token_dict = {'token': token, 'cookie': cookie, 'gzh_name': gzh_name, 'gzh_time': gzh_time}
  72. print(type(token_dict['gzh_time']))
  73. print(token_dict['gzh_time'])
  74. return token_dict
  75. except Exception as e:
  76. Common.logger(log_type, crawler).error(f"get_cookie_token异常:{e}\n")
  77. # 获取用户 fakeid
  78. @classmethod
  79. def get_fakeid(cls, log_type, crawler, user, index):
  80. try:
  81. while True:
  82. token_dict = cls.get_token(log_type, crawler)
  83. url = "https://mp.weixin.qq.com/cgi-bin/searchbiz?"
  84. headers = {
  85. "accept": "*/*",
  86. "accept-encoding": "gzip, deflate, br",
  87. "accept-language": "zh-CN,zh;q=0.9",
  88. "referer": "https://mp.weixin.qq.com/cgi-bin/appmsg?"
  89. "t=media/appmsg_edit_v2&action=edit&isNew=1"
  90. "&type=77&createType=5&token=1011071554&lang=zh_CN",
  91. 'sec-ch-ua': '" Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"',
  92. "sec-ch-ua-mobile": "?0",
  93. "sec-ch-ua-platform": '"Windows"',
  94. "sec-fetch-dest": "empty",
  95. "sec-fetch-mode": "cors",
  96. "sec-fetch-site": "same-origin",
  97. "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
  98. " (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36",
  99. "x-requested-with": "XMLHttpRequest",
  100. 'cookie': token_dict['cookie'],
  101. }
  102. params = {
  103. "action": "search_biz",
  104. "begin": "0",
  105. "count": "5",
  106. "query": str(user),
  107. "token": token_dict['token'],
  108. "lang": "zh_CN",
  109. "f": "json",
  110. "ajax": "1",
  111. }
  112. urllib3.disable_warnings()
  113. r = requests.get(url=url, headers=headers, params=params, verify=False)
  114. if r.json()["base_resp"]["err_msg"] == "invalid session":
  115. Common.logger(log_type, crawler).info(f"status_code:{r.status_code}")
  116. Common.logger(log_type, crawler).warning(f"get_fakeid:{r.text}\n")
  117. if 20 >= datetime.datetime.now().hour >= 10:
  118. Feishu.bot(log_type, crawler, f"token_1:{token_dict['gzh_name']}\n更换日期:{token_dict['gzh_time']}\n过期啦,请扫码更换token\nhttps://mp.weixin.qq.com/")
  119. time.sleep(60 * 10)
  120. continue
  121. if r.json()["base_resp"]["err_msg"] == "freq control":
  122. Common.logger(log_type, crawler).info(f"status_code:{r.status_code}")
  123. Common.logger(log_type, crawler).warning(f"get_fakeid:{r.text}\n")
  124. if 20 >= datetime.datetime.now().hour >= 10:
  125. Feishu.bot(log_type, crawler, f"公众号_1:{token_dict['gzh_name']}\n更换日期:{token_dict['gzh_time']}\n频控啦,请扫码更换其他公众号token\nhttps://mp.weixin.qq.com/")
  126. time.sleep(60 * 10)
  127. continue
  128. if "list" not in r.json() or len(r.json()["list"]) == 0:
  129. Common.logger(log_type, crawler).info(f"status_code:{r.status_code}")
  130. Common.logger(log_type, crawler).warning(f"get_fakeid:{r.text}\n")
  131. if 20 >= datetime.datetime.now().hour >= 10:
  132. Feishu.bot(log_type, crawler, f"公众号_1:{token_dict['gzh_name']}\n更换日期:{token_dict['gzh_time']}\n频控啦,请扫码更换其他公众号token\nhttps://mp.weixin.qq.com/")
  133. time.sleep(60 * 10)
  134. continue
  135. fakeid = r.json()["list"][int(index) - 1]["fakeid"]
  136. head_url = r.json()["list"][int(index) - 1]["round_head_img"]
  137. fakeid_dict = {'fakeid': fakeid, 'head_url': head_url}
  138. return fakeid_dict
  139. except Exception as e:
  140. Common.logger(log_type, crawler).error(f"get_fakeid异常:{e}\n")
  141. # 获取腾讯视频下载链接
  142. @classmethod
  143. def get_tencent_video_url(cls, log_type, crawler, video_id):
  144. try:
  145. url = 'https://vv.video.qq.com/getinfo?vids=' + str(video_id) + '&platform=101001&charge=0&otype=json'
  146. response = requests.get(url=url).text.replace('QZOutputJson=', '').replace('"};', '"}')
  147. response = json.loads(response)
  148. url = response['vl']['vi'][0]['ul']['ui'][0]['url']
  149. fvkey = response['vl']['vi'][0]['fvkey']
  150. video_url = url + str(video_id) + '.mp4?vkey=' + fvkey
  151. return video_url
  152. except Exception as e:
  153. Common.logger(log_type, crawler).error(f"get_tencent_video_url异常:{e}\n")
  154. @classmethod
  155. def get_video_url(cls, log_type, crawler, article_url, env):
  156. try:
  157. # 打印请求配置
  158. ca = DesiredCapabilities.CHROME
  159. ca["goog:loggingPrefs"] = {"performance": "ALL"}
  160. # 不打开浏览器运行
  161. chrome_options = webdriver.ChromeOptions()
  162. chrome_options.add_argument("headless")
  163. chrome_options.add_argument(
  164. f'user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.79 Safari/537.36')
  165. chrome_options.add_argument("--no-sandbox")
  166. # driver初始化
  167. if env == "prod":
  168. driver = webdriver.Chrome(desired_capabilities=ca, options=chrome_options)
  169. else:
  170. driver = webdriver.Chrome(desired_capabilities=ca, options=chrome_options, service=Service(
  171. '/Users/wangkun/Downloads/chromedriver/chromedriver_v111/chromedriver'))
  172. driver.implicitly_wait(10)
  173. # Common.logger(log_type, crawler).info('打开文章链接')
  174. driver.get(article_url)
  175. time.sleep(1)
  176. if len(driver.find_elements(By.XPATH, '//div[@class="js_video_poster video_poster"]/*[2]')) != 0:
  177. video_url = driver.find_element(
  178. By.XPATH, '//div[@class="js_video_poster video_poster"]/*[2]').get_attribute('src')
  179. elif len(driver.find_elements(By.XPATH, '//span[@class="js_tx_video_container"]/*[1]')) != 0:
  180. iframe = driver.find_element(By.XPATH, '//span[@class="js_tx_video_container"]/*[1]').get_attribute(
  181. 'src')
  182. video_id = iframe.split('vid=')[-1].split('&')[0]
  183. video_url = cls.get_tencent_video_url(log_type, crawler, video_id)
  184. else:
  185. video_url = 0
  186. driver.quit()
  187. return video_url
  188. except Exception as e:
  189. Common.logger(log_type, crawler).info(f'get_video_url异常:{e}\n')
  190. # 获取文章列表
  191. @classmethod
  192. def get_videoList(cls, log_type, crawler, user, index, oss_endpoint, env):
  193. try:
  194. while True:
  195. token_dict = cls.get_token(log_type, crawler)
  196. fakeid_dict = cls.get_fakeid(log_type, crawler, user, index)
  197. url = "https://mp.weixin.qq.com/cgi-bin/appmsg?"
  198. headers = {
  199. "accept": "*/*",
  200. "accept-encoding": "gzip, deflate, br",
  201. "accept-language": "zh-CN,zh;q=0.9",
  202. "referer": "https://mp.weixin.qq.com/cgi-bin/appmsg?"
  203. "t=media/appmsg_edit_v2&action=edit&isNew=1"
  204. "&type=77&createType=5&token=" + str(token_dict['token']) + "&lang=zh_CN",
  205. 'sec-ch-ua': '" Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"',
  206. "sec-ch-ua-mobile": "?0",
  207. "sec-ch-ua-platform": '"Windows"',
  208. "sec-fetch-dest": "empty",
  209. "sec-fetch-mode": "cors",
  210. "sec-fetch-site": "same-origin",
  211. "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
  212. " (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36",
  213. "x-requested-with": "XMLHttpRequest",
  214. 'cookie': token_dict['cookie'],
  215. }
  216. params = {
  217. "action": "list_ex",
  218. "begin": str(cls.begin),
  219. "count": "5",
  220. "fakeid": fakeid_dict['fakeid'],
  221. "type": "9",
  222. "query": "",
  223. "token": str(token_dict['token']),
  224. "lang": "zh_CN",
  225. "f": "json",
  226. "ajax": "1",
  227. }
  228. urllib3.disable_warnings()
  229. r = requests.get(url=url, headers=headers, params=params, verify=False)
  230. if r.json()["base_resp"]["err_msg"] == "invalid session":
  231. Common.logger(log_type, crawler).info(f"status_code:{r.status_code}")
  232. Common.logger(log_type, crawler).info(f"get_videoList:{r.text}\n")
  233. if 20 >= datetime.datetime.now().hour >= 10:
  234. Feishu.bot(log_type, crawler, f"token_1:{token_dict['gzh_name']}\n更换日期:{token_dict['gzh_time']}\n过期啦,请扫码更换token\nhttps://mp.weixin.qq.com/")
  235. time.sleep(60 * 10)
  236. continue
  237. if r.json()["base_resp"]["err_msg"] == "freq control":
  238. Common.logger(log_type, crawler).info(f"status_code:{r.status_code}")
  239. Common.logger(log_type, crawler).warning(f"get_videoList:{r.text}\n")
  240. if 20 >= datetime.datetime.now().hour >= 10:
  241. Feishu.bot(log_type, crawler,f"公众号_1:{token_dict['gzh_name']}\n更换日期:{token_dict['gzh_time']}\n频控啦,请扫码更换其他公众号token\nhttps://mp.weixin.qq.com/")
  242. time.sleep(60 * 10)
  243. continue
  244. if 'app_msg_list' not in r.json():
  245. Common.logger(log_type, crawler).info(f"status_code:{r.status_code}")
  246. Common.logger(log_type, crawler).warning(f"get_videoList:{r.text}\n")
  247. if 20 >= datetime.datetime.now().hour >= 10:
  248. Feishu.bot(log_type, crawler, f"公众号_1:{token_dict['gzh_name']}\n更换日期:{token_dict['gzh_time']}\n频控啦,请扫码更换其他公众号token\nhttps://mp.weixin.qq.com/")
  249. time.sleep(60 * 10)
  250. continue
  251. if len(r.json()['app_msg_list']) == 0:
  252. Common.logger(log_type, crawler).info('没有更多视频了\n')
  253. return
  254. else:
  255. cls.begin += 5
  256. app_msg_list = r.json()['app_msg_list']
  257. for article_url in app_msg_list:
  258. # title
  259. if 'title' in article_url:
  260. title = article_url['title'].replace('/', '').replace('\n', '') \
  261. .replace('.', '').replace('“', '').replace('”', '').replace(' ', '')
  262. else:
  263. title = 0
  264. # aid
  265. if 'aid' in article_url:
  266. aid = article_url['aid']
  267. else:
  268. aid = 0
  269. # create_time
  270. if 'create_time' in article_url:
  271. create_time = article_url['create_time']
  272. else:
  273. create_time = 0
  274. publish_time_stamp = int(create_time)
  275. publish_time_str = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(publish_time_stamp))
  276. avatar_url = fakeid_dict['head_url']
  277. # cover_url
  278. if 'cover' in article_url:
  279. cover_url = article_url['cover']
  280. else:
  281. cover_url = 0
  282. # article_url
  283. if 'link' in article_url:
  284. article_url = article_url['link']
  285. else:
  286. article_url = 0
  287. video_url = cls.get_video_url(log_type, crawler, article_url, env)
  288. video_dict = {
  289. 'video_id': aid,
  290. 'video_title': title,
  291. 'publish_time_stamp': publish_time_stamp,
  292. 'publish_time_str': publish_time_str,
  293. 'user_name': user,
  294. 'play_cnt': 0,
  295. 'comment_cnt': 0,
  296. 'like_cnt': 0,
  297. 'share_cnt': 0,
  298. 'user_id': fakeid_dict['fakeid'],
  299. 'avatar_url': avatar_url,
  300. 'cover_url': cover_url,
  301. 'article_url': article_url,
  302. 'video_url': video_url,
  303. 'session': f'gongzhonghao-follow-{int(time.time())}'
  304. }
  305. for k, v in video_dict.items():
  306. Common.logger(log_type, crawler).info(f"{k}:{v}")
  307. if int(time.time()) - publish_time_stamp > 3600 * 24 * 3:
  308. Common.logger(log_type, crawler).info(f'发布时间{publish_time_str} > 3 天\n')
  309. cls.begin = 0
  310. return
  311. cls.download_publish(log_type, crawler, video_dict, oss_endpoint, env)
  312. Common.logger(log_type, crawler).info('随机休眠 60*5, 60*10 秒\n')
  313. time.sleep(random.randint(60*5, 60*10))
  314. except Exception as e:
  315. Common.logger(log_type, crawler).error(f"get_videoList异常:{e}\n")
  316. @classmethod
  317. def repeat_video(cls, log_type, crawler, video_id, env):
  318. sql = f""" select * from crawler_video where platform="公众号" and out_video_id="{video_id}"; """
  319. repeat_video = MysqlHelper.get_values(log_type, crawler, sql, env)
  320. return len(repeat_video)
  321. # 下载/上传
  322. @classmethod
  323. def download_publish(cls, log_type, crawler, video_dict, oss_endpoint, env):
  324. try:
  325. if video_dict['article_url'] == 0 or video_dict['video_url'] == 0:
  326. Common.logger(log_type, crawler).info("文章涉嫌违反相关法律法规和政策\n")
  327. # 标题敏感词过滤
  328. elif any(word if word in video_dict['video_title'] else False for word in
  329. filter_word(log_type, crawler, "公众号", env)) is True:
  330. Common.logger(log_type, crawler).info("标题已中过滤词\n")
  331. # 已下载判断
  332. elif cls.repeat_video(log_type, crawler, video_dict['video_id'], env) != 0:
  333. Common.logger(log_type, crawler).info("视频已下载\n")
  334. # 标题相似度
  335. elif cls.title_like(log_type, crawler, video_dict['video_title'], env) is True:
  336. Common.logger(log_type, crawler).info(f'标题相似度>=80%:{video_dict["video_title"]}\n')
  337. else:
  338. # 下载视频
  339. Common.download_method(log_type=log_type, crawler=crawler, text="video",
  340. title=video_dict["video_title"], url=video_dict["video_url"])
  341. md_title = md5(video_dict['video_title'].encode('utf8')).hexdigest()
  342. # 获取视频时长
  343. ffmpeg_dict = Common.ffmpeg(log_type, crawler,
  344. f"./{crawler}/videos/{video_dict['video_title']}/video.mp4")
  345. if ffmpeg_dict is None:
  346. # 删除视频文件夹
  347. shutil.rmtree(f"./{crawler}/videos/{md_title}")
  348. Common.logger(log_type, crawler).info("视频size=0,删除成功\n")
  349. return
  350. video_dict["video_width"] = ffmpeg_dict["width"]
  351. video_dict["video_height"] = ffmpeg_dict["height"]
  352. video_dict["duration"] = ffmpeg_dict["duration"]
  353. video_size = ffmpeg_dict["size"]
  354. Common.logger(log_type, crawler).info(f'video_width:{video_dict["video_width"]}')
  355. Common.logger(log_type, crawler).info(f'video_height:{video_dict["video_height"]}')
  356. Common.logger(log_type, crawler).info(f'duration:{video_dict["duration"]}')
  357. Common.logger(log_type, crawler).info(f'video_size:{video_size}')
  358. # 视频size=0,直接删除
  359. if int(video_size) == 0 or cls.download_rule(video_dict) is False:
  360. # 删除视频文件夹
  361. shutil.rmtree(f"./{crawler}/videos/{md_title}")
  362. Common.logger(log_type, crawler).info("视频size=0,删除成功\n")
  363. return
  364. # 下载封面
  365. Common.download_method(log_type=log_type, crawler=crawler, text="cover",
  366. title=video_dict["video_title"], url=video_dict["cover_url"])
  367. # 保存视频信息至 "./videos/{video_title}/info.txt"
  368. Common.save_video_info(log_type=log_type, crawler=crawler, video_dict=video_dict)
  369. # 上传视频
  370. Common.logger(log_type, crawler).info("开始上传视频...")
  371. strategy = "定向爬虫策略"
  372. our_video_id = Publish.upload_and_publish(log_type=log_type,
  373. crawler=crawler,
  374. strategy=strategy,
  375. our_uid="follow",
  376. oss_endpoint=oss_endpoint,
  377. env=env)
  378. if env == 'prod':
  379. our_video_link = f"https://admin.piaoquantv.com/cms/post-detail/{str(our_video_id)}/info"
  380. else:
  381. our_video_link = f"https://testadmin.piaoquantv.com/cms/post-detail/{str(our_video_id)}/info"
  382. Common.logger(log_type, crawler).info("视频上传完成")
  383. if our_video_id is None:
  384. # 删除视频文件夹
  385. shutil.rmtree(f"./{crawler}/videos/{video_dict['video_title']}")
  386. return
  387. # 视频信息保存数据库
  388. rule_dict = {
  389. "duration": {"min": 20, "max": 45 * 60},
  390. "publish_day": {"min": 3}
  391. }
  392. insert_sql = f""" insert into crawler_video(video_id,
  393. out_user_id,
  394. platform,
  395. strategy,
  396. out_video_id,
  397. video_title,
  398. cover_url,
  399. video_url,
  400. duration,
  401. publish_time,
  402. play_cnt,
  403. crawler_rule,
  404. width,
  405. height)
  406. values({our_video_id},
  407. "{video_dict['user_id']}",
  408. "{cls.platform}",
  409. "定向爬虫策略",
  410. "{video_dict['video_id']}",
  411. "{video_dict['video_title']}",
  412. "{video_dict['cover_url']}",
  413. "{video_dict['video_url']}",
  414. {int(video_dict['duration'])},
  415. "{video_dict['publish_time_str']}",
  416. {int(video_dict['play_cnt'])},
  417. '{json.dumps(rule_dict)}',
  418. {int(video_dict['video_width'])},
  419. {int(video_dict['video_height'])}) """
  420. Common.logger(log_type, crawler).info(f"insert_sql:{insert_sql}")
  421. MysqlHelper.update_values(log_type, crawler, insert_sql, env)
  422. Common.logger(log_type, crawler).info('视频信息插入数据库成功!')
  423. # 视频写入飞书
  424. Feishu.insert_columns(log_type, crawler, "47e39d", "ROWS", 1, 2)
  425. # 视频ID工作表,首行写入数据
  426. upload_time = int(time.time())
  427. values = [[time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(upload_time)),
  428. "用户主页",
  429. video_dict['video_title'],
  430. video_dict['video_id'],
  431. our_video_link,
  432. int(video_dict['duration']),
  433. f"{video_dict['video_width']}*{video_dict['video_height']}",
  434. video_dict['publish_time_str'],
  435. video_dict['user_name'],
  436. video_dict['user_id'],
  437. video_dict['avatar_url'],
  438. video_dict['cover_url'],
  439. video_dict['article_url'],
  440. video_dict['video_url']]]
  441. time.sleep(0.5)
  442. Feishu.update_values(log_type, crawler, "47e39d", "F2:Z2", values)
  443. Common.logger(log_type, crawler).info('视频下载/上传成功\n')
  444. except Exception as e:
  445. Common.logger(log_type, crawler).error(f"download_publish异常:{e}\n")
  446. @classmethod
  447. def get_users(cls):
  448. # user_sheet = Feishu.get_values_batch("follow", 'gongzhonghao', 'Bzv72P')
  449. # user_list = []
  450. # for i in range(1, 41):
  451. # user_name = user_sheet[i][0]
  452. # index = user_sheet[i][1]
  453. # user_dict = {
  454. # "user_name": user_name,
  455. # "index": index,
  456. # }
  457. # user_list.append(user_dict)
  458. # print(len(user_list))
  459. # print(user_list)
  460. user_list = [{'user_name': '香音难忘', 'index': 1}, {'user_name': '墨儿心灵驿站', 'index': 1}, {'user_name': '荒烟茶生', 'index': 1}, {'user_name': '幸福花朵', 'index': 1}, {'user_name': '我的节日祝福', 'index': 1}, {'user_name': '生活创意妙招', 'index': 1}, {'user_name': '二大妈有话说', 'index': 1}, {'user_name': '医路健康美食', 'index': 1}, {'user_name': '老年相知相伴', 'index': 1}, {'user_name': '一争', 'index': 1}, {'user_name': '老年企退群', 'index': 1}, {'user_name': '消逝的哨声', 'index': 1}, {'user_name': '一颗打破石头的蛋', 'index': 1}, {'user_name': '叩问苍穹荒烟茶生', 'index': 1}, {'user_name': '布衣星火', 'index': 1}, {'user_name': '叩问苍穹', 'index': 1}, {'user_name': '微观调查', 'index': 2}, {'user_name': '传统节日祝福', 'index': 1}, {'user_name': '因和德尚', 'index': 1}, {'user_name': '飨宴心灵', 'index': 1}, {'user_name': '朝闻解局', 'index': 1}, {'user_name': '远见光芒', 'index': 1}, {'user_name': '墨儿微刊', 'index': 1}, {'user_name': '博爱论', 'index': 1}, {'user_name': '张大春讲堂', 'index': 1}, {'user_name': ' 司马南频道', 'index': 1}, {'user_name': '音乐小镇', 'index': 1}, {'user_name': '节日祝福365', 'index': 1}, {'user_name': '动画音乐相册', 'index': 1}, {'user_name': '音乐动漫相册', 'index': 1}, {'user_name': '早点谈健康', 'index': 1}, {'user_name': '早点谈养生', 'index': 1}, {'user_name': '早点谈养身', 'index': 1}, {'user_name': '医道谈养身', 'index': 1}, {'user_name': '中老年谈养身', 'index': 1}, {'user_name': '尼古拉斯瞭望', 'index': 1}, {'user_name': '奇易时光百姓的福音', 'index': 1}, {'user_name': '寰宇时光', 'index': 1}, {'user_name': '红兴文化公苑', 'index': 1}, {'user_name': '早点音乐', 'index': 1}]
  461. return user_list
  462. @classmethod
  463. def get_all_videos(cls, log_type, crawler, oss_endpoint, env):
  464. user_list = cls.get_users()
  465. for user_dict in user_list:
  466. try:
  467. user_name = user_dict['user_name']
  468. index = user_dict['index']
  469. Common.logger(log_type, crawler).info(f'获取 {user_name} 公众号视频\n')
  470. cls.get_videoList(log_type, crawler, user_name, index, oss_endpoint, env)
  471. cls.begin = 0
  472. Common.logger(log_type, crawler).info('随机休眠 60*5, 60*10 秒\n')
  473. time.sleep(random.randint(60*5, 60*10))
  474. except Exception as e:
  475. Common.logger(log_type, crawler).info(f'get_all_videos异常:{e}\n')
  476. if __name__ == "__main__":
  477. GongzhonghaoFollow.get_token(log_type="follow", crawler="gongzhonghao")
  478. # GongzhonghaoFollow.get_users()
  479. # GongzhonghaoFollow.get_videoList(log_type="follow",
  480. # crawler="gongzhonghao",
  481. # user="香音难忘",
  482. # index=1,
  483. # oss_endpoint="out",
  484. # env="dev")
  485. pass