xiaoniangao_plus_offliner.py 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346
  1. # -*- coding: utf-8 -*-
  2. # @Time: 2023/10/31
  3. import json
  4. import os
  5. from datetime import date, timedelta
  6. import requests
  7. import sys
  8. import time
  9. from hashlib import md5
  10. from appium import webdriver
  11. from appium.webdriver.extensions.android.nativekey import AndroidKey
  12. from appium.webdriver.webdriver import WebDriver
  13. from bs4 import BeautifulSoup
  14. from selenium.common import NoSuchElementException
  15. from selenium.webdriver.common.by import By
  16. sys.path.append(os.getcwd())
  17. from common.common import Common
  18. from common.mq import MQ
  19. from common.public import download_rule, get_config_from_mysql
  20. from common.scheduling_db import MysqlHelper
  21. def get_redirect_url(url):
  22. res = requests.get(url, allow_redirects=False)
  23. if res.status_code == 302 or res.status_code == 301:
  24. return res.headers['Location']
  25. else:
  26. return url
  27. class XiaoNianGaoPlusRecommend:
  28. platform = "小年糕-offline"
  29. download_cnt = 0
  30. element_list = []
  31. i = 0
  32. @classmethod
  33. def start_wechat(cls, log_type, crawler, env, rule_dict, our_uid):
  34. if env == "dev":
  35. chromedriverExecutable = "/Users/tzld/Downloads/chromedriver_v111/chromedriver"
  36. else:
  37. chromedriverExecutable = "/Users/tzld/Downloads/chromedriver_v111/chromedriver"
  38. Common.logger(log_type, crawler).info("启动微信")
  39. Common.logging(log_type, crawler, env, '启动微信')
  40. caps = {
  41. "platformName": "Android",
  42. "devicesName": "Android",
  43. # "platformVersion": "11",
  44. # "udid": "emulator-5554",
  45. "appPackage": "com.tencent.mm",
  46. "appActivity": ".ui.LauncherUI",
  47. "autoGrantPermissions": "true",
  48. "noReset": True,
  49. "resetkeyboard": True,
  50. "unicodekeyboard": True,
  51. "showChromedriverLog": True,
  52. "printPageSourceOnFailure": True,
  53. "recreateChromeDriverSessions": True,
  54. "enableWebviewDetailsCollection": True,
  55. "setWebContentsDebuggingEnabled": True,
  56. "newCommandTimeout": 6000,
  57. "automationName": "UiAutomator2",
  58. "chromedriverExecutable": chromedriverExecutable,
  59. "chromeOptions": {"androidProcess": "com.tencent.mm:appbrand0"},
  60. }
  61. driver = webdriver.Remote("http://localhost:4723/wd/hub", caps)
  62. driver.implicitly_wait(30)
  63. for i in range(120):
  64. try:
  65. if driver.find_elements(By.ID, 'com.tencent.mm:id/ky9'):
  66. Common.logger(log_type, crawler).info("微信启动成功")
  67. Common.logging(log_type, crawler, env, '微信启动成功')
  68. break
  69. elif driver.find_element(By.ID, "com.android.systemui:id/dismiss_view"):
  70. Common.logger(log_type, crawler).info("发现并关闭系统下拉菜单")
  71. Common.logging(log_type, crawler, env, '发现并关闭系统下拉菜单')
  72. driver.find_element(By.ID, "com.android.system:id/dismiss_view").click()
  73. else:
  74. pass
  75. except NoSuchElementException:
  76. time.sleep(1)
  77. Common.logger(log_type, crawler).info("下滑,展示小程序选择面板")
  78. Common.logging(log_type, crawler, env, '下滑,展示小程序选择面板')
  79. size = driver.get_window_size()
  80. driver.swipe(int(size['width'] * 0.5), int(size['height'] * 0.2),
  81. int(size['width'] * 0.5), int(size['height'] * 0.8), 200)
  82. time.sleep(1)
  83. Common.logger(log_type, crawler).info('打开小程序"小年糕+"')
  84. Common.logging(log_type, crawler, env, '打开小程序"小年糕+"')
  85. driver.find_elements(By.XPATH, '//*[@text="小年糕+"]')[-1].click()
  86. time.sleep(5)
  87. driver.find_elements(By.XPATH, '//*[@text="我"]')[-1].click()
  88. time.sleep(5)
  89. driver.find_elements(By.XPATH, '//*[@text="关注"]')[-1].click()
  90. time.sleep(5)
  91. cls.get_videoList(log_type, crawler, driver, env, rule_dict, our_uid)
  92. time.sleep(1)
  93. driver.quit()
  94. @classmethod
  95. def search_elements(cls, driver: WebDriver, xpath):
  96. time.sleep(1)
  97. windowHandles = driver.window_handles
  98. for handle in windowHandles:
  99. driver.switch_to.window(handle)
  100. time.sleep(1)
  101. try:
  102. elements = driver.find_elements(By.XPATH, xpath)
  103. if elements:
  104. return elements
  105. except NoSuchElementException:
  106. pass
  107. @classmethod
  108. def check_to_applet(cls, log_type, crawler, env, driver: WebDriver):
  109. time.sleep(1)
  110. webViews = driver.contexts
  111. Common.logger(log_type, crawler).info(f"webViews:{webViews}")
  112. Common.logging(log_type, crawler, env, f"webViews:{webViews}")
  113. driver.switch_to.context(webViews[1])
  114. windowHandles = driver.window_handles
  115. for handle in windowHandles:
  116. driver.switch_to.window(handle)
  117. time.sleep(1)
  118. try:
  119. driver.find_element(By.XPATH, '//*[@class="tab-bar--tab tab-bar--tab-selected"]')
  120. Common.logger(log_type, crawler).info("切换到小程序成功\n")
  121. Common.logging(log_type, crawler, env, '切换到小程序成功\n')
  122. return
  123. except NoSuchElementException:
  124. time.sleep(1)
  125. @classmethod
  126. def repeat_video(cls, log_type, crawler, video_id, env):
  127. sql = f""" select * from crawler_video where platform in ("{crawler}","{cls.platform}") and out_video_id="{video_id}"; """
  128. repeat_video = MysqlHelper.get_values(log_type, crawler, sql, env)
  129. return len(repeat_video)
  130. @classmethod
  131. def swipe_up(cls, driver: WebDriver):
  132. cls.search_elements(driver, '//*[@class="expose--adapt-parent"]')
  133. size = driver.get_window_size()
  134. driver.swipe(int(size["width"] * 0.7), int(size["height"] * 0.8),
  135. int(size["width"] * 0.7), int(size["height"] * 0.4), 200)
  136. @classmethod
  137. def get_video_url(cls, log_type, crawler, driver: WebDriver, video_title_element):
  138. for i in range(3):
  139. cls.search_elements(driver, '//wx-view[@class="expose--adapt-parent"]')
  140. Common.logger(log_type, crawler).info(f"video_title_element:{video_title_element[0]}")
  141. time.sleep(1)
  142. Common.logger(log_type, crawler).info("滑动标题至可见状态")
  143. driver.execute_script("arguments[0].scrollIntoView({block:'center',inline:'center'});",
  144. video_title_element[0])
  145. time.sleep(3)
  146. Common.logger(log_type, crawler).info("点击标题")
  147. video_title_element[0].click()
  148. # driver.execute_script("arguments[0].click();", video_title_element[0])
  149. Common.logger(log_type, crawler).info("点击标题完成")
  150. time.sleep(1)
  151. video_url_elements = cls.search_elements(driver,
  152. '//wx-video[@class="video-player--video"]')
  153. if video_url_elements:
  154. return video_url_elements[0].get_attribute("src")
  155. @classmethod
  156. def get_videoList(cls, log_type, crawler, driver: WebDriver, env, rule_dict, our_uid):
  157. mq = MQ(topic_name="topic_crawler_etl_" + env)
  158. driver.implicitly_wait(20)
  159. # 鼠标左键点击, 1为x坐标, 2为y坐标
  160. cls.check_to_applet(log_type=log_type, crawler=crawler, env=env, driver=driver)
  161. time.sleep(1)
  162. page = 0
  163. element_name = cls.search_elements(driver, "//wx-view[@class='box_user_nick_txt']")
  164. driver.execute_script("arguments[0].scrollIntoView({block:'center',inline:'center'});",
  165. element_name[0])
  166. time.sleep(3)
  167. count = len(element_name)
  168. for i in range(count-1):
  169. element_name[i].click()
  170. user_name = element_name[i].text
  171. time.sleep(10)
  172. while True:
  173. if cls.search_elements(driver, '//wx-view[@class="expose--adapt-parent"]') is None:
  174. Common.logger(log_type, crawler).info("窗口已销毁\n")
  175. Common.logging(log_type, crawler, env, '窗口已销毁\n')
  176. cls.i = 0
  177. cls.download_cnt = 0
  178. cls.element_list = []
  179. driver.press_keycode(AndroidKey.BACK)
  180. continue
  181. page_source = driver.page_source
  182. soup = BeautifulSoup(page_source, 'html.parser')
  183. soup.prettify()
  184. video_url_elements = soup.findAll("wx-view", class_="expose--adapt-parent")
  185. video_url_elements = list(set(video_url_elements).difference(set(cls.element_list)))
  186. cls.element_list = list(set(video_url_elements) | set(cls.element_list))
  187. Common.logger(log_type, crawler).info(f"正在抓取第{page + 1}页,共:{len(video_url_elements)}条视频")
  188. Common.logging(log_type, crawler, env, f"正在抓取第{page + 1}页,共:{len(video_url_elements)}条视频")
  189. if len(video_url_elements) == 0:
  190. for i in range(5):
  191. Common.logger(log_type, crawler).info(f"向上滑动第{i + 1}次")
  192. cls.swipe_up(driver)
  193. time.sleep(0.5)
  194. continue
  195. for i, video_element in enumerate(video_url_elements):
  196. try:
  197. Common.logger(log_type, crawler).info(f"本轮已抓取{cls.download_cnt}条视频\n")
  198. Common.logging(log_type, crawler, env, f"本轮已抓取{cls.download_cnt}条视频\n")
  199. if cls.download_cnt >= int(rule_dict.get("videos_cnt", {}).get("min", 10)):
  200. cls.i = 0
  201. cls.download_cnt = 0
  202. cls.element_list = []
  203. return
  204. cls.i += 1
  205. Common.logger(log_type, crawler).info(f"第{cls.i}条视频")
  206. Common.logging(log_type, crawler, env, f"第{cls.i}条视频")
  207. video_title = video_element.find("wx-view", class_="album--album-cover-title").text
  208. # 播放量字符串
  209. play_str = video_element.find("wx-view", class_="album--album-cover-views").text
  210. # 封面 URL
  211. cover_url = video_element.find("wx-image", class_="album--album-cover-bg")["src"]
  212. play_cnt = int(play_str.replace("+", "").replace("次播放", ""))
  213. # 发布时间
  214. play_time = video_element.find("wx-view", class_="album--album-time").text
  215. date_three_days_ago_string = (date.today() + timedelta(days=-3)).strftime("%Y.%m.%d")
  216. rule = play_time > date_three_days_ago_string
  217. if rule == False:
  218. Common.logger(log_type, crawler).info(f"发布时间小于3天,发布时间:{rule}")
  219. Common.logging(log_type, crawler, env, f"发布时间小于3天,发布时间:{rule}")
  220. break
  221. out_video_id = md5(video_title.encode('utf8')).hexdigest()
  222. out_video_id = out_video_id + "user"
  223. out_user_id = md5(user_name.encode('utf8')).hexdigest()
  224. out_user_id = out_user_id + "user"
  225. video_dict = {
  226. "video_title": video_title,
  227. "video_id": out_video_id,
  228. "duration_str": '',
  229. "duration": 0,
  230. "play_str": play_str,
  231. "play_cnt": play_cnt,
  232. "like_str": "",
  233. "like_cnt": 0,
  234. "comment_cnt": 0,
  235. "share_cnt": 0,
  236. "user_name": user_name,
  237. "user_id": out_user_id,
  238. 'publish_time_stamp': int(time.time()),
  239. 'publish_time_str': time.strftime("%Y-%m-%d %H:%M:%S",
  240. time.localtime(int(time.time()))),
  241. "avatar_url": cover_url,
  242. "cover_url": cover_url,
  243. "session": f"xiaoniangao-{int(time.time())}"
  244. }
  245. for k, v in video_dict.items():
  246. Common.logger(log_type, crawler).info(f"{k}:{v}")
  247. Common.logging(log_type, crawler, env, f"video_dict:{video_dict}")
  248. # Common.logger(log_type, crawler).info(f"==========分割线==========\n")
  249. if video_title is None or cover_url is None:
  250. Common.logger(log_type, crawler).info("无效视频")
  251. Common.logging(log_type, crawler, env, '无效视频')
  252. time.sleep(0.5)
  253. elif download_rule(log_type=log_type, crawler=crawler, video_dict=video_dict,
  254. rule_dict=rule_dict) is False:
  255. Common.logger(log_type, crawler).info("不满足抓取规则")
  256. Common.logging(log_type, crawler, env, "不满足抓取规则")
  257. time.sleep(0.5)
  258. elif any(str(word) if str(word) in video_dict["video_title"] else False
  259. for word in get_config_from_mysql(log_type=log_type,
  260. source=crawler,
  261. env=env,
  262. text="filter",
  263. action="")) is True:
  264. Common.logger(log_type, crawler).info('已中过滤词')
  265. Common.logging(log_type, crawler, env, '已中过滤词')
  266. time.sleep(0.5)
  267. elif cls.repeat_video(log_type, crawler, out_video_id, env) != 0:
  268. Common.logger(log_type, crawler).info('视频已下载\n')
  269. Common.logging(log_type, crawler, env, '视频已下载\n')
  270. time.sleep(5)
  271. else:
  272. video_title_element = cls.search_elements(driver,
  273. f'//*[contains(text(), "{video_title}")]')
  274. if video_title_element is None:
  275. Common.logger(log_type, crawler).warning(f"未找到该视频标题的element:{video_title_element}")
  276. Common.logging(log_type, crawler, env, f"未找到该视频标题的element:{video_title_element}")
  277. continue
  278. Common.logger(log_type, crawler).info("点击标题,进入视频详情页")
  279. Common.logging(log_type, crawler, env, "点击标题,进入视频详情页")
  280. video_url = cls.get_video_url(log_type, crawler, driver, video_title_element)
  281. video_url = get_redirect_url(video_url)
  282. if video_url is None:
  283. Common.logger(log_type, crawler).info("未获取到视频播放地址\n")
  284. driver.press_keycode(AndroidKey.BACK)
  285. time.sleep(5)
  286. continue
  287. video_dict['video_url'] = video_url
  288. Common.logger(log_type, crawler).info(f"video_url:{video_url}")
  289. video_dict["platform"] = crawler
  290. video_dict["strategy"] = log_type
  291. video_dict["out_video_id"] = video_dict["video_id"]
  292. video_dict["crawler_rule"] = json.dumps(rule_dict)
  293. video_dict["user_id"] = our_uid
  294. video_dict["publish_time"] = video_dict["publish_time_str"]
  295. Common.logger(log_type, crawler).info(f"video_dict:{video_dict}")
  296. Common.logging(log_type, crawler, env, f"video_dict:{video_dict}")
  297. mq.send_msg(video_dict)
  298. cls.download_cnt += 1
  299. driver.press_keycode(AndroidKey.BACK)
  300. time.sleep(5)
  301. except Exception as e:
  302. Common.logger(log_type, crawler).error(f"抓取单条视频异常:{e}\n")
  303. Common.logging(log_type, crawler, env, f"抓取单条视频异常:{e}\n")
  304. page += 1
  305. driver.press_keycode(AndroidKey.BACK)
  306. driver.execute_script("arguments[0].scrollIntoView({block:'center',inline:'center'});",
  307. element_name[i])
  308. Common.logger(log_type, crawler).info("f已抓取完用户{user_name},休眠 5 秒")
  309. Common.logging(log_type, crawler, env, "f已抓取完用户{user_name},休眠 5 秒")
  310. time.sleep(5)
  311. if __name__ == "__main__":
  312. rule_dict1 = {"period": {"min": 0, "max": 365},
  313. "duration": {"min": 0, "max": 1800},
  314. "favorite_cnt": {"min": 0, "max": 0},
  315. "videos_cnt": {"min": 10, "max": 20},
  316. "share_cnt": {"min": 0, "max": 0}}
  317. XiaoNianGaoPlusRecommend.start_wechat("recommend", "xiaoniangao", "dev", rule_dict1, 6267141)