shipinhao_search.py 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366
  1. # -*- coding: utf-8 -*-
  2. # @Author: wangkun
  3. # @Time: 2023/4/25
  4. import datetime
  5. import difflib
  6. import os
  7. import sys
  8. import time
  9. from datetime import date, timedelta
  10. from hashlib import md5
  11. from appium import webdriver
  12. from appium.webdriver.extensions.android.nativekey import AndroidKey
  13. from appium.webdriver.webdriver import WebDriver
  14. from selenium.common import NoSuchElementException
  15. from selenium.webdriver.common.by import By
  16. sys.path.append(os.getcwd())
  17. from common.feishu import Feishu
  18. from common.publish import Publish
  19. from common.common import Common
  20. from common.public import get_config_from_mysql
  21. class ShipinhaoSearch:
  22. i = 0
  23. @classmethod
  24. def start_wechat(cls, log_type, crawler, word, env):
  25. Common.logger(log_type, crawler).info('启动微信')
  26. if env == "dev":
  27. chromedriverExecutable = "/Users/wangkun/Downloads/chromedriver/chromedriver_v107/chromedriver"
  28. else:
  29. chromedriverExecutable = '/Users/piaoquan/Downloads/chromedriver'
  30. caps = {
  31. "platformName": "Android", # 手机操作系统 Android / iOS
  32. "deviceName": "Android", # 连接的设备名(模拟器或真机),安卓可以随便写
  33. "platforVersion": "13", # 手机对应的系统版本(Android 13)
  34. "appPackage": "com.tencent.mm", # 被测APP的包名,乐活圈 Android
  35. "appActivity": ".ui.LauncherUI", # 启动的Activity名
  36. "autoGrantPermissions": True, # 让 appium 自动授权 base 权限,
  37. # 如果 noReset 为 True,则该条不生效(该参数为 Android 独有),对应的值为 True 或 False
  38. "unicodekeyboard": True, # 使用自带输入法,输入中文时填True
  39. "resetkeyboard": True, # 执行完程序恢复原来输入法
  40. "noReset": True, # 不重置APP
  41. "recreateChromeDriverSessions": True, # 切换到非 chrome-Driver 会 kill 掉 session,就不需要手动 kill 了
  42. "printPageSourceOnFailure": True, # 找不到元素时,appium log 会完整记录当前页面的 pagesource
  43. "newCommandTimeout": 6000, # 初始等待时间
  44. "automationName": "UiAutomator2", # 使用引擎,默认为 Appium,
  45. # 其中 Appium、UiAutomator2、Selendroid、Espresso 用于 Android,XCUITest 用于 iOS
  46. "showChromedriverLog": True,
  47. # "chromeOptions": {"androidProcess": "com.tencent.mm:appbrand0"},
  48. "chromeOptions": {"androidProcess": "com.tencent.mm:tools"},
  49. 'enableWebviewDetailsCollection': True,
  50. 'setWebContentsDebuggingEnabled': True,
  51. 'chromedriverExecutable': chromedriverExecutable,
  52. }
  53. driver = webdriver.Remote("http://localhost:4723/wd/hub", caps)
  54. driver.implicitly_wait(10)
  55. # # 向下滑动页面,展示出小程序选择面板
  56. # for i in range(120):
  57. # try:
  58. # # 发现微信消息 TAB,代表微信已启动成功
  59. # if driver.find_elements(By.ID, 'com.tencent.mm:id/f2s'):
  60. # break
  61. # # 发现并关闭系统菜单栏
  62. # elif driver.find_element(By.ID, 'com.android.systemui:id/dismiss_view'):
  63. # Common.logger(log_type, crawler).info('发现并关闭系统下拉菜单栏')
  64. # driver.find_element(By.ID, 'com.android.systemui:id/dismiss_view').click()
  65. # else:
  66. # pass
  67. # except NoSuchElementException:
  68. # time.sleep(1)
  69. if len(driver.find_elements(By.ID, 'android:id/text1')) != 0:
  70. driver.find_elements(By.ID, 'android:id/text1')[0].click()
  71. time.sleep(5)
  72. cls.search_video(log_type=log_type,
  73. crawler=crawler,
  74. word=word,
  75. driver=driver,
  76. env=env)
  77. cls.close_wechat(log_type=log_type,
  78. crawler=crawler,
  79. driver=driver)
  80. @classmethod
  81. def close_wechat(cls, log_type, crawler, driver: WebDriver):
  82. driver.quit()
  83. Common.logger(log_type, crawler).info(f"微信退出成功\n")
  84. @classmethod
  85. def is_contain_chinese(cls, strword):
  86. for ch in strword:
  87. if u'\u4e00' <= ch <= u'\u9fff':
  88. return True
  89. return False
  90. # 查找元素
  91. @classmethod
  92. def search_elements(cls, driver: WebDriver, xpath):
  93. time.sleep(1)
  94. windowHandles = driver.window_handles
  95. for handle in windowHandles:
  96. driver.switch_to.window(handle)
  97. time.sleep(1)
  98. try:
  99. elements = driver.find_elements(By.XPATH, xpath)
  100. if elements:
  101. return elements
  102. except NoSuchElementException:
  103. pass
  104. @classmethod
  105. def check_to_webview(cls, log_type, crawler, driver: WebDriver):
  106. # Common.logger(log_type, crawler).info('切换到webview')
  107. webviews = driver.contexts
  108. driver.switch_to.context(webviews[1])
  109. time.sleep(1)
  110. windowHandles = driver.window_handles
  111. for handle in windowHandles:
  112. driver.switch_to.window(handle)
  113. try:
  114. shipinhao_webview = driver.find_element(By.XPATH, '//div[@class="unit"]')
  115. if shipinhao_webview:
  116. Common.logger(log_type, crawler).info('切换到视频号 webview 成功')
  117. return "成功"
  118. except Exception as e:
  119. Common.logger(log_type, crawler).info(f"{e}\n")
  120. @classmethod
  121. def search_video(cls, log_type, crawler, word, driver: WebDriver, env):
  122. # 点击微信搜索框,并输入搜索词
  123. driver.implicitly_wait(10)
  124. driver.find_element(By.ID, 'com.tencent.mm:id/j5t').click()
  125. time.sleep(0.5)
  126. Common.logger(log_type, crawler).info(f'输入搜索词:{word}')
  127. driver.find_element(By.ID, 'com.tencent.mm:id/cd7').clear().send_keys(word)
  128. driver.press_keycode(AndroidKey.ENTER)
  129. # driver.find_elements(By.ID, 'com.tencent.mm:id/oi4')[0].click()
  130. driver.find_element(By.ID, 'com.tencent.mm:id/m94').click()
  131. time.sleep(5)
  132. # 切换到微信搜索结果页 webview
  133. check_to_webview = cls.check_to_webview(log_type, crawler, driver)
  134. if check_to_webview is None:
  135. Common.logger(log_type, crawler).info("切换到视频号 webview 失败\n")
  136. return
  137. time.sleep(1)
  138. # 切换到"视频号"分类
  139. shipinhao_tags = cls.search_elements(driver, '//div[@class="unit"]/*[2]')
  140. Common.logger(log_type, crawler).info('点击"视频号"分类')
  141. shipinhao_tags[0].click()
  142. time.sleep(5)
  143. index = 0
  144. while True:
  145. if cls.i >= 100:
  146. Common.logger(log_type, crawler).info(f'搜索词:"{word}",已抓取视频数:{index}')
  147. cls.i = 0
  148. return
  149. # try:
  150. if cls.search_elements(driver, '//*[@class="double-rich double-rich_vertical"]') is None:
  151. Common.logger(log_type, crawler).info('窗口已销毁\n')
  152. return
  153. Common.logger(log_type, crawler).info('获取视频列表\n')
  154. video_elements = cls.search_elements(driver, '//div[@class="vc active__mask"]')
  155. if video_elements is None:
  156. Common.logger(log_type, crawler).warning(f'video_elements:{video_elements}')
  157. return
  158. video_element_temp = video_elements[index:]
  159. if len(video_element_temp) == 0:
  160. Common.logger(log_type, crawler).info('到底啦~~~~~~~~~~~~~\n')
  161. return
  162. for i, video_element in enumerate(video_element_temp):
  163. if video_element is None:
  164. Common.logger(log_type, crawler).info('到底啦~\n')
  165. return
  166. cls.i += 1
  167. cls.search_elements(driver, '//div[@class="vc active__mask"]')
  168. Common.logger(log_type, crawler).info(f'拖动"视频"列表第{cls.i}个至屏幕中间')
  169. time.sleep(3)
  170. driver.execute_script("arguments[0].scrollIntoView({block:'center',inline:'center'})",
  171. video_element)
  172. if len(video_element.find_elements(By.XPATH, "//*[@text='没有更多的搜索结果']")) != 0:
  173. Common.logger(log_type, crawler).info("没有更多的搜索结果\n")
  174. return
  175. video_title = video_element.find_elements(By.XPATH, '//div[@class="title ellipsis_2"]/*[2]')[index + i].text
  176. video_url = video_element.find_elements(By.XPATH, '//div[@class="video-player"]')[index+i].get_attribute('src')
  177. cover_url = video_element.find_elements(By.XPATH, '//div[@class="video-player__bd"]')[index+i].get_attribute('style')
  178. duration = video_element.find_elements(By.XPATH, '//div[@class="play-mask__text"]/*[2]')[index+i].text
  179. duration = int(duration.split(':')[0]) * 60 + int(duration.split(':')[-1])
  180. user_name = video_element.find_elements(By.XPATH, '//p[@class="vc-source__text"]')[index+i].text
  181. avatar_url = video_element.find_elements(By.XPATH, '//div[@class="ui-image-image ui-image vc-source__thumb"]')[index+i].get_attribute('style')
  182. # Common.logger(log_type, crawler).info(f"video_title:{video_title}")
  183. # Common.logger(log_type, crawler).info(f"duration:{duration}")
  184. video_element.click()
  185. time.sleep(3)
  186. video_dict = cls.get_video_info(log_type=log_type,
  187. crawler=crawler,
  188. driver=driver)
  189. video_dict["video_title"] = video_title
  190. video_dict["duration"] = duration
  191. video_dict["video_url"] = video_url
  192. for k, v in video_dict.items():
  193. Common.logger(log_type, crawler).info(f"{k}:{v}")
  194. if video_title in [x for y in Feishu.get_values_batch(log_type, crawler, "xYWCzf") for x in y]:
  195. Common.logger(log_type, crawler).info("视频已存在\n")
  196. else:
  197. cls.download_publish(log_type, crawler, word, video_dict)
  198. Common.logger(log_type, crawler).info('已抓取完一组视频,休眠1秒\n')
  199. time.sleep(1)
  200. index = index + len(video_element_temp)
  201. # except Exception as e:
  202. # Common.logger(log_type, crawler).info(f"get_videoList:{e}\n")
  203. # cls.i = 0
  204. @classmethod
  205. def download_publish(cls, log_type, crawler, word, video_dict):
  206. Feishu.insert_columns(log_type, crawler, "xYWCzf", "ROWS", 1, 2)
  207. time.sleep(0.5)
  208. values = [[time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(int(time.time()))),
  209. "视频号搜索",
  210. word,
  211. video_dict["video_title"],
  212. video_dict["duration"],
  213. video_dict["like_cnt"],
  214. video_dict["share_cnt"],
  215. video_dict["favorite_cnt"],
  216. video_dict["comment_cnt"],
  217. video_dict["publish_time_str"],
  218. "待获取",
  219. "待获取",
  220. "待获取",
  221. video_dict["video_url"]]]
  222. Feishu.update_values(log_type, crawler, "xYWCzf", "F2:Z2", values)
  223. Common.logger(log_type, crawler).info("写入飞书成功\n")
  224. @classmethod
  225. def get_video_info(cls, log_type, crawler, driver: WebDriver):
  226. # Common.logger(log_type, crawler).info('切回NATIVE_APP')
  227. driver.switch_to.context('NATIVE_APP')
  228. # 点赞
  229. like_id = driver.find_element(By.ID, 'com.tencent.mm:id/k04')
  230. like_cnt = like_id.get_attribute('name')
  231. if like_cnt == "" or like_cnt == "喜欢" or like_cnt == "火" or cls.is_contain_chinese(like_cnt) is True:
  232. like_cnt = 0
  233. elif '万' in like_cnt:
  234. like_cnt = int(float(like_cnt.split('万')[0]) * 10000)
  235. elif '万+' in like_cnt:
  236. like_cnt = int(float(like_cnt.split('万+')[0]) * 10000)
  237. else:
  238. like_cnt = int(float(like_cnt))
  239. # 分享
  240. share_id = driver.find_element(By.ID, 'com.tencent.mm:id/jhv')
  241. share_cnt = share_id.get_attribute('name')
  242. if share_cnt == "" or share_cnt == "转发" or cls.is_contain_chinese(share_cnt) is True:
  243. share_cnt = 0
  244. elif '万' in share_cnt:
  245. share_cnt = int(float(share_cnt.split('万')[0]) * 10000)
  246. elif '万+' in share_cnt:
  247. share_cnt = int(float(share_cnt.split('万+')[0]) * 10000)
  248. else:
  249. share_cnt = int(float(share_cnt))
  250. # 收藏
  251. favorite_id = driver.find_element(By.ID, 'com.tencent.mm:id/fnp')
  252. favorite_cnt = favorite_id.get_attribute('name')
  253. if favorite_cnt == "" or favorite_cnt == "收藏" or favorite_cnt == "推荐" or favorite_cnt == "火" or cls.is_contain_chinese(favorite_cnt) is True:
  254. favorite_cnt = 0
  255. elif '万' in favorite_cnt:
  256. favorite_cnt = int(float(favorite_cnt.split('万')[0]) * 10000)
  257. elif '万+' in favorite_cnt:
  258. favorite_cnt = int(float(favorite_cnt.split('万+')[0]) * 10000)
  259. else:
  260. favorite_cnt = int(float(favorite_cnt))
  261. # 评论
  262. comment_id = driver.find_element(By.ID, 'com.tencent.mm:id/bje')
  263. comment_cnt = comment_id.get_attribute('name')
  264. if comment_cnt == "" or comment_cnt == "评论" or cls.is_contain_chinese(comment_cnt) is True:
  265. comment_cnt = 0
  266. elif '万' in comment_cnt:
  267. comment_cnt = int(float(comment_cnt.split('万')[0]) * 10000)
  268. elif '万+' in comment_cnt:
  269. comment_cnt = int(float(comment_cnt.split('万+')[0]) * 10000)
  270. else:
  271. comment_cnt = int(float(comment_cnt))
  272. # 发布时间
  273. comment_id.click()
  274. time.sleep(1)
  275. publish_time = driver.find_element(By.ID, "com.tencent.mm:id/bre").get_attribute("name")
  276. if "天前" in publish_time:
  277. days = int(publish_time.replace("天前", ""))
  278. publish_time_str = (date.today() + timedelta(days=-days)).strftime("%Y-%m-%d")
  279. elif "年" in publish_time:
  280. # publish_time_str = publish_time.replace("年", "-").replace("月", "-").replace("日", "")
  281. year_str = publish_time.split("年")[0]
  282. month_str = publish_time.split("年")[-1].split("月")[0]
  283. day_str = publish_time.split("月")[-1].split("日")[0]
  284. if int(month_str) < 10:
  285. month_str = f"0{month_str}"
  286. if int(day_str) < 10:
  287. day_str = f"0{day_str}"
  288. publish_time_str = f"{year_str}-{month_str}-{day_str}"
  289. else:
  290. year_str = str(datetime.datetime.now().year)
  291. month_str = publish_time.split("月")[0]
  292. day_str = publish_time.split("月")[-1].split("日")[0]
  293. if int(month_str) < 10:
  294. month_str = f"0{month_str}"
  295. if int(day_str) < 10:
  296. day_str = f"0{day_str}"
  297. publish_time_str = f"{year_str}-{month_str}-{day_str}"
  298. # publish_time_str = f'2023-{publish_time.replace("月", "-").replace("日", "")}'
  299. publish_time_stamp = int(time.mktime(time.strptime(publish_time_str, "%Y-%m-%d")))
  300. # 收起评论
  301. # Common.logger(log_type, crawler).info("收起评论")
  302. driver.find_element(By.ID, "com.tencent.mm:id/be_").click()
  303. time.sleep(0.5)
  304. # 返回 webview
  305. # Common.logger(log_type, crawler).info(f"操作手机返回按键")
  306. driver.find_element(By.ID, "com.tencent.mm:id/a2z").click()
  307. time.sleep(0.5)
  308. # driver.press_keycode(AndroidKey.BACK)
  309. # cls.check_to_webview(log_type=log_type, crawler=crawler, driver=driver)
  310. webviews = driver.contexts
  311. driver.switch_to.context(webviews[1])
  312. video_dict = {
  313. "like_cnt": like_cnt,
  314. "share_cnt": share_cnt,
  315. "favorite_cnt": favorite_cnt,
  316. "comment_cnt": comment_cnt,
  317. "publish_time_str": publish_time_str,
  318. "publish_time_stamp": publish_time_stamp,
  319. }
  320. return video_dict
  321. @classmethod
  322. def search_all_videos(cls, log_type, crawler, env):
  323. word_list = get_config_from_mysql(log_type, crawler, env, "search_word", action="")
  324. for word in word_list:
  325. cls.i = 0
  326. Common.logger(log_type, crawler).info(f"开始抓取搜索词:{word}")
  327. # try:
  328. cls.start_wechat(log_type=log_type,
  329. crawler=crawler,
  330. word=word,
  331. env=env)
  332. # except Exception as e:
  333. # Common.logger(log_type, crawler).error(f"search_video:{e}\n")
  334. if __name__ == '__main__':
  335. ShipinhaoSearch.search_all_videos(log_type="search", crawler="shipinhao", env="dev")
  336. # print(datetime.datetime.now().year)
  337. pass