haitunzhufu_recommend.py 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318
  1. # -*- coding: utf-8 -*-
  2. # @Author: wangkun
  3. # @Time: 2023/9/13
  4. import json
  5. import os
  6. import random
  7. import re
  8. import shutil
  9. import sys
  10. import time
  11. from hashlib import md5
  12. from appium import webdriver
  13. from appium.webdriver.extensions.android.nativekey import AndroidKey
  14. from appium.webdriver.webdriver import WebDriver
  15. from selenium.common import NoSuchElementException
  16. from selenium.webdriver.common.by import By
  17. sys.path.append(os.getcwd())
  18. from common.common import Common
  19. from common.feishu import Feishu
  20. from common.publish import Publish
  21. from common.scheduling_db import MysqlHelper
  22. class HTZFRecommend:
  23. platform = "海豚祝福"
  24. i = 0
  25. @classmethod
  26. def today_download_cnt(cls, log_type, crawler, env):
  27. select_sql = """ SELECT COUNT(*) FROM crawler_video WHERE platform IN ("haitunzhufu", "海豚祝福") AND DATE(create_time) = CURDATE(); """
  28. today_download_cnt = MysqlHelper.get_values(log_type, crawler, select_sql, env, action="")[0]['COUNT(*)']
  29. # print(today_download_cnt)
  30. return today_download_cnt
  31. @classmethod
  32. def start_wechat(cls, log_type, crawler, videos_cnt, env):
  33. if env == "dev":
  34. chromedriverExecutable = "/Users/wangkun/Downloads/chromedriver/chromedriver_v111/chromedriver"
  35. else:
  36. chromedriverExecutable = "/Users/piaoquan/Downloads/chromedriver"
  37. Common.logger(log_type, crawler).info("启动微信")
  38. caps = {
  39. "platformName": "Android",
  40. "platformVersion": "11",
  41. "devicesName": "Android",
  42. "appPackage": "com.tencent.mm",
  43. "appActivity": ".ui.LauncherUI",
  44. "noReset": True,
  45. "resetkeyboard": True,
  46. "unicodekeyboard": True,
  47. "showChromedriverLog": True,
  48. "autoGrantPermissions": True,
  49. "printPageSourceOnFailure": True,
  50. "recreateChromeDriverSessions": True,
  51. "enableWebviewDetailsCollention": True,
  52. "newCommandTimeout": 6000,
  53. "automationName": "UiAutomator2",
  54. "chromedriverExecutable": chromedriverExecutable,
  55. "chromeOptions": {"androidProcess": "com.tencent.mm:appbrand0"},
  56. }
  57. driver = webdriver.Remote("http://localhost:4723/wd/hub", caps)
  58. driver.implicitly_wait(20)
  59. for i in range(120):
  60. try:
  61. if driver.find_element(By.ID, "com.tencent.mm:id/f2s"):
  62. break
  63. elif driver.find_element(By.ID, "com.android.system:id/dismiss_view"):
  64. Common.logger(log_type, crawler).info("发现并关闭系统下拉菜单栏")
  65. else:
  66. pass
  67. except NoSuchElementException:
  68. pass
  69. Common.logger(log_type, crawler).info("下滑,展示小程序选择面板")
  70. size = driver.get_window_size()
  71. driver.swipe(int(size["width"] * 0.5), int(size["height"] * 0.2),
  72. int(size["width"] * 0.5), int(size["height"] * 0.8), 200)
  73. time.sleep(3)
  74. Common.logger(log_type, crawler).info('打开小程序"海豚祝福"')
  75. driver.find_elements(By.XPATH, '//*[@text="海豚祝福"]')[-1].click()
  76. time.sleep(5)
  77. cls.get_videoList(log_type, crawler, driver, videos_cnt, env)
  78. time.sleep(1)
  79. driver.quit()
  80. @classmethod
  81. def search_elements(cls, driver: WebDriver, xpath):
  82. time.sleep(1)
  83. windowHandles = driver.window_handles
  84. for handle in windowHandles:
  85. driver.switch_to.window(handle)
  86. time.sleep(1)
  87. try:
  88. elements = driver.find_elements(By.XPATH, xpath)
  89. if elements:
  90. return elements
  91. except NoSuchElementException:
  92. pass
  93. @classmethod
  94. def repeat_out_video_id(cls, log_type, crawler, out_video_id, env):
  95. sql = f""" select * from crawler_video where platform in ("众妙音信", "刚刚都传", "吉祥幸福", "知青天天看", "zhufuquanzi", "祝福圈子", "haitunzhufu", "海豚祝福") and out_video_id="{out_video_id}"; """
  96. repeat_video = MysqlHelper.get_values(log_type, crawler, sql, env)
  97. return len(repeat_video)
  98. @classmethod
  99. def get_video_url(cls, driver: WebDriver, video_element):
  100. video_element.click()
  101. time.sleep(5)
  102. video_url_elements = cls.search_elements(driver, '//*[@id="myVideo"]')
  103. if video_url_elements:
  104. return video_url_elements[0].get_attribute("src")
  105. else:
  106. return
  107. @classmethod
  108. def get_videoList(cls, log_type, crawler, driver: WebDriver, videos_cnt, env):
  109. driver.implicitly_wait(20)
  110. webviews = driver.contexts
  111. Common.logger(log_type, crawler).info(f"webviews:{webviews}")
  112. driver.switch_to.context(webviews[1])
  113. windowHandles = driver.window_handles
  114. for handle in windowHandles:
  115. driver.switch_to.window(handle)
  116. time.sleep(1)
  117. try:
  118. if cls.search_elements(driver, '//*[@class="bottom_scroll"]'):
  119. Common.logger(log_type, crawler).info("切换到小程序")
  120. break
  121. except NoSuchElementException:
  122. time.sleep(1)
  123. cls.search_elements(driver, '//*[@class="nav cur"]')[-1].click()
  124. Common.logger(log_type, crawler).info('点击"推荐"列表成功\n')
  125. index = 0
  126. while True:
  127. if cls.search_elements(driver, '//*[@class="list"]') is None:
  128. Common.logger(log_type, crawler).info("列表页窗口已销毁\n")
  129. cls.i = 0
  130. return
  131. videoList_elements = cls.search_elements(driver, '//*[@class="img_bf"]')
  132. if videoList_elements is None or len(videoList_elements) == 0:
  133. cls.i = 0
  134. Common.logger(log_type, crawler).warning(f"videoList_elements:{videoList_elements}")
  135. return
  136. video_list = videoList_elements[index:]
  137. if len(video_list) == 0 or video_list is None:
  138. Common.logger(log_type, crawler).info("到底啦~~~~~~~~~~")
  139. cls.i = 0
  140. return
  141. for i, video_element in enumerate(video_list):
  142. try:
  143. today_download_cnt = cls.today_download_cnt(log_type, crawler, env)
  144. Common.logger(log_type, crawler).info(f"今日已下载{today_download_cnt}条视频\n")
  145. if today_download_cnt >= videos_cnt:
  146. cls.i = 0
  147. return
  148. if video_element is None:
  149. Common.logger(log_type, crawler).info("没有更多数据啦~\n")
  150. cls.i = 0
  151. return
  152. cls.i += 1
  153. cls.search_elements(driver, '//*[@class="img_bf"]')
  154. Common.logger(log_type, crawler).info(f"拖动第{cls.i}条视频至屏幕顶部")
  155. driver.execute_script("arguments[0].scrollIntoView({block:'center',inline:'center'})", video_element)
  156. time.sleep(3)
  157. video_title = video_element.find_elements(By.XPATH, '//*[@class="title"]')[i+index].text
  158. plat_cnt_str = video_element.find_elements(By.XPATH, '//*[@class="wan"]')[i+index].text
  159. play_cnt = int(re.sub(r"\D", "", plat_cnt_str)) * 10000 if "万" in plat_cnt_str else plat_cnt_str
  160. cover_url = video_element.find_elements(By.XPATH, '//*[@class="img"]')[i+index].get_attribute('src')
  161. play_icon = video_element.find_elements(By.XPATH, '//*[@class="bf"]')[i+index]
  162. out_video_id = md5(video_title.encode('utf8')).hexdigest()
  163. video_dict = {
  164. "video_title": video_title,
  165. 'video_id': out_video_id,
  166. "plat_cnt_str": plat_cnt_str,
  167. "play_cnt": play_cnt,
  168. 'comment_cnt': 0,
  169. 'like_cnt': 0,
  170. 'share_cnt': 0,
  171. 'publish_time_stamp': int(time.time()),
  172. 'publish_time_str': time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(int(time.time()))),
  173. 'user_name': "haitunzhufu",
  174. 'user_id': "haitunzhufu",
  175. "cover_url": cover_url,
  176. 'avatar_url': cover_url,
  177. 'session': f"haitunzhufu-{int(time.time())}"
  178. }
  179. for k, v in video_dict.items():
  180. Common.logger(log_type, crawler).info(f"{k}:{v}")
  181. if video_title is None or cover_url is None:
  182. Common.logger(log_type, crawler).info("无效视频\n")
  183. elif cls.repeat_out_video_id(log_type, crawler, out_video_id, env) != 0:
  184. Common.logger(log_type, crawler).info('视频已下载\n')
  185. else:
  186. video_url = cls.get_video_url(driver, play_icon)
  187. if video_url is None:
  188. Common.logger(log_type, crawler).info("未获取到视频播放地址\n")
  189. # driver.press_keycode(AndroidKey.BACK)
  190. continue
  191. video_dict["video_url"] = video_url
  192. Common.logger(log_type, crawler).info(f"video_url:{video_url}")
  193. cls.download_publish(log_type, crawler, video_dict, env)
  194. driver.press_keycode(AndroidKey.BACK)
  195. except Exception as e:
  196. Common.logger(log_type, crawler).error(f"抓取单条视频异常:{e}\n")
  197. Common.logger(log_type, crawler).info("已抓取完一组,休眠 10 秒\n")
  198. time.sleep(10)
  199. index = index + len(video_list)
  200. @classmethod
  201. def get_our_uid(cls, log_type, crawler, env):
  202. select_sql = f""" SELECT uid FROM crawler_user_v3 WHERE `source`="{crawler}"; """
  203. uids = MysqlHelper.get_values(log_type, crawler, select_sql, env, action="")
  204. uid_list = []
  205. for uid_dict in uids:
  206. uid_list.append(uid_dict["uid"])
  207. return random.choice(uid_list)
  208. @classmethod
  209. def download_publish(cls, log_type, crawler, video_dict, env):
  210. # 下载视频
  211. Common.download_method(log_type=log_type, crawler=crawler, text='video', title=video_dict['video_title'],
  212. url=video_dict['video_url'])
  213. ffmpeg_dict = Common.ffmpeg(log_type, crawler, f"./{crawler}/videos/{video_dict['video_title']}/video.mp4")
  214. if ffmpeg_dict is None:
  215. md_title = md5(video_dict['video_title'].encode('utf8')).hexdigest()
  216. shutil.rmtree(f"./{crawler}/videos/{md_title}/")
  217. Common.logger(log_type, crawler).info("视频size=0,删除成功\n")
  218. return
  219. video_dict["duration"] = ffmpeg_dict["duration"]
  220. video_dict["video_width"] = ffmpeg_dict["width"]
  221. video_dict["video_height"] = ffmpeg_dict["height"]
  222. # 下载封面
  223. Common.download_method(log_type=log_type, crawler=crawler, text='cover', title=video_dict['video_title'],
  224. url=video_dict['cover_url'])
  225. # 保存视频信息至txt
  226. Common.save_video_info(log_type=log_type, crawler=crawler, video_dict=video_dict)
  227. # 上传视频
  228. Common.logger(log_type, crawler).info("开始上传视频...")
  229. our_video_id = Publish.upload_and_publish(log_type=log_type,
  230. crawler=crawler,
  231. strategy="推荐榜爬虫策略",
  232. our_uid=cls.get_our_uid(log_type, crawler, env),
  233. env=env,
  234. oss_endpoint="out")
  235. if env == 'dev':
  236. our_video_link = f"https://testadmin.piaoquantv.com/cms/post-detail/{our_video_id}/info"
  237. else:
  238. our_video_link = f"https://admin.piaoquantv.com/cms/post-detail/{our_video_id}/info"
  239. Common.logger(log_type, crawler).info("视频上传完成")
  240. if our_video_id is None:
  241. # 删除视频文件夹
  242. shutil.rmtree(f"./{crawler}/videos/{video_dict['video_title']}")
  243. return
  244. # 视频信息保存至飞书
  245. Feishu.insert_columns(log_type, crawler, "d51d20", "ROWS", 1, 2)
  246. # 视频ID工作表,首行写入数据
  247. upload_time = int(time.time())
  248. values = [[time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(upload_time)),
  249. "推荐榜爬虫策略",
  250. video_dict["video_title"],
  251. video_dict["video_id"],
  252. video_dict["play_cnt"],
  253. video_dict["duration"],
  254. f'{video_dict["video_width"]}*{video_dict["video_height"]}',
  255. our_video_link,
  256. video_dict["cover_url"],
  257. video_dict["video_url"]]]
  258. time.sleep(1)
  259. Feishu.update_values(log_type, crawler, "d51d20", "F2:V2", values)
  260. Common.logger(log_type, crawler).info(f"视频已保存至飞书文档\n")
  261. rule_dict = {}
  262. # 视频信息保存数据库
  263. insert_sql = f""" insert into crawler_video(video_id,
  264. out_user_id,
  265. platform,
  266. strategy,
  267. out_video_id,
  268. video_title,
  269. cover_url,
  270. video_url,
  271. duration,
  272. publish_time,
  273. play_cnt,
  274. crawler_rule,
  275. width,
  276. height)
  277. values({our_video_id},
  278. "{video_dict['user_id']}",
  279. "{cls.platform}",
  280. "推荐榜爬虫策略",
  281. "{video_dict['video_id']}",
  282. "{video_dict['video_title']}",
  283. "{video_dict['cover_url']}",
  284. "{video_dict['video_url']}",
  285. {int(video_dict['duration'])},
  286. "{video_dict['publish_time_str']}",
  287. {int(video_dict['play_cnt'])},
  288. '{json.dumps(rule_dict)}',
  289. {int(video_dict['video_width'])},
  290. {int(video_dict['video_height'])}) """
  291. Common.logger(log_type, crawler).info(f"insert_sql:{insert_sql}")
  292. MysqlHelper.update_values(log_type, crawler, insert_sql, env, action='')
  293. Common.logger(log_type, crawler).info('视频信息写入数据库成功!\n')
  294. if __name__ == "__main__":
  295. # HTZFRecommend.start_wechat("recommend", "haitunzhufu", 5, "dev")
  296. # HTZFRecommend.today_download_cnt("recommend", "haitunzhufu", "dev")
  297. # HTZFRecommend.get_play_cnt()
  298. print(HTZFRecommend.get_our_uid("recommend", "haitunzhufu", "prod"))
  299. pass