|
|
@@ -54,12 +54,14 @@ class ZhiqingtiantiankanRecommend:
|
|
|
if env == "dev":
|
|
|
chromedriverExecutable = '/Users/wangkun/Downloads/chromedriver/chromedriver_v107/chromedriver'
|
|
|
else:
|
|
|
- chromedriverExecutable = '/Users/piaoquan/Downloads/chromedriver'
|
|
|
+ # chromedriverExecutable = '/Users/piaoquan/Downloads/chromedriver' # Mac 爬虫机器
|
|
|
+ chromedriverExecutable = 'C:\\chromedriver\\chromedriver.exe' # 阿里云 Windows
|
|
|
caps = {
|
|
|
"platformName": "Android", # 手机操作系统 Android / iOS
|
|
|
"deviceName": "Android", # 连接的设备名（模拟器或真机），安卓可以随便写
|
|
|
- "platforVersion": "11", # 手机对应的系统版本（Android 11）
|
|
|
- "appPackage": "com.tencent.mm", # 被测APP的包名，乐活圈 Android
|
|
|
+ "udid": "emulator-5554", # 指定 adb devices 中的哪一台设备
|
|
|
+ "platforVersion": "12", # 手机对应的系统版本
|
|
|
+ "appPackage": "com.tencent.mm", # 被测APP的包名
|
|
|
"appActivity": ".ui.LauncherUI", # 启动的Activity名
|
|
|
"autoGrantPermissions": "true", # 让 appium 自动授权 base 权限，
|
|
|
# 如果 noReset 为 True，则该条不生效（该参数为 Android 独有），对应的值为 True 或 False
|
|
|
@@ -79,7 +81,7 @@ class ZhiqingtiantiankanRecommend:
|
|
|
# "chromeOptions": {"androidProcess": "com.tencent.mm:tools"},
|
|
|
'browserName': ''
|
|
|
}
|
|
|
- driver = webdriver.Remote("http://localhost:4723/wd/hub", caps)
|
|
|
+ driver = webdriver.Remote("http://localhost:4723/wd/hub", caps) # 指定 Appium 端口号：4723
|
|
|
driver.implicitly_wait(30)
|
|
|
# 向下滑动页面，展示出小程序选择面板
|
|
|
for i in range(120):
|
|
|
@@ -219,7 +221,6 @@ class ZhiqingtiantiankanRecommend:
|
|
|
time.sleep(3)
|
|
|
TouchAction(driver).tap(x=int(size['width'] * 0.4), y=int(size['height'] * 0.1)).perform()
|
|
|
|
|
|
-
|
|
|
@classmethod
|
|
|
def get_videoList(cls, log_type, crawler, driver: WebDriver, env):
|
|
|
driver.implicitly_wait(20)
|
|
|
@@ -231,89 +232,89 @@ class ZhiqingtiantiankanRecommend:
|
|
|
time.sleep(5)
|
|
|
index = 0
|
|
|
while True:
|
|
|
- try:
|
|
|
- if cls.search_elements(driver, '//wx-view[@class="listbox"]') is None:
|
|
|
- Common.logger(log_type, crawler).info('窗口已销毁\n')
|
|
|
- return
|
|
|
+ # try:
|
|
|
+ if cls.search_elements(driver, '//wx-view[@class="listbox"]') is None:
|
|
|
+ Common.logger(log_type, crawler).info('窗口已销毁\n')
|
|
|
+ return
|
|
|
|
|
|
- Common.logger(log_type, crawler).info('获取视频列表\n')
|
|
|
- video_elements = cls.search_elements(driver, '//wx-view[@class="videolist-box videolist--videolist-box"]')
|
|
|
- if video_elements is None:
|
|
|
- Common.logger(log_type, crawler).warning(f'video_elements:{video_elements}')
|
|
|
- return
|
|
|
+ Common.logger(log_type, crawler).info('获取视频列表\n')
|
|
|
+ video_elements = cls.search_elements(driver, '//wx-view[@class="videolist-box videolist--videolist-box"]')
|
|
|
+ if video_elements is None:
|
|
|
+ Common.logger(log_type, crawler).warning(f'video_elements:{video_elements}')
|
|
|
+ return
|
|
|
+
|
|
|
+ video_element_temp = video_elements[index:]
|
|
|
+ if len(video_element_temp) == 0:
|
|
|
+ Common.logger(log_type, crawler).info('到底啦~~~~~~~~~~~~~\n')
|
|
|
+ return
|
|
|
|
|
|
- video_element_temp = video_elements[index:]
|
|
|
- if len(video_element_temp) == 0:
|
|
|
- Common.logger(log_type, crawler).info('到底啦~~~~~~~~~~~~~\n')
|
|
|
+ for i, video_element in enumerate(video_element_temp):
|
|
|
+ if video_element is None:
|
|
|
+ Common.logger(log_type, crawler).info('到底啦~\n')
|
|
|
return
|
|
|
+ cls.i += 1
|
|
|
+ cls.search_elements(driver, '//wx-view[@class="videolist-box videolist--videolist-box"]')
|
|
|
+
|
|
|
+ Common.logger(log_type, crawler).info(f'拖动"视频"列表第{cls.i}个至屏幕中间')
|
|
|
+ time.sleep(3)
|
|
|
+ driver.execute_script("arguments[0].scrollIntoView({block:'center',inline:'center'})", video_element)
|
|
|
+
|
|
|
+ # video_title = video_element.find_elements(By.XPATH, '//wx-view[@class="video_title videolist--video_title"]')[cls.i-1].text
|
|
|
+ # cover_url = video_element.find_elements(By.XPATH, '//wx-image[@class="itemimage videolist--itemimage"]')[cls.i-1].get_attribute('src')
|
|
|
+ # play_cnt = video_element.find_elements(By.XPATH, '//wx-view[@class="clickbox videolist--clickbox"]')[cls.i-1].text
|
|
|
+
|
|
|
+ video_title = video_element.find_elements(By.XPATH, '//wx-view[@class="video_title videolist--video_title"]')[index+i].text
|
|
|
+ cover_url = video_element.find_elements(By.XPATH, '//wx-image[@class="itemimage videolist--itemimage"]')[index+i].get_attribute('src')
|
|
|
+ play_cnt = video_element.find_elements(By.XPATH, '//wx-view[@class="clickbox videolist--clickbox"]')[index+i].text
|
|
|
+
|
|
|
+ play_cnt = int(float(play_cnt.replace("阅读数", "").strip().split("万")[0]) * 10000)
|
|
|
+ out_video_id = md5(video_title.encode('utf8')).hexdigest()
|
|
|
+ video_dict = {
|
|
|
+ 'video_title': video_title,
|
|
|
+ 'video_id': out_video_id,
|
|
|
+ 'play_cnt': play_cnt,
|
|
|
+ 'comment_cnt': 0,
|
|
|
+ 'like_cnt': 0,
|
|
|
+ 'share_cnt': 0,
|
|
|
+ 'publish_time_stamp': int(time.time()),
|
|
|
+ 'publish_time_str': time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(int(time.time()))),
|
|
|
+ 'user_name': "zhiqingtiantiankan",
|
|
|
+ 'user_id': "zhiqingtiantiankan",
|
|
|
+ 'avatar_url': cover_url,
|
|
|
+ 'cover_url': cover_url,
|
|
|
+ 'session': f"zhiqingtiantiankan-{int(time.time())}"
|
|
|
+ }
|
|
|
+ for k, v in video_dict.items():
|
|
|
+ Common.logger(log_type, crawler).info(f"{k}:{v}")
|
|
|
+
|
|
|
+ if video_title is None or cover_url is None:
|
|
|
+ Common.logger(log_type, crawler).info("无效视频\n")
|
|
|
+ elif any(str(word) if str(word) in video_title else False for word in
|
|
|
+ cls.zhiqingtiantiankan_config(log_type, crawler, "filter", env)) is True:
|
|
|
+ Common.logger(log_type, crawler).info('已中过滤词\n')
|
|
|
+ elif cls.repeat_out_video_id(log_type, crawler, out_video_id, env) != 0:
|
|
|
+ Common.logger(log_type, crawler).info('视频已下载\n')
|
|
|
+ else:
|
|
|
+ video_url = cls.get_video_url(log_type, crawler, driver, video_element)
|
|
|
|
|
|
- for i, video_element in enumerate(video_element_temp):
|
|
|
- if video_element is None:
|
|
|
- Common.logger(log_type, crawler).info('到底啦~\n')
|
|
|
- return
|
|
|
- cls.i += 1
|
|
|
- cls.search_elements(driver, '//wx-view[@class="videolist-box videolist--videolist-box"]')
|
|
|
-
|
|
|
- Common.logger(log_type, crawler).info(f'拖动"视频"列表第{cls.i}个至屏幕中间')
|
|
|
- time.sleep(3)
|
|
|
- driver.execute_script("arguments[0].scrollIntoView({block:'center',inline:'center'})", video_element)
|
|
|
-
|
|
|
- # video_title = video_element.find_elements(By.XPATH, '//wx-view[@class="video_title videolist--video_title"]')[cls.i-1].text
|
|
|
- # cover_url = video_element.find_elements(By.XPATH, '//wx-image[@class="itemimage videolist--itemimage"]')[cls.i-1].get_attribute('src')
|
|
|
- # play_cnt = video_element.find_elements(By.XPATH, '//wx-view[@class="clickbox videolist--clickbox"]')[cls.i-1].text
|
|
|
-
|
|
|
- video_title = video_element.find_elements(By.XPATH, '//wx-view[@class="video_title videolist--video_title"]')[index+i].text
|
|
|
- cover_url = video_element.find_elements(By.XPATH, '//wx-image[@class="itemimage videolist--itemimage"]')[index+i].get_attribute('src')
|
|
|
- play_cnt = video_element.find_elements(By.XPATH, '//wx-view[@class="clickbox videolist--clickbox"]')[index+i].text
|
|
|
-
|
|
|
- play_cnt = int(float(play_cnt.replace("阅读数", "").strip().split("万")[0]) * 10000)
|
|
|
- out_video_id = md5(video_title.encode('utf8')).hexdigest()
|
|
|
- video_dict = {
|
|
|
- 'video_title': video_title,
|
|
|
- 'video_id': out_video_id,
|
|
|
- 'play_cnt': play_cnt,
|
|
|
- 'comment_cnt': 0,
|
|
|
- 'like_cnt': 0,
|
|
|
- 'share_cnt': 0,
|
|
|
- 'publish_time_stamp': int(time.time()),
|
|
|
- 'publish_time_str': time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(int(time.time()))),
|
|
|
- 'user_name': "zhiqingtiantiankan",
|
|
|
- 'user_id': "zhiqingtiantiankan",
|
|
|
- 'avatar_url': cover_url,
|
|
|
- 'cover_url': cover_url,
|
|
|
- 'session': f"zhiqingtiantiankan-{int(time.time())}"
|
|
|
- }
|
|
|
- for k, v in video_dict.items():
|
|
|
- Common.logger(log_type, crawler).info(f"{k}:{v}")
|
|
|
-
|
|
|
- if video_title is None or cover_url is None:
|
|
|
- Common.logger(log_type, crawler).info("无效视频\n")
|
|
|
- elif any(str(word) if str(word) in video_title else False for word in
|
|
|
- cls.zhiqingtiantiankan_config(log_type, crawler, "filter", env)) is True:
|
|
|
- Common.logger(log_type, crawler).info('已中过滤词\n')
|
|
|
- elif cls.repeat_out_video_id(log_type, crawler, out_video_id, env) != 0:
|
|
|
+ if video_url is None:
|
|
|
+ Common.logger(log_type, crawler).info("未获取到视频播放地址\n")
|
|
|
+ driver.press_keycode(AndroidKey.BACK)
|
|
|
+ elif cls.repeat_video_url(log_type, crawler, video_url, env) != 0:
|
|
|
Common.logger(log_type, crawler).info('视频已下载\n')
|
|
|
+ driver.press_keycode(AndroidKey.BACK)
|
|
|
else:
|
|
|
- video_url = cls.get_video_url(log_type, crawler, driver, video_element)
|
|
|
-
|
|
|
- if video_url is None:
|
|
|
- Common.logger(log_type, crawler).info("未获取到视频播放地址\n")
|
|
|
- driver.press_keycode(AndroidKey.BACK)
|
|
|
- elif cls.repeat_video_url(log_type, crawler, video_url, env) != 0:
|
|
|
- Common.logger(log_type, crawler).info('视频已下载\n')
|
|
|
- driver.press_keycode(AndroidKey.BACK)
|
|
|
- else:
|
|
|
- video_dict["video_url"] = video_url
|
|
|
- Common.logger(log_type, crawler).info(f"video_url:{video_url}\n")
|
|
|
- # driver.press_keycode(AndroidKey.BACK)
|
|
|
- cls.download_publish(log_type, crawler, video_dict, env, driver)
|
|
|
- Common.logger(log_type, crawler).info('已抓取完一组视频，休眠10秒\n')
|
|
|
- time.sleep(10)
|
|
|
- index = index + len(video_element_temp)
|
|
|
- except Exception as e:
|
|
|
- Common.logger(log_type, crawler).info(f"get_videoList:{e}\n")
|
|
|
- cls.i = 0
|
|
|
- return
|
|
|
+ video_dict["video_url"] = video_url
|
|
|
+ Common.logger(log_type, crawler).info(f"video_url:{video_url}\n")
|
|
|
+ # driver.press_keycode(AndroidKey.BACK)
|
|
|
+ cls.download_publish(log_type, crawler, video_dict, env, driver)
|
|
|
+ Common.logger(log_type, crawler).info('已抓取完一组视频，休眠10秒\n')
|
|
|
+ time.sleep(10)
|
|
|
+ index = index + len(video_element_temp)
|
|
|
+ # except Exception as e:
|
|
|
+ # Common.logger(log_type, crawler).info(f"get_videoList:{e}\n")
|
|
|
+ # cls.i = 0
|
|
|
+ # return
|
|
|
|
|
|
@classmethod
|
|
|
def download_publish(cls, log_type, crawler, video_dict, env, driver: WebDriver):
|
