|
|
@@ -25,76 +25,76 @@ class GanggangdouchuanRecommend:
|
|
|
|
|
|
@classmethod
|
|
|
def start_wechat(cls, log_type, crawler, env):
|
|
|
- # try:
|
|
|
- if env == "dev":
|
|
|
- chromedriverExecutable = '/Users/wangkun/Downloads/chromedriver/chromedriver_v107/chromedriver'
|
|
|
- # chromedriverExecutable = 'C:\\chromedriver\\chromedriver.exe' # 阿里云 Windows
|
|
|
- else:
|
|
|
- chromedriverExecutable = '/Users/piaoquan/Downloads/chromedriver' # Mac 爬虫机器
|
|
|
- # chromedriverExecutable = 'C:\\chromedriver\\chromedriver.exe' # 阿里云 Windows
|
|
|
+ try:
|
|
|
+ if env == "dev":
|
|
|
+ chromedriverExecutable = '/Users/wangkun/Downloads/chromedriver/chromedriver_v107/chromedriver'
|
|
|
+ # chromedriverExecutable = 'C:\\chromedriver\\chromedriver.exe' # 阿里云 Windows
|
|
|
+ else:
|
|
|
+ chromedriverExecutable = '/Users/piaoquan/Downloads/chromedriver' # Mac 爬虫机器
|
|
|
+ # chromedriverExecutable = 'C:\\chromedriver\\chromedriver.exe' # 阿里云 Windows
|
|
|
|
|
|
- Common.logger(log_type, crawler).info('启动微信')
|
|
|
- Common.logging(log_type, crawler, env, '启动微信')
|
|
|
- caps = {
|
|
|
- "platformName": "Android", # 手机操作系统 Android / iOS
|
|
|
- "deviceName": "a0a65126", # 连接的设备名（模拟器或真机），安卓可以随便写
|
|
|
- # "udid": "emulator-5554", # 指定 adb devices 中的哪一台设备
|
|
|
- "platforVersion": "11", # 手机对应的系统版本
|
|
|
- "appPackage": "com.tencent.mm", # 被测APP的包名，乐活圈 Android
|
|
|
- "appActivity": ".ui.LauncherUI", # 启动的Activity名
|
|
|
- "autoGrantPermissions": "true", # 让 appium 自动授权 base 权限，
|
|
|
- # 如果 noReset 为 True，则该条不生效（该参数为 Android 独有），对应的值为 True 或 False
|
|
|
- "unicodekeyboard": True, # 使用自带输入法，输入中文时填True
|
|
|
- "resetkeyboard": True, # 执行完程序恢复原来输入法
|
|
|
- "noReset": True, # 不重置APP
|
|
|
- "printPageSourceOnFailure": True, # 找不到元素时，appium log 会完整记录当前页面的 pagesource
|
|
|
- "newCommandTimeout": 6000, # 初始等待时间
|
|
|
- "automationName": "UiAutomator2", # 使用引擎，默认为 Appium，
|
|
|
- # 其中 Appium、UiAutomator2、Selendroid、Espresso 用于 Android，XCUITest 用于 iOS
|
|
|
- "showChromedriverLog": True,
|
|
|
- 'enableWebviewDetailsCollection': True,
|
|
|
- 'setWebContentsDebuggingEnabled': True,
|
|
|
- 'recreateChromeDriverSessions': True,
|
|
|
- 'chromedriverExecutable': chromedriverExecutable,
|
|
|
- "chromeOptions": {"androidProcess": "com.tencent.mm:appbrand0"},
|
|
|
- 'browserName': ''
|
|
|
- }
|
|
|
- driver = webdriver.Remote("http://localhost:4723/wd/hub", caps) # 指定 Appium 端口号：4723
|
|
|
- driver.implicitly_wait(30)
|
|
|
- # 向下滑动页面，展示出小程序选择面板
|
|
|
- for i in range(120):
|
|
|
- try:
|
|
|
- # 发现微信消息 TAB，代表微信已启动成功
|
|
|
- if driver.find_elements(By.ID, 'com.tencent.mm:id/f2s'):
|
|
|
- break
|
|
|
- # 发现并关闭系统菜单栏
|
|
|
- elif driver.find_element(By.ID, 'com.android.systemui:id/dismiss_view'):
|
|
|
- Common.logger(log_type, crawler).info('发现并关闭系统下拉菜单栏')
|
|
|
- Common.logging(log_type, crawler, env, '发现并关闭系统下拉菜单栏')
|
|
|
- driver.find_element(By.ID, 'com.android.systemui:id/dismiss_view').click()
|
|
|
- else:
|
|
|
- pass
|
|
|
- except NoSuchElementException:
|
|
|
- time.sleep(1)
|
|
|
- Common.logger(log_type, crawler).info('下滑，展示小程序选择面板')
|
|
|
- Common.logging(log_type, crawler, env, '下滑，展示小程序选择面板')
|
|
|
- size = driver.get_window_size()
|
|
|
- driver.swipe(int(size['width'] * 0.5), int(size['height'] * 0.2),
|
|
|
- int(size['width'] * 0.5), int(size['height'] * 0.8), 200)
|
|
|
- # 打开小程序"刚刚都传"
|
|
|
- time.sleep(5)
|
|
|
- Common.logger(log_type, crawler).info('打开小程序"刚刚都传"')
|
|
|
- Common.logging(log_type, crawler, env, '打开小程序"刚刚都传"')
|
|
|
- driver.find_elements(By.XPATH, '//*[@text="刚刚都传"]')[-1].click()
|
|
|
- # time.sleep(40)
|
|
|
- time.sleep(10)
|
|
|
- cls.get_videoList(log_type, crawler, env, driver)
|
|
|
- driver.quit()
|
|
|
- Common.logger(log_type, crawler).info('退出微信成功\n')
|
|
|
- Common.logging(log_type, crawler, env, '退出微信成功\n')
|
|
|
- # except Exception as e:
|
|
|
- # Common.logger(log_type, crawler).error(f'start_wechat异常:{e}\n')
|
|
|
- # Common.logging(log_type, crawler, env, f'start_wechat异常:{e}\n')
|
|
|
+ Common.logger(log_type, crawler).info('启动微信')
|
|
|
+ Common.logging(log_type, crawler, env, '启动微信')
|
|
|
+ caps = {
|
|
|
+ "platformName": "Android", # 手机操作系统 Android / iOS
|
|
|
+ "deviceName": "a0a65126", # 连接的设备名（模拟器或真机），安卓可以随便写
|
|
|
+ # "udid": "emulator-5554", # 指定 adb devices 中的哪一台设备
|
|
|
+ "platforVersion": "11", # 手机对应的系统版本
|
|
|
+ "appPackage": "com.tencent.mm", # 被测APP的包名，乐活圈 Android
|
|
|
+ "appActivity": ".ui.LauncherUI", # 启动的Activity名
|
|
|
+ "autoGrantPermissions": "true", # 让 appium 自动授权 base 权限，
|
|
|
+ # 如果 noReset 为 True，则该条不生效（该参数为 Android 独有），对应的值为 True 或 False
|
|
|
+ "unicodekeyboard": True, # 使用自带输入法，输入中文时填True
|
|
|
+ "resetkeyboard": True, # 执行完程序恢复原来输入法
|
|
|
+ "noReset": True, # 不重置APP
|
|
|
+ "printPageSourceOnFailure": True, # 找不到元素时，appium log 会完整记录当前页面的 pagesource
|
|
|
+ "newCommandTimeout": 6000, # 初始等待时间
|
|
|
+ "automationName": "UiAutomator2", # 使用引擎，默认为 Appium，
|
|
|
+ # 其中 Appium、UiAutomator2、Selendroid、Espresso 用于 Android，XCUITest 用于 iOS
|
|
|
+ "showChromedriverLog": True,
|
|
|
+ 'enableWebviewDetailsCollection': True,
|
|
|
+ 'setWebContentsDebuggingEnabled': True,
|
|
|
+ 'recreateChromeDriverSessions': True,
|
|
|
+ 'chromedriverExecutable': chromedriverExecutable,
|
|
|
+ "chromeOptions": {"androidProcess": "com.tencent.mm:appbrand0"},
|
|
|
+ 'browserName': ''
|
|
|
+ }
|
|
|
+ driver = webdriver.Remote("http://localhost:4723/wd/hub", caps) # 指定 Appium 端口号：4723
|
|
|
+ driver.implicitly_wait(30)
|
|
|
+ # 向下滑动页面，展示出小程序选择面板
|
|
|
+ for i in range(120):
|
|
|
+ try:
|
|
|
+ # 发现微信消息 TAB，代表微信已启动成功
|
|
|
+ if driver.find_elements(By.ID, 'com.tencent.mm:id/f2s'):
|
|
|
+ break
|
|
|
+ # 发现并关闭系统菜单栏
|
|
|
+ elif driver.find_element(By.ID, 'com.android.systemui:id/dismiss_view'):
|
|
|
+ Common.logger(log_type, crawler).info('发现并关闭系统下拉菜单栏')
|
|
|
+ Common.logging(log_type, crawler, env, '发现并关闭系统下拉菜单栏')
|
|
|
+ driver.find_element(By.ID, 'com.android.systemui:id/dismiss_view').click()
|
|
|
+ else:
|
|
|
+ pass
|
|
|
+ except NoSuchElementException:
|
|
|
+ time.sleep(1)
|
|
|
+ Common.logger(log_type, crawler).info('下滑，展示小程序选择面板')
|
|
|
+ Common.logging(log_type, crawler, env, '下滑，展示小程序选择面板')
|
|
|
+ size = driver.get_window_size()
|
|
|
+ driver.swipe(int(size['width'] * 0.5), int(size['height'] * 0.2),
|
|
|
+ int(size['width'] * 0.5), int(size['height'] * 0.8), 200)
|
|
|
+ # 打开小程序"刚刚都传"
|
|
|
+ time.sleep(5)
|
|
|
+ Common.logger(log_type, crawler).info('打开小程序"刚刚都传"')
|
|
|
+ Common.logging(log_type, crawler, env, '打开小程序"刚刚都传"')
|
|
|
+ driver.find_elements(By.XPATH, '//*[@text="刚刚都传"]')[-1].click()
|
|
|
+ # time.sleep(40)
|
|
|
+ time.sleep(10)
|
|
|
+ cls.get_videoList(log_type, crawler, env, driver)
|
|
|
+ driver.quit()
|
|
|
+ Common.logger(log_type, crawler).info('退出微信成功\n')
|
|
|
+ Common.logging(log_type, crawler, env, '退出微信成功\n')
|
|
|
+ except Exception as e:
|
|
|
+ Common.logger(log_type, crawler).error(f'start_wechat异常:{e}\n')
|
|
|
+ Common.logging(log_type, crawler, env, f'start_wechat异常:{e}\n')
|
|
|
|
|
|
@classmethod
|
|
|
def get_video_url(cls, log_type, crawler, env, driver: WebDriver, video_element):
|
|
|
@@ -185,95 +185,95 @@ class GanggangdouchuanRecommend:
|
|
|
time.sleep(10)
|
|
|
index = 0
|
|
|
while True:
|
|
|
- # try:
|
|
|
- if cls.search_elements(log_type, crawler, env, driver, '//wx-view[@class="lists"]') is None:
|
|
|
- Common.logger(log_type, crawler).info('窗口已销毁\n')
|
|
|
- Common.logging(log_type, crawler, env, '窗口已销毁\n')
|
|
|
- return
|
|
|
-
|
|
|
- Common.logger(log_type, crawler).info('获取视频列表\n')
|
|
|
- Common.logging(log_type, crawler, env, '获取视频列表\n')
|
|
|
- video_elements = cls.search_elements(log_type, crawler, env, driver, '//wx-view[@class="list"]')
|
|
|
- if video_elements is None:
|
|
|
- Common.logger(log_type, crawler).warning(f'video_elements:{video_elements}')
|
|
|
- Common.logging(log_type, crawler, env, f'video_elements:{video_elements}')
|
|
|
- return
|
|
|
+ try:
|
|
|
+ if cls.search_elements(log_type, crawler, env, driver, '//wx-view[@class="lists"]') is None:
|
|
|
+ Common.logger(log_type, crawler).info('窗口已销毁\n')
|
|
|
+ Common.logging(log_type, crawler, env, '窗口已销毁\n')
|
|
|
+ return
|
|
|
|
|
|
- video_element_temp = video_elements[index:]
|
|
|
- if len(video_element_temp) == 0:
|
|
|
- Common.logger(log_type, crawler).info('到底啦~~~~~~~~~~~~~\n')
|
|
|
- Common.logging(log_type, crawler, env, '到底啦~~~~~~~~~~~~~\n')
|
|
|
- return
|
|
|
+ Common.logger(log_type, crawler).info('获取视频列表\n')
|
|
|
+ Common.logging(log_type, crawler, env, '获取视频列表\n')
|
|
|
+ video_elements = cls.search_elements(log_type, crawler, env, driver, '//wx-view[@class="list"]')
|
|
|
+ if video_elements is None:
|
|
|
+ Common.logger(log_type, crawler).warning(f'video_elements:{video_elements}')
|
|
|
+ Common.logging(log_type, crawler, env, f'video_elements:{video_elements}')
|
|
|
+ return
|
|
|
|
|
|
- for i, video_element in enumerate(video_element_temp):
|
|
|
- if video_element is None:
|
|
|
- Common.logger(log_type, crawler).info('到底啦~\n')
|
|
|
- Common.logging(log_type, crawler, env, '到底啦~\n')
|
|
|
+ video_element_temp = video_elements[index:]
|
|
|
+ if len(video_element_temp) == 0:
|
|
|
+ Common.logger(log_type, crawler).info('到底啦~~~~~~~~~~~~~\n')
|
|
|
+ Common.logging(log_type, crawler, env, '到底啦~~~~~~~~~~~~~\n')
|
|
|
return
|
|
|
- cls.i += 1
|
|
|
- cls.search_elements(log_type, crawler, env, driver, '//wx-view[@class="list"]')
|
|
|
- Common.logger(log_type, crawler).info(f'拖动"视频"列表第{cls.i}个至屏幕中间')
|
|
|
- Common.logging(log_type, crawler, env, f'拖动"视频"列表第{cls.i}个至屏幕中间')
|
|
|
- time.sleep(3)
|
|
|
- driver.execute_script("arguments[0].scrollIntoView({block:'center',inline:'center'})", video_element)
|
|
|
|
|
|
- # video_title = video_element.find_elements(By.XPATH, '//wx-view[@class="title"]//span[2]')[cls.i - 1].text
|
|
|
- # cover_url = video_element.find_elements(By.XPATH, '//wx-image[@class="poster-img"]')[cls.i - 1].get_attribute('src')
|
|
|
+ for i, video_element in enumerate(video_element_temp):
|
|
|
+ if video_element is None:
|
|
|
+ Common.logger(log_type, crawler).info('到底啦~\n')
|
|
|
+ Common.logging(log_type, crawler, env, '到底啦~\n')
|
|
|
+ return
|
|
|
+ cls.i += 1
|
|
|
+ cls.search_elements(log_type, crawler, env, driver, '//wx-view[@class="list"]')
|
|
|
+ Common.logger(log_type, crawler).info(f'拖动"视频"列表第{cls.i}个至屏幕中间')
|
|
|
+ Common.logging(log_type, crawler, env, f'拖动"视频"列表第{cls.i}个至屏幕中间')
|
|
|
+ time.sleep(3)
|
|
|
+ driver.execute_script("arguments[0].scrollIntoView({block:'center',inline:'center'})", video_element)
|
|
|
+
|
|
|
+ # video_title = video_element.find_elements(By.XPATH, '//wx-view[@class="title"]//span[2]')[cls.i - 1].text
|
|
|
+ # cover_url = video_element.find_elements(By.XPATH, '//wx-image[@class="poster-img"]')[cls.i - 1].get_attribute('src')
|
|
|
|
|
|
- video_title = video_element.find_elements(By.XPATH, '//wx-view[@class="title"]//span[2]')[index+i].text
|
|
|
- cover_url = video_element.find_elements(By.XPATH, '//wx-image[@class="poster-img"]')[index+i].get_attribute('src')
|
|
|
+ video_title = video_element.find_elements(By.XPATH, '//wx-view[@class="title"]//span[2]')[index+i].text
|
|
|
+ cover_url = video_element.find_elements(By.XPATH, '//wx-image[@class="poster-img"]')[index+i].get_attribute('src')
|
|
|
|
|
|
- out_video_id = md5(video_title.encode('utf8')).hexdigest()
|
|
|
- video_dict = {
|
|
|
- 'video_title': video_title,
|
|
|
- 'video_id': out_video_id,
|
|
|
- 'play_cnt': 0,
|
|
|
- 'comment_cnt': 0,
|
|
|
- 'like_cnt': 0,
|
|
|
- 'share_cnt': 0,
|
|
|
- 'publish_time_stamp': int(time.time()),
|
|
|
- 'publish_time_str': time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(int(time.time()))),
|
|
|
- 'user_name': "ganggangdouchuan",
|
|
|
- 'user_id': "ganggangdouchuan",
|
|
|
- 'avatar_url': cover_url,
|
|
|
- 'cover_url': cover_url,
|
|
|
- 'session': f"ganggangdouchuan-{int(time.time())}"
|
|
|
- }
|
|
|
- for k, v in video_dict.items():
|
|
|
- Common.logger(log_type, crawler).info(f"{k}:{v}")
|
|
|
- Common.logging(log_type, crawler, env, f'video_dict:{video_dict}')
|
|
|
+ out_video_id = md5(video_title.encode('utf8')).hexdigest()
|
|
|
+ video_dict = {
|
|
|
+ 'video_title': video_title,
|
|
|
+ 'video_id': out_video_id,
|
|
|
+ 'play_cnt': 0,
|
|
|
+ 'comment_cnt': 0,
|
|
|
+ 'like_cnt': 0,
|
|
|
+ 'share_cnt': 0,
|
|
|
+ 'publish_time_stamp': int(time.time()),
|
|
|
+ 'publish_time_str': time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(int(time.time()))),
|
|
|
+ 'user_name': "ganggangdouchuan",
|
|
|
+ 'user_id': "ganggangdouchuan",
|
|
|
+ 'avatar_url': cover_url,
|
|
|
+ 'cover_url': cover_url,
|
|
|
+ 'session': f"ganggangdouchuan-{int(time.time())}"
|
|
|
+ }
|
|
|
+ for k, v in video_dict.items():
|
|
|
+ Common.logger(log_type, crawler).info(f"{k}:{v}")
|
|
|
+ Common.logging(log_type, crawler, env, f'video_dict:{video_dict}')
|
|
|
|
|
|
- if video_title is None or cover_url is None:
|
|
|
- Common.logger(log_type, crawler).info("无效视频\n")
|
|
|
- Common.logging(log_type, crawler, env, "无效视频\n")
|
|
|
- elif cls.repeat_out_video_id(log_type, crawler, out_video_id, env) != 0:
|
|
|
- Common.logger(log_type, crawler).info('视频已下载\n')
|
|
|
- Common.logging(log_type, crawler, env, '视频已下载\n')
|
|
|
- else:
|
|
|
- video_url = cls.get_video_url(log_type, crawler, env, driver, video_element)
|
|
|
- Common.logger(log_type, crawler).info(f'video_url:{video_url}')
|
|
|
- Common.logging(log_type, crawler, env, f'video_url:{video_url}')
|
|
|
- if video_url is None:
|
|
|
- Common.logger(log_type, crawler).info("未获取到视频播放地址\n")
|
|
|
- Common.logging(log_type, crawler, env, "未获取到视频播放地址\n")
|
|
|
- driver.press_keycode(AndroidKey.BACK)
|
|
|
- elif cls.repeat_video_url(log_type, crawler, video_url, env) != 0:
|
|
|
+ if video_title is None or cover_url is None:
|
|
|
+ Common.logger(log_type, crawler).info("无效视频\n")
|
|
|
+ Common.logging(log_type, crawler, env, "无效视频\n")
|
|
|
+ elif cls.repeat_out_video_id(log_type, crawler, out_video_id, env) != 0:
|
|
|
Common.logger(log_type, crawler).info('视频已下载\n')
|
|
|
Common.logging(log_type, crawler, env, '视频已下载\n')
|
|
|
- driver.press_keycode(AndroidKey.BACK)
|
|
|
else:
|
|
|
- video_dict["video_url"]=video_url
|
|
|
- cls.download_publish(log_type, crawler, video_dict, env, driver)
|
|
|
- Common.logger(log_type, crawler).info('已抓取完一组视频，休眠10秒\n')
|
|
|
- Common.logging(log_type, crawler, env, '已抓取完一组视频，休眠10秒\n')
|
|
|
- time.sleep(10)
|
|
|
- index = index + len(video_element_temp)
|
|
|
+ video_url = cls.get_video_url(log_type, crawler, env, driver, video_element)
|
|
|
+ Common.logger(log_type, crawler).info(f'video_url:{video_url}')
|
|
|
+ Common.logging(log_type, crawler, env, f'video_url:{video_url}')
|
|
|
+ if video_url is None:
|
|
|
+ Common.logger(log_type, crawler).info("未获取到视频播放地址\n")
|
|
|
+ Common.logging(log_type, crawler, env, "未获取到视频播放地址\n")
|
|
|
+ driver.press_keycode(AndroidKey.BACK)
|
|
|
+ elif cls.repeat_video_url(log_type, crawler, video_url, env) != 0:
|
|
|
+ Common.logger(log_type, crawler).info('视频已下载\n')
|
|
|
+ Common.logging(log_type, crawler, env, '视频已下载\n')
|
|
|
+ driver.press_keycode(AndroidKey.BACK)
|
|
|
+ else:
|
|
|
+ video_dict["video_url"]=video_url
|
|
|
+ cls.download_publish(log_type, crawler, video_dict, env, driver)
|
|
|
+ Common.logger(log_type, crawler).info('已抓取完一组视频，休眠10秒\n')
|
|
|
+ Common.logging(log_type, crawler, env, '已抓取完一组视频，休眠10秒\n')
|
|
|
+ time.sleep(10)
|
|
|
+ index = index + len(video_element_temp)
|
|
|
|
|
|
- # except Exception as e:
|
|
|
- # Common.logger(log_type, crawler).error(f'get_videoList异常:{e}\n')
|
|
|
- # Common.logging(log_type, crawler, env, f'get_videoList异常:{e}\n')
|
|
|
- # cls.i = 0
|
|
|
- # return
|
|
|
+ except Exception as e:
|
|
|
+ Common.logger(log_type, crawler).error(f'get_videoList异常:{e}\n')
|
|
|
+ Common.logging(log_type, crawler, env, f'get_videoList异常:{e}\n')
|
|
|
+ cls.i = 0
|
|
|
+ return
|
|
|
|
|
|
@classmethod
|
|
|
def download_publish(cls, log_type, crawler, video_dict, env, driver: WebDriver):
|
