|
|
@@ -25,39 +25,43 @@ class Recommend:
|
|
|
# 启动微信，并打开视频号
|
|
|
@classmethod
|
|
|
def start_wechat(cls, log_type):
|
|
|
- Common.logger(log_type).info('启动微信')
|
|
|
- caps = {
|
|
|
- "platformName": "Android", # 手机操作系统 Android / iOS
|
|
|
- "deviceName": "Android", # 连接的设备名（模拟器或真机），安卓可以随便写
|
|
|
- "platforVersion": "11", # 手机对应的系统版本（Android 11）
|
|
|
- "appPackage": "com.tencent.mm", # 被测APP的包名，乐活圈 Android
|
|
|
- "appActivity": ".ui.LauncherUI", # 启动的Activity名
|
|
|
- "autoGrantPermissions": "true", # 让 appium 自动授权 base 权限，
|
|
|
- # 如果 noReset 为 True，则该条不生效（该参数为 Android 独有），对应的值为 True 或 False
|
|
|
- "unicodekeyboard": True, # 使用自带输入法，输入中文时填True
|
|
|
- "resetkeyboard": True, # 执行完程序恢复原来输入法
|
|
|
- "noReset": True, # 不重置APP
|
|
|
- "printPageSourceOnFailure": True, # 找不到元素时，appium log 会完整记录当前页面的 pagesource
|
|
|
- "newCommandTimeout": 6000, # 初始等待时间
|
|
|
- "automationName": "UiAutomator2" # 使用引擎，默认为 Appium，
|
|
|
- # 其中 Appium、UiAutomator2、Selendroid、Espresso 用于 Android，XCUITest 用于 iOS
|
|
|
- }
|
|
|
- # global driver
|
|
|
- driver = webdriver.Remote("http://localhost:4723/wd/hub", caps)
|
|
|
- driver.implicitly_wait(10)
|
|
|
-
|
|
|
- Common.logger(log_type).info('点击发现TAB')
|
|
|
- driver.find_elements(By.ID, 'com.tencent.mm:id/f2s')[2].click()
|
|
|
-
|
|
|
- Common.logger(log_type).info('点击视频号\n')
|
|
|
- driver.find_elements(By.ID, 'com.tencent.mm:id/gv6')[1].click()
|
|
|
- time.sleep(5)
|
|
|
-
|
|
|
- cls.get_feeds(log_type, driver)
|
|
|
-
|
|
|
- Common.logger(log_type).info('休眠 3s')
|
|
|
- time.sleep(3)
|
|
|
- cls.quit(log_type, driver)
|
|
|
+ try:
|
|
|
+ Common.logger(log_type).info('启动微信')
|
|
|
+ caps = {
|
|
|
+ "platformName": "Android", # 手机操作系统 Android / iOS
|
|
|
+ "deviceName": "Android", # 连接的设备名（模拟器或真机），安卓可以随便写
|
|
|
+ "platforVersion": "11", # 手机对应的系统版本（Android 11）
|
|
|
+ "appPackage": "com.tencent.mm", # 被测APP的包名，乐活圈 Android
|
|
|
+ "appActivity": ".ui.LauncherUI", # 启动的Activity名
|
|
|
+ "autoGrantPermissions": "true", # 让 appium 自动授权 base 权限，
|
|
|
+ # 如果 noReset 为 True，则该条不生效（该参数为 Android 独有），对应的值为 True 或 False
|
|
|
+ "unicodekeyboard": True, # 使用自带输入法，输入中文时填True
|
|
|
+ "resetkeyboard": True, # 执行完程序恢复原来输入法
|
|
|
+ "noReset": True, # 不重置APP
|
|
|
+ "printPageSourceOnFailure": True, # 找不到元素时，appium log 会完整记录当前页面的 pagesource
|
|
|
+ "newCommandTimeout": 6000, # 初始等待时间
|
|
|
+ "automationName": "UiAutomator2" # 使用引擎，默认为 Appium，
|
|
|
+ # 其中 Appium、UiAutomator2、Selendroid、Espresso 用于 Android，XCUITest 用于 iOS
|
|
|
+ }
|
|
|
+ # global driver
|
|
|
+ driver = webdriver.Remote("http://localhost:4723/wd/hub", caps)
|
|
|
+ driver.implicitly_wait(10)
|
|
|
+
|
|
|
+ Common.logger(log_type).info('点击发现TAB')
|
|
|
+ driver.find_elements(By.ID, 'com.tencent.mm:id/f2s')[2].click()
|
|
|
+
|
|
|
+ Common.logger(log_type).info('点击视频号\n')
|
|
|
+ driver.find_elements(By.ID, 'com.tencent.mm:id/gv6')[1].click()
|
|
|
+ time.sleep(5)
|
|
|
+
|
|
|
+ cls.get_feeds(log_type, driver)
|
|
|
+
|
|
|
+ Common.logger(log_type).info('休眠 3s')
|
|
|
+ time.sleep(3)
|
|
|
+ cls.quit(log_type, driver)
|
|
|
+
|
|
|
+ except Exception as e:
|
|
|
+ Common.logger(log_type).error('start_wechat异常', e)
|
|
|
|
|
|
# 退出 APP
|
|
|
@classmethod
|
|
|
@@ -88,164 +92,167 @@ class Recommend:
|
|
|
# 操作安卓手机，自己滑动首页视频，并获取视频信息
|
|
|
@classmethod
|
|
|
def get_feeds(cls, log_type, driver: WebDriver):
|
|
|
- driver.implicitly_wait(10)
|
|
|
- for i in range(10):
|
|
|
- # 关闭页面弹窗
|
|
|
- try:
|
|
|
- driver.find_element(By.XPATH, '//*[@text="我知道了"]').click()
|
|
|
- driver.find_element(By.XPATH, '//*[@text="知道了"]').click()
|
|
|
- Common.logger(log_type).info('已关闭未成年模式弹窗')
|
|
|
- except NoSuchElementException:
|
|
|
- pass
|
|
|
-
|
|
|
- try:
|
|
|
- driver.find_element(By.ID, 'com.tencent.mm:id/dkf')
|
|
|
- Common.logger(log_type).info('直播视频，向上滑动页面\n')
|
|
|
- driver.swipe(500, 1000, 500, 300, 300)
|
|
|
- except NoSuchElementException:
|
|
|
- pass
|
|
|
-
|
|
|
- # Common.logger(log_type).info('暂停播放')
|
|
|
- pause_btn = driver.find_element(By.ID, 'com.tencent.mm:id/eh4')
|
|
|
- pause_btn.click()
|
|
|
-
|
|
|
- # 视频标题
|
|
|
- try:
|
|
|
- title_id = driver.find_element(By.ID, 'com.tencent.mm:id/ki5')
|
|
|
- video_title = title_id.get_attribute('name')
|
|
|
- except NoSuchElementException:
|
|
|
- video_title = ''
|
|
|
-
|
|
|
- # 点击播放器，获取视频时长
|
|
|
- start_time = driver.find_element(By.ID, 'com.tencent.mm:id/l59').get_attribute('name')
|
|
|
- start_time = int(start_time.split(':')[0])*60 + int(start_time.split(':')[-1])
|
|
|
- try:
|
|
|
- end_time = driver.find_element(By.ID, 'com.tencent.mm:id/l7i').get_attribute('name')
|
|
|
- except NoSuchElementException:
|
|
|
- end_time = driver.find_element(By.ID, 'com.tencent.mm:id/g73').get_attribute('name')
|
|
|
- end_time = int(end_time.split(':')[0]) * 60 + int(end_time.split(':')[-1])
|
|
|
- duration = start_time + end_time
|
|
|
-
|
|
|
- # 点赞
|
|
|
- like_id = driver.find_element(By.ID, 'com.tencent.mm:id/k04')
|
|
|
- like_cnt = like_id.get_attribute('name')
|
|
|
- if like_cnt == "" or like_cnt == "喜欢":
|
|
|
- like_cnt = 0
|
|
|
- elif '万' in like_cnt:
|
|
|
- like_cnt = float(like_cnt.split('万')[0]) * 10000
|
|
|
- elif '万+' in like_cnt:
|
|
|
- like_cnt = float(like_cnt.split('万+')[0]) * 10000
|
|
|
- else:
|
|
|
- like_cnt = float(like_cnt)
|
|
|
-
|
|
|
- # 分享
|
|
|
- share_id = driver.find_element(By.ID, 'com.tencent.mm:id/jhv')
|
|
|
- share_cnt = share_id.get_attribute('name')
|
|
|
- if share_cnt == "" or share_cnt == "转发":
|
|
|
- share_cnt = 0
|
|
|
- elif '万' in share_cnt:
|
|
|
- share_cnt = float(share_cnt.split('万')[0]) * 10000
|
|
|
- elif '万+' in share_cnt:
|
|
|
- share_cnt = float(share_cnt.split('万+')[0]) * 10000
|
|
|
- else:
|
|
|
- share_cnt = float(share_cnt)
|
|
|
-
|
|
|
- # 收藏
|
|
|
- favorite_id = driver.find_element(By.ID, 'com.tencent.mm:id/fnp')
|
|
|
- favorite_cnt = favorite_id.get_attribute('name')
|
|
|
- if favorite_cnt == "" or favorite_cnt == "收藏":
|
|
|
- favorite_cnt = 0
|
|
|
- elif '万' in favorite_cnt:
|
|
|
- favorite_cnt = float(favorite_cnt.split('万')[0]) * 10000
|
|
|
- elif '万+' in favorite_cnt:
|
|
|
- favorite_cnt = float(favorite_cnt.split('万+')[0]) * 10000
|
|
|
- else:
|
|
|
- favorite_cnt = float(favorite_cnt)
|
|
|
-
|
|
|
- # 评论
|
|
|
- comment_id = driver.find_element(By.ID, 'com.tencent.mm:id/bje')
|
|
|
- comment_cnt = comment_id.get_attribute('name')
|
|
|
- if comment_cnt == "" or comment_cnt == "评论":
|
|
|
- comment_cnt = 0
|
|
|
- elif '万' in comment_cnt:
|
|
|
- comment_cnt = float(comment_cnt.split('万')[0]) * 10000
|
|
|
- elif '万+' in comment_cnt:
|
|
|
- comment_cnt = float(comment_cnt.split('万+')[0]) * 10000
|
|
|
- else:
|
|
|
- comment_cnt = float(comment_cnt)
|
|
|
-
|
|
|
- # 用户名
|
|
|
- username_id = driver.find_element(By.ID, 'com.tencent.mm:id/hft')
|
|
|
- user_name = username_id.get_attribute('name')
|
|
|
-
|
|
|
- Common.logger(log_type).info('video_title:{}', video_title)
|
|
|
- Common.logger(log_type).info('duration:{}', duration)
|
|
|
- Common.logger(log_type).info('like_cnt:{}', like_cnt)
|
|
|
- Common.logger(log_type).info('share_cnt:{}', share_cnt)
|
|
|
- Common.logger(log_type).info('favorite_cnt:{}', favorite_cnt)
|
|
|
- Common.logger(log_type).info('comment_cnt:{}', comment_cnt)
|
|
|
- Common.logger(log_type).info('user_name:{}', user_name)
|
|
|
-
|
|
|
- # 判断无效视频
|
|
|
- if video_title == '' or user_name == '':
|
|
|
- Common.logger(log_type).info('向上滑动页面')
|
|
|
- driver.swipe(500, 1000, 500, 300, 300)
|
|
|
- Common.logger(log_type).info('无效视频\n')
|
|
|
-
|
|
|
- # 判断下载规则
|
|
|
- elif cls.download_rule(duration, like_cnt, share_cnt, favorite_cnt, comment_cnt) is False:
|
|
|
- Common.logger(log_type).info('向上滑动页面')
|
|
|
- driver.swipe(500, 1000, 500, 300, 300)
|
|
|
- Common.logger(log_type).info('不满足抓取规则\n')
|
|
|
-
|
|
|
- # 已下载表去重
|
|
|
- elif str(video_title) in [x for y in Feishu.get_values_batch(log_type, 'shipinhao', 'c77cf9') for x in y]:
|
|
|
- Common.logger(log_type).info('向上滑动页面')
|
|
|
- driver.swipe(500, 1000, 500, 300, 300)
|
|
|
- Common.logger(log_type).info('视频已下载\n')
|
|
|
-
|
|
|
- # feeds 表去重
|
|
|
- elif str(video_title) in [x for y in Feishu.get_values_batch(log_type, 'shipinhao', 'FSDlBy') for x in y]:
|
|
|
- Common.logger(log_type).info('向上滑动页面')
|
|
|
- driver.swipe(500, 1000, 500, 300, 300)
|
|
|
- Common.logger(log_type).info('视频已存在\n')
|
|
|
-
|
|
|
- # 分享给 windows 爬虫机
|
|
|
- else:
|
|
|
- share_id.click()
|
|
|
- driver.find_element(By.XPATH, '//*[@text="转发给朋友"]').click()
|
|
|
- driver.find_element(By.XPATH, '//*[@text="爬虫群"]').click()
|
|
|
- driver.find_element(By.ID, 'com.tencent.mm:id/guw').click()
|
|
|
+ try:
|
|
|
+ driver.implicitly_wait(10)
|
|
|
+ for i in range(10):
|
|
|
+ # 关闭页面弹窗
|
|
|
+ try:
|
|
|
+ driver.find_element(By.XPATH, '//*[@text="我知道了"]').click()
|
|
|
+ driver.find_element(By.XPATH, '//*[@text="知道了"]').click()
|
|
|
+ Common.logger(log_type).info('已关闭未成年模式弹窗')
|
|
|
+ except NoSuchElementException:
|
|
|
+ pass
|
|
|
+
|
|
|
+ try:
|
|
|
+ driver.find_element(By.ID, 'com.tencent.mm:id/dkf')
|
|
|
+ Common.logger(log_type).info('直播视频，向上滑动页面\n')
|
|
|
+ driver.swipe(500, 1500, 500, 500, 300)
|
|
|
+ except NoSuchElementException:
|
|
|
+ pass
|
|
|
+
|
|
|
+ # Common.logger(log_type).info('暂停播放')
|
|
|
+ pause_btn = driver.find_element(By.ID, 'com.tencent.mm:id/eh4')
|
|
|
+ pause_btn.click()
|
|
|
+
|
|
|
+ # 视频标题
|
|
|
+ try:
|
|
|
+ title_id = driver.find_element(By.ID, 'com.tencent.mm:id/ki5')
|
|
|
+ video_title = title_id.get_attribute('name')
|
|
|
+ except NoSuchElementException:
|
|
|
+ video_title = ''
|
|
|
+
|
|
|
+ # 点击播放器，获取视频时长
|
|
|
+ start_time = driver.find_element(By.ID, 'com.tencent.mm:id/l59').get_attribute('name')
|
|
|
+ start_time = int(start_time.split(':')[0])*60 + int(start_time.split(':')[-1])
|
|
|
+ try:
|
|
|
+ end_time = driver.find_element(By.ID, 'com.tencent.mm:id/l7i').get_attribute('name')
|
|
|
+ except NoSuchElementException:
|
|
|
+ end_time = driver.find_element(By.ID, 'com.tencent.mm:id/g73').get_attribute('name')
|
|
|
+ end_time = int(end_time.split(':')[0]) * 60 + int(end_time.split(':')[-1])
|
|
|
+ duration = start_time + end_time
|
|
|
+
|
|
|
+ # 点赞
|
|
|
+ like_id = driver.find_element(By.ID, 'com.tencent.mm:id/k04')
|
|
|
+ like_cnt = like_id.get_attribute('name')
|
|
|
+ if like_cnt == "" or like_cnt == "喜欢":
|
|
|
+ like_cnt = 0
|
|
|
+ elif '万' in like_cnt:
|
|
|
+ like_cnt = float(like_cnt.split('万')[0]) * 10000
|
|
|
+ elif '万+' in like_cnt:
|
|
|
+ like_cnt = float(like_cnt.split('万+')[0]) * 10000
|
|
|
+ else:
|
|
|
+ like_cnt = float(like_cnt)
|
|
|
+
|
|
|
+ # 分享
|
|
|
+ share_id = driver.find_element(By.ID, 'com.tencent.mm:id/jhv')
|
|
|
+ share_cnt = share_id.get_attribute('name')
|
|
|
+ if share_cnt == "" or share_cnt == "转发":
|
|
|
+ share_cnt = 0
|
|
|
+ elif '万' in share_cnt:
|
|
|
+ share_cnt = float(share_cnt.split('万')[0]) * 10000
|
|
|
+ elif '万+' in share_cnt:
|
|
|
+ share_cnt = float(share_cnt.split('万+')[0]) * 10000
|
|
|
+ else:
|
|
|
+ share_cnt = float(share_cnt)
|
|
|
+
|
|
|
+ # 收藏
|
|
|
+ favorite_id = driver.find_element(By.ID, 'com.tencent.mm:id/fnp')
|
|
|
+ favorite_cnt = favorite_id.get_attribute('name')
|
|
|
+ if favorite_cnt == "" or favorite_cnt == "收藏":
|
|
|
+ favorite_cnt = 0
|
|
|
+ elif '万' in favorite_cnt:
|
|
|
+ favorite_cnt = float(favorite_cnt.split('万')[0]) * 10000
|
|
|
+ elif '万+' in favorite_cnt:
|
|
|
+ favorite_cnt = float(favorite_cnt.split('万+')[0]) * 10000
|
|
|
+ else:
|
|
|
+ favorite_cnt = float(favorite_cnt)
|
|
|
+
|
|
|
+ # 评论
|
|
|
+ comment_id = driver.find_element(By.ID, 'com.tencent.mm:id/bje')
|
|
|
+ comment_cnt = comment_id.get_attribute('name')
|
|
|
+ if comment_cnt == "" or comment_cnt == "评论":
|
|
|
+ comment_cnt = 0
|
|
|
+ elif '万' in comment_cnt:
|
|
|
+ comment_cnt = float(comment_cnt.split('万')[0]) * 10000
|
|
|
+ elif '万+' in comment_cnt:
|
|
|
+ comment_cnt = float(comment_cnt.split('万+')[0]) * 10000
|
|
|
+ else:
|
|
|
+ comment_cnt = float(comment_cnt)
|
|
|
+
|
|
|
+ # 用户名
|
|
|
+ username_id = driver.find_element(By.ID, 'com.tencent.mm:id/hft')
|
|
|
+ user_name = username_id.get_attribute('name')
|
|
|
+
|
|
|
+ Common.logger(log_type).info('video_title:{}', video_title)
|
|
|
+ Common.logger(log_type).info('duration:{}', duration)
|
|
|
+ Common.logger(log_type).info('like_cnt:{}', like_cnt)
|
|
|
+ Common.logger(log_type).info('share_cnt:{}', share_cnt)
|
|
|
+ Common.logger(log_type).info('favorite_cnt:{}', favorite_cnt)
|
|
|
+ Common.logger(log_type).info('comment_cnt:{}', comment_cnt)
|
|
|
+ Common.logger(log_type).info('user_name:{}', user_name)
|
|
|
+
|
|
|
+ # 判断无效视频
|
|
|
+ if video_title == '' or user_name == '':
|
|
|
+ Common.logger(log_type).info('向上滑动页面')
|
|
|
+ driver.swipe(500, 1500, 500, 500, 300)
|
|
|
+ Common.logger(log_type).info('无效视频\n')
|
|
|
|
|
|
- Click.click_video(log_type, video_title)
|
|
|
- time.sleep(1)
|
|
|
+ # 判断下载规则
|
|
|
+ elif cls.download_rule(duration, like_cnt, share_cnt, favorite_cnt, comment_cnt) is False:
|
|
|
+ Common.logger(log_type).info('向上滑动页面')
|
|
|
+ driver.swipe(500, 1500, 500, 500, 300)
|
|
|
+ Common.logger(log_type).info('不满足抓取规则\n')
|
|
|
+
|
|
|
+ # 已下载表去重
|
|
|
+ elif str(video_title) in [x for y in Feishu.get_values_batch(log_type, 'shipinhao', 'c77cf9') for x in y]:
|
|
|
+ Common.logger(log_type).info('向上滑动页面')
|
|
|
+ driver.swipe(500, 1500, 500, 500, 300)
|
|
|
+ Common.logger(log_type).info('视频已下载\n')
|
|
|
+
|
|
|
+ # feeds 表去重
|
|
|
+ elif str(video_title) in [x for y in Feishu.get_values_batch(log_type, 'shipinhao', 'FSDlBy') for x in y]:
|
|
|
+ Common.logger(log_type).info('向上滑动页面')
|
|
|
+ driver.swipe(500, 1500, 500, 500, 300)
|
|
|
+ Common.logger(log_type).info('视频已存在\n')
|
|
|
|
|
|
- urls = GetUrl.get_url(log_type)
|
|
|
- if urls is None:
|
|
|
- Common.logger(log_type).info('未获取到视频 URL')
|
|
|
+ # 分享给 windows 爬虫机
|
|
|
else:
|
|
|
- video_url = urls[0]
|
|
|
- cover_url = urls[1]
|
|
|
-
|
|
|
- # 把视频信息写入飞书feeds文档
|
|
|
- Feishu.insert_columns(log_type, 'shipinhao', 'FSDlBy', 'ROWS', 1, 2)
|
|
|
- get_feeds_time = int(time.time())
|
|
|
- values = [[time.strftime('%Y/%m/%d %H:%M:%S', time.localtime(get_feeds_time)),
|
|
|
- '推荐榜',
|
|
|
- str(video_title),
|
|
|
- duration,
|
|
|
- like_cnt,
|
|
|
- share_cnt,
|
|
|
- favorite_cnt,
|
|
|
- comment_cnt,
|
|
|
- str(user_name),
|
|
|
- cover_url,
|
|
|
- video_url]]
|
|
|
+ share_id.click()
|
|
|
+ driver.find_element(By.XPATH, '//*[@text="转发给朋友"]').click()
|
|
|
+ driver.find_element(By.XPATH, '//*[@text="爬虫群"]').click()
|
|
|
+ driver.find_element(By.ID, 'com.tencent.mm:id/guw').click()
|
|
|
+
|
|
|
+ Click.click_video(log_type, video_title)
|
|
|
time.sleep(1)
|
|
|
- Feishu.update_values(log_type, 'shipinhao', 'FSDlBy', 'A2:Z2', values)
|
|
|
- Common.logger(log_type).info('向上滑动页面')
|
|
|
- driver.swipe(500, 1000, 500, 300, 300)
|
|
|
- Common.logger(log_type).info('视频信息写入飞书文档成功\n')
|
|
|
+
|
|
|
+ urls = GetUrl.get_url(log_type)
|
|
|
+ if urls is None:
|
|
|
+ Common.logger(log_type).info('未获取到视频 URL')
|
|
|
+ else:
|
|
|
+ video_url = urls[0]
|
|
|
+ cover_url = urls[1]
|
|
|
+
|
|
|
+ # 把视频信息写入飞书feeds文档
|
|
|
+ Feishu.insert_columns(log_type, 'shipinhao', 'FSDlBy', 'ROWS', 1, 2)
|
|
|
+ get_feeds_time = int(time.time())
|
|
|
+ values = [[time.strftime('%Y/%m/%d %H:%M:%S', time.localtime(get_feeds_time)),
|
|
|
+ '推荐榜',
|
|
|
+ str(video_title),
|
|
|
+ duration,
|
|
|
+ like_cnt,
|
|
|
+ share_cnt,
|
|
|
+ favorite_cnt,
|
|
|
+ comment_cnt,
|
|
|
+ str(user_name),
|
|
|
+ cover_url,
|
|
|
+ video_url]]
|
|
|
+ time.sleep(1)
|
|
|
+ Feishu.update_values(log_type, 'shipinhao', 'FSDlBy', 'A2:Z2', values)
|
|
|
+ Common.logger(log_type).info('向上滑动页面')
|
|
|
+ driver.swipe(500, 1500, 500, 500, 300)
|
|
|
+ Common.logger(log_type).info('视频信息写入飞书文档成功\n')
|
|
|
+ except Exception as e:
|
|
|
+ Common.logger(log_type).error('get_feeds异常', e)
|
|
|
|
|
|
|
|
|
if __name__ == '__main__':
|
