# -*- coding: utf-8 -*- # @Author: wangkun # @Time: 2022/12/14 import os import sys import time from appium import webdriver from appium.webdriver.extensions.android.nativekey import AndroidKey from selenium.common import NoSuchElementException from appium.webdriver.webdriver import WebDriver from selenium.webdriver.common.by import By sys.path.append(os.getcwd()) from main.common import Common from main.feishu_lib import Feishu from shipinhao.shipinhao_publish import Publish class Follow: # 过滤词库 @classmethod def filter_words(cls, log_type): try: filter_words_sheet = Feishu.get_values_batch(log_type, 'shipinhao', 'gmeOgJ') filter_words_list = [] for x in filter_words_sheet: for y in x: if y is None: pass else: filter_words_list.append(y) return filter_words_list except Exception as e: Common.logger(log_type).error('filter_words异常:{}\n', e) @classmethod def get_users_from_feishu(cls, log_type): try: users_sheet = Feishu.get_values_batch(log_type, 'shipinhao', 'yVFqxa') user_list = [] for i in range(1, len(users_sheet)): user_name = users_sheet[i][1] if user_name is not None: user_list.append(user_name) return user_list except Exception as e: Common.logger(log_type).error(f'get_users_from_feishu异常:{e}\n') @classmethod def start_follow_wechat(cls, log_type, user_name, env): # try: Common.logger(log_type).info('启动微信') caps = { "platformName": "Android", # 手机操作系统 Android / iOS "deviceName": "Android", # 连接的设备名(模拟器或真机),安卓可以随便写 "platforVersion": "11", # 手机对应的系统版本(Android 11) "appPackage": "com.tencent.mm", # 被测APP的包名,乐活圈 Android "appActivity": ".ui.LauncherUI", # 启动的Activity名 "autoGrantPermissions": "true", # 让 appium 自动授权 base 权限, # 如果 noReset 为 True,则该条不生效(该参数为 Android 独有),对应的值为 True 或 False "unicodekeyboard": True, # 使用自带输入法,输入中文时填True "resetkeyboard": True, # 执行完程序恢复原来输入法 "noReset": True, # 不重置APP "printPageSourceOnFailure": True, # 找不到元素时,appium log 会完整记录当前页面的 pagesource "newCommandTimeout": 6000, # 初始等待时间 "automationName": "UiAutomator2", # 使用引擎,默认为 Appium, # 其中 Appium、UiAutomator2、Selendroid、Espresso 用于 Android,XCUITest 用于 iOS "showChromedriverLog": True, # "chromeOptions": {"androidProcess": "com.tencent.mm:appbrand0"}, "chromeOptions": {"androidProcess": "com.tencent.mm:tools"}, 'enableWebviewDetailsCollection': True, 'setWebContentsDebuggingEnabled': True, # 'chromedriverExecutable': '/Users/wangkun/Downloads/chromedriver_v86/chromedriver', 'chromedriverExecutable': '/Users/lieyunye/Downloads/chromedriver_v86/chromedriver', } driver = webdriver.Remote("http://localhost:4723/wd/hub", caps) driver.implicitly_wait(10) cls.search_to_user_homepage(log_type, user_name, driver) cls.search_user_videos(log_type, driver, env) Common.logger(log_type).info('休眠 3s') time.sleep(3) cls.quit(log_type, driver) # except Exception as e: # Common.logger(log_type).error('start_follow_wechat异常:{}\n', e) @classmethod def quit(cls, log_type, driver: WebDriver): driver.quit() Common.logger(log_type).info('退出 APP 成功\n') @classmethod def search_element(cls, log_type, driver: WebDriver, element): try: windowHandles = driver.window_handles # 遍历所有的handles,找到当前页面所在的handle:如果pageSource有包含你想要的元素,就是所要找的handle # 小程序的页面来回切换也需要:遍历所有的handles,切换到元素所在的handle for handle in windowHandles: driver.switch_to.window(handle) time.sleep(3) if len(driver.find_elements(By.XPATH, element)) != 0: return driver.find_element(By.XPATH, element) else: pass except Exception as e: Common.logger(log_type).warning('search_element异常:{}\n', e) @classmethod def search_to_user_homepage(cls, log_type, user_name, driver: WebDriver): Common.logger(log_type).info('点击搜索按钮') driver.find_element(By.ID, 'com.tencent.mm:id/j5t').click() Common.logger(log_type).info(f'输入搜索词:{user_name}') driver.find_element(By.ID, 'com.tencent.mm:id/cd7').send_keys(user_name) driver.press_keycode(AndroidKey.ENTER) Common.logger(log_type).info('点击进入搜索结果页') driver.find_element(By.ID, 'com.tencent.mm:id/m94').click() Common.logger(log_type).info('切换到webview') webview = driver.contexts driver.switch_to.context(webview[1]) time.sleep(3) Common.logger(log_type).info('点击"视频号"分类') cls.search_element(log_type, driver, '//div[@class="unit"]/*[2]').click() time.sleep(3) Common.logger(log_type).info(f'进入用户主页:{user_name}') user_element = cls.search_element(log_type, driver, '//div[@class="video-account__container search_item_inner"]') if user_element is None: Common.logger(log_type).info(f'未搜索到用户:{user_name}\n') return else: user_element.click() time.sleep(1) Common.logger(log_type).info(f'进入 {user_name} 主页成功\n') @classmethod def search_user_videos(cls, log_type, driver: WebDriver, env): Common.logger(log_type).info('切回NATIVE_APP') driver.switch_to.context('NATIVE_APP') # 判断置顶视频 top_videos = driver.find_elements(By.ID, 'com.tencent.mm:id/i56') Common.logger(log_type).info(f'发现 {len(top_videos)} 个置顶视频\n') if len(top_videos) == 0: Common.logger(log_type).info('当前用户没有置顶视频') else: for i in range(len(top_videos)): top_videos[i].click() cls.download_publish(log_type, driver, env) driver.press_keycode(AndroidKey.BACK) # 判断非置顶视频 not_top_videos = driver.find_elements(By.ID, 'com.tencent.mm:id/nmz') Common.logger(log_type).info(f'发现 {len(not_top_videos)} 个非置顶视频') not_top_first_video = not_top_videos[len(top_videos)] not_top_first_video.click() while True: cls.download_publish(log_type, driver, env) driver.swipe(10, 1600, 10, 300, 200) if len(driver.find_elements(By.ID, 'com.tencent.mm:id/g2s')) > 0: Common.logger(log_type).info('到底啦 ~\n') return @classmethod def get_video_info(cls, log_type, driver: WebDriver): driver.implicitly_wait(10) # 视频标题 try: title_id = driver.find_element(By.ID, 'com.tencent.mm:id/ki5') video_title = title_id.get_attribute('name').split('\n')[0].strip() except NoSuchElementException: video_title = '' # 点击播放器,获取视频时长 # Common.logger(log_type).info('暂停播放') pause_btn = driver.find_element(By.ID, 'com.tencent.mm:id/eh4') pause_btn.click() start_time = driver.find_element(By.ID, 'com.tencent.mm:id/l59').get_attribute('name') start_time = int(start_time.split(':')[0]) * 60 + int(start_time.split(':')[-1]) try: end_time = driver.find_element(By.ID, 'com.tencent.mm:id/l7i').get_attribute('name') except NoSuchElementException: end_time = driver.find_element(By.ID, 'com.tencent.mm:id/g73').get_attribute('name') end_time = int(end_time.split(':')[0]) * 60 + int(end_time.split(':')[-1]) duration = start_time + end_time # 点赞 like_id = driver.find_element(By.ID, 'com.tencent.mm:id/k04') like_cnt = like_id.get_attribute('name') if like_cnt == "" or like_cnt == "喜欢": like_cnt = 0 elif '万' in like_cnt: like_cnt = float(like_cnt.split('万')[0]) * 10000 elif '万+' in like_cnt: like_cnt = float(like_cnt.split('万+')[0]) * 10000 else: like_cnt = float(like_cnt) # 分享 share_id = driver.find_element(By.ID, 'com.tencent.mm:id/jhv') share_cnt = share_id.get_attribute('name') if share_cnt == "" or share_cnt == "转发": share_cnt = 0 elif '万' in share_cnt: share_cnt = float(share_cnt.split('万')[0]) * 10000 elif '万+' in share_cnt: share_cnt = float(share_cnt.split('万+')[0]) * 10000 else: share_cnt = float(share_cnt) # 收藏 favorite_id = driver.find_element(By.ID, 'com.tencent.mm:id/fnp') favorite_cnt = favorite_id.get_attribute('name') if favorite_cnt == "" or favorite_cnt == "收藏": favorite_cnt = 0 elif '万' in favorite_cnt: favorite_cnt = float(favorite_cnt.split('万')[0]) * 10000 elif '万+' in favorite_cnt: favorite_cnt = float(favorite_cnt.split('万+')[0]) * 10000 else: favorite_cnt = float(favorite_cnt) # 评论 comment_id = driver.find_element(By.ID, 'com.tencent.mm:id/bje') comment_cnt = comment_id.get_attribute('name') if comment_cnt == "" or comment_cnt == "评论": comment_cnt = 0 elif '万' in comment_cnt: comment_cnt = float(comment_cnt.split('万')[0]) * 10000 elif '万+' in comment_cnt: comment_cnt = float(comment_cnt.split('万+')[0]) * 10000 else: comment_cnt = float(comment_cnt) # 用户名 username_id = driver.find_element(By.ID, 'com.tencent.mm:id/hft') user_name = username_id.get_attribute('name') Common.logger(log_type).info('video_title:{}', video_title) Common.logger(log_type).info('duration:{}', duration) Common.logger(log_type).info('like_cnt:{}', like_cnt) Common.logger(log_type).info('share_cnt:{}', share_cnt) Common.logger(log_type).info('favorite_cnt:{}', favorite_cnt) Common.logger(log_type).info('comment_cnt:{}', comment_cnt) Common.logger(log_type).info('user_name:{}', user_name) video_dict = { 'video_title': video_title, 'duration': duration, 'like_cnt': like_cnt, 'share_cnt': share_cnt, 'share_id': share_id, 'favorite_cnt': favorite_cnt, 'comment_cnt': comment_cnt, 'user_name': user_name } return video_dict @classmethod def download_publish(cls, log_type, driver: WebDriver, env): video_dict = cls.get_video_info(log_type, driver) Common.logger(log_type).info(f'video_dict:{video_dict}') Common.logger(log_type).info(f'video_title:{video_dict["video_title"]}') return if int(video_dict['duration']) < 50: Common.logger(log_type).info(f'时长:{int(video_dict["duration"])} < 50 秒\n') elif video_dict['video_title'] == '': Common.logger(log_type).info('视频标题为空\n') # 过滤词库(视频标题) elif any(word if word in video_dict['video_title'] else False for word in cls.filter_words(log_type)) is True: Common.logger(log_type).info(f'视频已中过滤词:{video_dict["video_title"]}\n') # 视频号推荐_已下载表 elif str(video_dict['video_title']) in [x for y in Feishu.get_values_batch(log_type, 'shipinhao', 'c77cf9') for x in y]: Common.logger(log_type).info('视频已下载\n') # 视频号定向_已下载表 elif str(video_dict['video_title']) in [x for y in Feishu.get_values_batch(log_type, 'shipinhao', 'XxmRlE') for x in y]: Common.logger(log_type).info('视频已下载\n') # feeds 表去重 elif str(video_dict['video_title']) in [x for y in Feishu.get_values_batch(log_type, 'shipinhao', 'FSDlBy') for x in y]: Common.logger(log_type).info('视频已存在\n') # feeds 表去重 elif str(video_dict['video_title']) in [x for y in Feishu.get_values_batch(log_type, 'shipinhao', 'qzDljJ') for x in y]: Common.logger(log_type).info('视频已存在\n') # 分享给 windows 爬虫机 else: video_dict['share_id'].click() driver.find_element(By.XPATH, '//*[@text="转发给朋友"]').click() driver.find_element(By.XPATH, '//*[@text="爬虫群"]').click() driver.find_element(By.ID, 'com.tencent.mm:id/guw').click() # 把视频信息写入飞书feeds文档 Feishu.insert_columns(log_type, 'shipinhao', 'qzDljJ', 'ROWS', 1, 2) get_feeds_time = int(time.time()) values = [[time.strftime('%Y/%m/%d %H:%M:%S', time.localtime(get_feeds_time)), '定向榜', str(video_dict['video_title']), int(video_dict['duration']), int(video_dict['like_cnt']), int(video_dict['share_cnt']), int(video_dict['favorite_cnt']), int(video_dict['comment_cnt']), str(video_dict['user_name'])]] time.sleep(1) Feishu.update_values(log_type, 'shipinhao', 'qzDljJ', 'A2:Z2', values) Common.logger(log_type).info('视频信息写入飞书文档成功\n') while True: if Feishu.get_values_batch(log_type, 'shipinhao', 'qzDljJ')[1][11] is None: Common.logger(log_type).info('等待更新 URL 信息') time.sleep(10) else: Common.logger(log_type).info('URL 信息已更新\n') break cls.publish(log_type, env) # 下载 、上传 @classmethod def publish(cls, log_type, env): try: follow_feeds_sheet = Feishu.get_values_batch(log_type, 'shipinhao', 'qzDljJ') for i in range(1, len(follow_feeds_sheet)): download_title = follow_feeds_sheet[i][2].strip().replace('"', '') \ .replace('“', '').replace('“', '…').replace("\n", "") \ .replace("/", "").replace("\r", "") \ .replace(".", "。").replace("\\", "").replace("&NBSP", "") \ .replace(":", "").replace("*", "").replace("?", "") \ .replace("?", "").replace('"', "").replace("<", "") \ .replace(">", "").replace("|", "").replace(" ", "") download_duration = follow_feeds_sheet[i][3] download_like_cnt = follow_feeds_sheet[i][4] download_share_cnt = follow_feeds_sheet[i][5] download_favorite_cnt = follow_feeds_sheet[i][6] download_comment_cnt = follow_feeds_sheet[i][7] download_username = follow_feeds_sheet[i][8] download_head_url = follow_feeds_sheet[i][9] download_cover_url = follow_feeds_sheet[i][10] download_video_url = follow_feeds_sheet[i][11] Common.logger(log_type).info("download_title:{}", download_title) Common.logger(log_type).info("download_username:{}", download_username) Common.logger(log_type).info("download_video_url:{}", download_video_url) if download_title is None or download_duration is None or download_video_url is None: Feishu.dimension_range(log_type, 'shipinhao', 'qzDljJ', 'ROWS', i + 1, i + 1) Common.logger(log_type).info('空行,删除成功\n') return else: # 下载封面 Common.download_method(log_type=log_type, text="cover", d_name=str(download_title), d_url=str(download_cover_url)) # 下载视频 Common.download_method(log_type=log_type, text="video", d_name=str(download_title), d_url=str(download_video_url)) # 保存视频信息至 "./videos/{download_video_title}/info.txt" with open("./videos/" + download_title + "/" + "info.txt", "a", encoding="UTF-8") as f_a: f_a.write('shipinhao' + str(int(time.time())) + "\n" + str(download_title) + "\n" + str(download_duration) + "\n" + str(download_favorite_cnt) + "\n" + str(download_comment_cnt) + "\n" + str(download_like_cnt) + "\n" + str(download_share_cnt) + "\n" + str(1920 * 1080) + "\n" + str(int(time.time())) + "\n" + str(download_username) + "\n" + str(download_head_url) + "\n" + str(download_video_url) + "\n" + str(download_cover_url) + "\n" + "SHIPINHAO" + str(int(time.time()))) Common.logger(log_type).info("==========视频信息已保存至info.txt==========") Common.logger(log_type).info("开始上传视频:{}".format(download_title)) our_video_id = Publish.upload_and_publish(log_type, env, "follow") if env == 'dev': our_video_link = "https://testadmin.piaoquantv.com/cms/post-detail/" + str( our_video_id) + "/info" else: our_video_link = "https://admin.piaoquantv.com/cms/post-detail/" + str( our_video_id) + "/info" Common.logger(log_type).info("视频上传完成:{}", our_video_link) # 视频ID工作表,插入首行 Feishu.insert_columns(log_type, "shipinhao", "XxmRlE", "ROWS", 1, 2) # 视频ID工作表,首行写入数据 upload_time = int(time.time()) values = [[time.strftime("%Y/%m/%d %H:%M:%S", time.localtime(upload_time)), "定向榜", str(download_title), our_video_link, download_duration, download_like_cnt, download_share_cnt, download_favorite_cnt, download_comment_cnt, download_username, str(download_head_url), str(download_cover_url), str(download_video_url)]] time.sleep(1) Feishu.update_values(log_type, "shipinhao", "XxmRlE", "F2:V2", values) # 删除行或列,可选 ROWS、COLUMNS time.sleep(1) Feishu.dimension_range(log_type, "shipinhao", "qzDljJ", "ROWS", i + 1, i + 1) Common.logger(log_type).info("下载/上传成功\n") # return except Exception as e: Feishu.dimension_range(log_type, "shipinhao", "qzDljJ", "ROWS", 2, 2) Common.logger(log_type).error('download_publish异常,删除视频信息成功:{}\n', e) @classmethod def search_to_all_user_homepage(cls, log_type, env): # try: user_list = cls.get_users_from_feishu(log_type) for user in user_list: cls.start_follow_wechat(log_type, user, env) Common.logger(log_type).info('所有用户已抓取完毕\n') # except Exception as e: # Common.logger(log_type).error(f'search_to_all_user_homepage异常:{e}\n') if __name__ == '__main__': # print(Follow.get_users_from_feishu('follow')) # print(len(Follow.get_users_from_feishu('follow'))) print(Follow.filter_words('follow')) # Follow.search_to_all_user_homepage('follow') pass