crawler_shipinhao/main/shipinhao_recommend.py

# @Author: wangkun
# @Time: 3月 02, 2022
import os
import sys
import time
from appium import webdriver
from appium.webdriver.webdriver import WebDriver
from selenium.common import NoSuchElementException
from selenium.webdriver.common.by import By
sys.path.append(os.getcwd())
from main.common import Common
from main.feishu_lib import Feishu


class Recommend:

    # 启动微信，并打开视频号
    @classmethod
    def start_wechat(cls, log_type):
        Common.logger(log_type).info('启动微信')
        caps = {
            "platformName": "Android",  # 手机操作系统 Android / iOS
            "deviceName": "Android",  # 连接的设备名（模拟器或真机），安卓可以随便写
            "platforVersion": "11",  # 手机对应的系统版本（Android 11）
            "appPackage": "com.tencent.mm",  # 被测APP的包名，乐活圈 Android
            "appActivity": ".ui.LauncherUI",  # 启动的Activity名
            "autoGrantPermissions": "true",  # 让 appium 自动授权 base 权限，
            # 如果 noReset 为 True，则该条不生效（该参数为 Android 独有），对应的值为 True 或 False
            "unicodekeyboard": True,  # 使用自带输入法，输入中文时填True
            "resetkeyboard": True,  # 执行完程序恢复原来输入法
            "noReset": True,  # 不重置APP
            "printPageSourceOnFailure": True,  # 找不到元素时，appium log 会完整记录当前页面的 pagesource
            "newCommandTimeout": 6000,  # 初始等待时间
            "automationName": "UiAutomator2"  # 使用引擎，默认为 Appium，
            # 其中 Appium、UiAutomator2、Selendroid、Espresso 用于 Android，XCUITest 用于 iOS
        }
        # global driver
        driver = webdriver.Remote("http://localhost:4723/wd/hub", caps)
        driver.implicitly_wait(10)

        Common.logger(log_type).info('点击发现TAB')
        driver.find_elements(By.ID, 'com.tencent.mm:id/f2s')[2].click()

        Common.logger(log_type).info('点击视频号\n')
        driver.find_elements(By.ID, 'com.tencent.mm:id/gv6')[1].click()
        time.sleep(5)

        cls.get_feeds(log_type, driver)

        Common.logger(log_type).info('休眠 3s')
        time.sleep(3)
        cls.quit(log_type, driver)

    # 退出 APP
    @classmethod
    def quit(cls, log_type, driver: WebDriver):
        driver.quit()
        Common.logger(log_type).info('退出 APP 成功\n')

    # 下载规则
    @staticmethod
    def download_rule(duration, like_cnt, share_cnt, favorite_cnt, comment_cnt):
        if int(duration) >= 10:
            if int(like_cnt) >= 0:
                if int(share_cnt) >= 0:
                    if int(favorite_cnt) >= 0:
                        if int(comment_cnt) >= 0:
                            return True
                        else:
                            return False
                    else:
                        return False
                else:
                    return False
            else:
                return False
        else:
            return False

    # 操作安卓手机，自己滑动首页视频，并获取视频信息
    @classmethod
    def get_feeds(cls, log_type, driver: WebDriver):
        driver.implicitly_wait(10)
        for i in range(10):
            # 关闭页面弹窗
            try:
                driver.find_element(By.XPATH, '//*[@text="我知道了"]')
                Common.logger(log_type).info('已关闭未成年模式弹窗')
            except NoSuchElementException:
                pass

            try:
                driver.find_element(By.ID, 'com.tencent.mm:id/dkf')
                Common.logger(log_type).info('直播视频，向上滑动页面\n')
                driver.swipe(500, 1000, 500, 300, 300)
            except NoSuchElementException:
                pass

            # Common.logger(log_type).info('暂停播放')
            pause_btn = driver.find_element(By.ID, 'com.tencent.mm:id/eh4')
            pause_btn.click()

            # 视频标题
            try:
                title_id = driver.find_element(By.ID, 'com.tencent.mm:id/ki5')
                video_title = title_id.get_attribute('name')
            except NoSuchElementException:
                video_title = ''

            # 点击播放器，获取视频时长
            start_time = driver.find_element(By.ID, 'com.tencent.mm:id/l59').get_attribute('name')
            start_time = int(start_time.split(':')[0])*60 + int(start_time.split(':')[-1])
            try:
                end_time = driver.find_element(By.ID, 'com.tencent.mm:id/l7i').get_attribute('name')
            except NoSuchElementException:
                end_time = driver.find_element(By.ID, 'com.tencent.mm:id/g73').get_attribute('name')
            end_time = int(end_time.split(':')[0]) * 60 + int(end_time.split(':')[-1])
            duration = start_time + end_time

            # 点赞
            like_id = driver.find_element(By.ID, 'com.tencent.mm:id/k04')
            like_cnt = like_id.get_attribute('name')
            if like_cnt == "" or like_cnt == "喜欢":
                like_cnt = 0
            elif '万' in like_cnt:
                like_cnt = float(like_cnt.split('万')[0]) * 10000
            elif '万+' in like_cnt:
                like_cnt = float(like_cnt.split('万+')[0]) * 10000
            else:
                like_cnt = float(like_cnt)

            # 分享
            share_id = driver.find_element(By.ID, 'com.tencent.mm:id/jhv')
            share_cnt = share_id.get_attribute('name')
            if share_cnt == "" or share_cnt == "转发":
                share_cnt = 0
            elif '万' in share_cnt:
                share_cnt = float(share_cnt.split('万')[0]) * 10000
            elif '万+' in share_cnt:
                share_cnt = float(share_cnt.split('万+')[0]) * 10000
            else:
                share_cnt = float(share_cnt)

            # 收藏
            favorite_id = driver.find_element(By.ID, 'com.tencent.mm:id/fnp')
            favorite_cnt = favorite_id.get_attribute('name')
            if favorite_cnt == "" or favorite_cnt == "收藏":
                favorite_cnt = 0
            elif '万' in favorite_cnt:
                favorite_cnt = float(favorite_cnt.split('万')[0]) * 10000
            elif '万+' in favorite_cnt:
                favorite_cnt = float(favorite_cnt.split('万+')[0]) * 10000
            else:
                favorite_cnt = float(favorite_cnt)

            # 评论
            comment_id = driver.find_element(By.ID, 'com.tencent.mm:id/bje')
            comment_cnt = comment_id.get_attribute('name')
            if comment_cnt == "" or comment_cnt == "评论":
                comment_cnt = 0
            elif '万' in comment_cnt:
                comment_cnt = float(comment_cnt.split('万')[0]) * 10000
            elif '万+' in comment_cnt:
                comment_cnt = float(comment_cnt.split('万+')[0]) * 10000
            else:
                comment_cnt = float(comment_cnt)

            # 用户名
            username_id = driver.find_element(By.ID, 'com.tencent.mm:id/hft')
            user_name = username_id.get_attribute('name')

            Common.logger(log_type).info('video_title:{}', video_title)
            Common.logger(log_type).info('duration:{}', duration)
            Common.logger(log_type).info('like_cnt:{}', like_cnt)
            Common.logger(log_type).info('share_cnt:{}', share_cnt)
            Common.logger(log_type).info('favorite_cnt:{}', favorite_cnt)
            Common.logger(log_type).info('comment_cnt:{}', comment_cnt)
            Common.logger(log_type).info('user_name:{}', user_name)

            # 判断无效视频
            if video_title == '' or user_name == '':
                Common.logger(log_type).info('向上滑动页面')
                driver.swipe(500, 1000, 500, 300, 300)
                Common.logger(log_type).info('无效视频\n')

            # 判断下载规则
            elif cls.download_rule(duration, like_cnt, share_cnt, favorite_cnt, comment_cnt) is False:
                Common.logger(log_type).info('向上滑动页面')
                driver.swipe(500, 1000, 500, 300, 300)
                Common.logger(log_type).info('不满足抓取规则\n')

            # 已下载表去重
            elif str(video_title) in [x for y in Feishu.get_values_batch(log_type, 'shipinhao', 'c77cf9') for x in y]:
                Common.logger(log_type).info('向上滑动页面')
                driver.swipe(500, 1000, 500, 300, 300)
                Common.logger(log_type).info('视频已下载\n')

            # feeds 表去重
            elif str(video_title) in [x for y in Feishu.get_values_batch(log_type, 'shipinhao', 'FSDlBy') for x in y]:
                Common.logger(log_type).info('向上滑动页面')
                driver.swipe(500, 1000, 500, 300, 300)
                Common.logger(log_type).info('视频已存在\n')

            # 分享给 windows 爬虫机
            else:
                share_id.click()
                driver.find_element(By.XPATH, '//*[@text="转发给朋友"]').click()
                driver.find_element(By.XPATH, '//*[@text="爬虫群"]').click()
                driver.find_element(By.ID, 'com.tencent.mm:id/guw').click()

                # 把视频信息写入飞书feeds文档
                Feishu.insert_columns(log_type, 'shipinhao', 'FSDlBy', 'ROWS', 1, 2)
                get_feeds_time = int(time.time())
                values = [[time.strftime('%Y/%m/%d %H:%M:%S', time.localtime(get_feeds_time)),
                           '推荐榜',
                           str(video_title),
                           duration,
                           like_cnt,
                           share_cnt,
                           favorite_cnt,
                           comment_cnt,
                           str(user_name)]]
                time.sleep(1)
                Feishu.update_values(log_type, 'shipinhao', 'FSDlBy', 'A2:Z2', values)
                Common.logger(log_type).info('向上滑动页面')
                driver.swipe(500, 1000, 500, 300, 300)
                Common.logger(log_type).info('视频信息写入飞书文档成功\n')


if __name__ == '__main__':
    Recommend.start_wechat('recommend')

    pass