piaoquan_crawler/ganggangdouchuan/ganggangdouchuan_recommend/ganggangdouchuan_recommend.py

# -*- coding: utf-8 -*-
# @Author: wangkun
# @Time: 2023/4/13
import difflib
import os
import sys
import time
import ffmpeg
from appium import webdriver
from appium.webdriver.extensions.android.nativekey import AndroidKey
from appium.webdriver.webdriver import WebDriver
from selenium.common import NoSuchElementException
from selenium.webdriver.common.by import By
sys.path.append(os.getcwd())
from common.common import Common


class GanggangdouchuanRecommend:
    i = 0

    @classmethod
    def start_wechat(cls, log_type, crawler, oss_endpoint, env):
        # try:
        if env == "dev":
            chromedriverExecutable = '/Users/wangkun/Downloads/chromedriver/chromedriver_v107/chromedriver'
        else:
            chromedriverExecutable = '/Users/piaoquan/Downloads/chromedriver'

        Common.logger(log_type, crawler).info('启动微信')
        caps = {
            "platformName": "Android",  # 手机操作系统 Android / iOS
            "deviceName": "a0a65126",  # 连接的设备名（模拟器或真机），安卓可以随便写
            "platforVersion": "11",  # 手机对应的系统版本（Android 11）
            "appPackage": "com.tencent.mm",  # 被测APP的包名，乐活圈 Android
            "appActivity": ".ui.LauncherUI",  # 启动的Activity名
            "autoGrantPermissions": "true",  # 让 appium 自动授权 base 权限，
            # 如果 noReset 为 True，则该条不生效（该参数为 Android 独有），对应的值为 True 或 False
            "unicodekeyboard": True,  # 使用自带输入法，输入中文时填True
            "resetkeyboard": True,  # 执行完程序恢复原来输入法
            "noReset": True,  # 不重置APP
            "printPageSourceOnFailure": True,  # 找不到元素时，appium log 会完整记录当前页面的 pagesource
            "newCommandTimeout": 6000,  # 初始等待时间
            "automationName": "UiAutomator2",  # 使用引擎，默认为 Appium，
            # 其中 Appium、UiAutomator2、Selendroid、Espresso 用于 Android，XCUITest 用于 iOS
            "showChromedriverLog": True,
            'enableWebviewDetailsCollection': True,
            'setWebContentsDebuggingEnabled': True,
            'recreateChromeDriverSessions': True,
            'chromedriverExecutable': chromedriverExecutable,
            "chromeOptions": {"androidProcess": "com.tencent.mm:appbrand0"},
            'browserName': ''
        }
        driver = webdriver.Remote("http://localhost:4723/wd/hub", caps)
        driver.implicitly_wait(20)
        # 向下滑动页面，展示出小程序选择面板
        for i in range(120):
            try:
                # 发现微信消息 TAB，代表微信已启动成功
                if driver.find_elements(By.ID, 'com.tencent.mm:id/f2s'):
                    break
                # 发现并关闭系统菜单栏
                elif driver.find_element(By.ID, 'com.android.systemui:id/dismiss_view'):
                    Common.logger(log_type, crawler).info('发现并关闭系统下拉菜单栏')
                    driver.find_element(By.ID, 'com.android.systemui:id/dismiss_view').click()
                else:
                    pass
            except NoSuchElementException:
                time.sleep(1)
        Common.logger(log_type, crawler).info('下滑，展示小程序选择面板')
        size = driver.get_window_size()
        driver.swipe(int(size['width'] * 0.5), int(size['height'] * 0.2),
                     int(size['width'] * 0.5), int(size['height'] * 0.8), 200)
        # 打开小程序"刚刚都传"
        time.sleep(3)
        Common.logger(log_type, crawler).info('打开小程序"刚刚都传"')
        driver.find_elements(By.XPATH, '//*[@text="刚刚都传"]')[-1].click()
        cls.get_videoList(log_type, crawler, oss_endpoint, env, driver)
        driver.quit()
        Common.logger(log_type, crawler).info('退出微信成功\n')
        # except Exception as e:
        #     Common.logger(log_type, crawler).error(f'start_wechat异常:{e}\n')
        #     cmd = "cd ~ && source .bash_profile && adb kill-server && adb start-server"
        #     os.system(cmd)

    @classmethod
    def search_elements(cls, log_type, crawler, driver: WebDriver, element):
        try:
            windowHandles = driver.window_handles
            for handle in windowHandles:
                driver.switch_to.window(handle)
                time.sleep(1)
                if len(driver.find_elements(By.XPATH, element)) != 0:
                    return driver.find_elements(By.XPATH, element)
                else:
                    return None
        except Exception as e:
            Common.logger(log_type, crawler).error(f'search_element异常:{e}\n')

    @classmethod
    def get_video_url(cls, log_type, crawler, driver: WebDriver, video_element):
        try:
            time.sleep(1)
            # Common.logger(log_type, crawler).info('进入视频详情')
            video_element.click()
            time.sleep(3)
            video_url_element = cls.search_elements(log_type, crawler, driver, '//wx-video[@id="v_id"]')
            if video_url_element is None or len(video_url_element) == 0:
                Common.logger(log_type, crawler).info('未获取到视频 URL')
                return 0
            else:
                return video_url_element[0].get_attribute('src')
        except Exception as e:
            Common.logger(log_type, crawler).error(f'get_video_url异常:{e}\n')

    @classmethod
    def get_videoList(cls, log_type, crawler, oss_endpoint, env, driver: WebDriver):
        # try:
        driver.implicitly_wait(15)
        Common.logger(log_type, crawler).info('切换到小程序\n')
        time.sleep(5)
        webviews = driver.contexts
        driver.switch_to.context(webviews[1])

        time.sleep(1)
        cls.search_elements(log_type, crawler, driver, '//wx-view[text()="视频"]')[0].click()

        time.sleep(1)
        index = 0

        while True:
            if cls.search_elements(log_type, crawler, driver, '//wx-view[@class="lists"]') is None:
                Common.logger(log_type, crawler).info('窗口已销毁\n')
                return

            Common.logger(log_type, crawler).info('获取视频列表\n')
            video_elements = cls.search_elements(log_type, crawler, driver, '//wx-view[@class="list"]')
            if video_elements is None:
                Common.logger(log_type, crawler).warning(f'video_elements:{video_elements}')
                return

            video_element_temp = video_elements[index:]
            if len(video_element_temp) == 0:
                Common.logger(log_type, crawler).info('到底啦~~~~~~~~~~~~~\n')
                return

            for i, video_element in enumerate(video_element_temp):
                if video_element is None:
                    Common.logger(log_type, crawler).info('到底啦~\n')
                    return
                cls.i += 1
                cls.search_elements(log_type, crawler, driver, '//wx-view[@class="list"]')
                Common.logger(log_type, crawler).info(f'拖动"视频"列表第{cls.i}个至屏幕中间')
                time.sleep(3)
                driver.execute_script("arguments[0].scrollIntoView({block:'center',inline:'center'})", video_element)

                video_title = video_element.find_elements(By.XPATH, '//wx-view[@class="title"]//span[2]')[cls.i - 1].get_attribute('innerHTML')
                cover_url = video_element.find_elements(By.XPATH, '//wx-image[@class="poster-img"]')[cls.i - 1].get_attribute('src')

                Common.logger(log_type, crawler).info(f'video_title:{video_title}')
                Common.logger(log_type, crawler).info(f'cover_url:{cover_url}')

                # cls.download_publish(log_type, crawler, oss_endpoint, env, job, driver, video_element, video_title, cover_url)
                # time.sleep(3)

            Common.logger(log_type, crawler).info('已抓取完一组视频，休眠10秒\n')
            time.sleep(10)
            index = index + len(video_element_temp)

        # except Exception as e:
        #     Common.logger(log_type, crawler).error(f'get_recommend异常，重启APP:{e}\n')
        #     cls.i = 0
        #     cls.quit(log_type, driver)
        #     cls.start_wechat(log_type, crawler, oss_endpoint, env, job)

    # @classmethod
    # def title_like(cls, log_type, title):
    #     sheet = Feishu.get_values_batch(log_type, 'ggdc', '070a67')
    #     for i in range(1, len(sheet)):
    #         video_title = sheet[i][7]
    #         if video_title is None:
    #             pass
    #         elif difflib.SequenceMatcher(None, title, video_title).quick_ratio() >= 0.8:
    #             return True
    #         else:
    #             pass
    #
    # @classmethod
    # def download_publish(cls, log_type, crawler, oss_endpoint, env, job, driver: WebDriver, video_element, video_title, cover_url):
    #     try:
    #         if video_title == 0 or cover_url == 0:
    #             Common.logger(log_type, crawler).info('无效视频\n')
    #         elif video_title in [x for y in Feishu.get_values_batch(log_type, 'ggdc', '070a67') for x in y]:
    #             Common.logger(log_type, crawler).info('视频已下载\n')
    #         elif any(word if word in video_title else False for word in cls.filter_words(log_type)) is True:
    #             Common.logger(log_type, crawler).info('已中过滤词\n')
    #         else:
    #             video_url = cls.get_video_url(log_type, driver, video_element)
    #             if video_url == 0:
    #                 Common.logger(log_type, crawler).info('video_url:未获取到\n')
    #             elif video_url in [x for y in Feishu.get_values_batch(log_type, 'ggdc', '070a67') for x in y]:
    #                 Common.logger(log_type, crawler).info('视频已下载\n')
    #             else:
    #                 Common.logger(log_type, crawler).info(f'cover_url:{cover_url}')
    #                 Common.logger(log_type, crawler).info(f'video_url:{video_url}')
    #
    #                 # 下载视频
    #                 Common.download_method(log_type, 'video', video_title, video_url)
    #                 # # 获取视频时长
    #                 # video_info = cls.get_video_info_from_local(log_type, "./videos/" + video_title + "/video.mp4")
    #                 # video_width = str(video_info[0])
    #                 # video_height = str(video_info[1])
    #                 # duration = video_info[2]
    #                 # 下载封面
    #                 Common.download_method(log_type, 'cover', video_title, cover_url)
    #                 # 保存视频信息至 "./videos/{download_video_title}/info.txt"
    #                 with open("./videos/" + video_title
    #                           + "/" + "info.txt", "a", encoding="UTF-8") as f_a:
    #                     f_a.write("ggdc" + str(int(time.time())) + "\n" +
    #                               str(video_title) + "\n" +
    #                               '100' + "\n" +
    #                               '100000' + "\n" +
    #                               '100000' + "\n" +
    #                               '100000' + "\n" +
    #                               '100000' + "\n" +
    #                               '1920*1080' + "\n" +
    #                               str(int(time.time())) + "\n" +
    #                               '刚刚都传小程序' + "\n" +
    #                               str(cover_url) + "\n" +
    #                               str(video_url) + "\n" +
    #                               str(cover_url) + "\n" +
    #                               "ganggangdouchuan" + str(int(time.time())))
    #                 Common.logger(log_type, crawler).info("==========视频信息已保存至info.txt==========")
    #
    #                 # 上传视频
    #                 Common.logger(log_type, crawler).info(f"开始上传视频:{video_title}")
    #                 if env == 'dev':
    #                     our_video_id = Publish.upload_and_publish(log_type, crawler, oss_endpoint, env, job)
    #                     our_video_link = "https://testadmin.piaoquantv.com/cms/post-detail/" + str(our_video_id) + "/info"
    #                 else:
    #                     our_video_id = Publish.upload_and_publish(log_type, crawler, oss_endpoint, env, job)
    #                     our_video_link = "https://admin.piaoquantv.com/cms/post-detail/" + str(our_video_id) + "/info"
    #                 Common.logger(log_type, crawler).info("视频上传完成")
    #
    #                 # 视频信息保存至飞书
    #                 Feishu.insert_columns(log_type, "ggdc", "070a67", "ROWS", 1, 2)
    #                 # 视频ID工作表，首行写入数据
    #                 upload_time = int(time.time())
    #                 values = [[time.strftime("%Y/%m/%d %H:%M:%S", time.localtime(upload_time)),
    #                            "推荐榜",
    #                            video_title,
    #                            our_video_link,
    #                            cover_url,
    #                            video_url]]
    #                 time.sleep(1)
    #                 Feishu.update_values(log_type, "ggdc", "070a67", "F2:V2", values)
    #                 driver.press_keycode(AndroidKey.BACK)
    #                 Common.logger(log_type, crawler).info(f"视频:{video_title}，下载/上传成功\n")
    #     except Exception as e:
    #         Common.logger(log_type, crawler).error(f'download_publish异常:{e}\n')


if __name__ == '__main__':
    GanggangdouchuanRecommend.start_wechat('recommend', 'ganggangdouchuan', 'out', 'dev')

    pass