crawler_shipinhao/shipinhao/shipinhao_recommend.py

# @Author: wangkun
# @Time: 3月 02, 2022
import datetime
import os
import sys
import time
from appium import webdriver
from appium.webdriver.webdriver import WebDriver
from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver.common.by import By

sys.path.append(os.getcwd())
from main.common import Common
from shipinhao.shipinhao_publish import Publish
from main.feishu_lib import Feishu


class Recommend:
    # 当日已下载数量
    download_cnt = []

    # 下载/上传指定数量的视频
    @classmethod
    def run_recommend(cls, log_type, env):
        try:
            while True:
                if len(cls.download_cnt) >= 100 or datetime.datetime.now().hour == 17:
                    Common.logger('recommend').info('已下载{}条视频\n', len(cls.download_cnt))
                    cls.download_cnt = []
                    return
                else:
                    cls.start_wechat(log_type, env)
        except Exception as e:
            Common.logger(log_type).error('run_recommend异常:{}\n', e)

    # 启动微信，并打开视频号
    @classmethod
    def start_wechat(cls, log_type, env):
        try:
            Common.logger(log_type).info('启动微信')
            caps = {
                "platformName": "Android",  # 手机操作系统 Android / iOS
                "deviceName": "Android",  # 连接的设备名（模拟器或真机），安卓可以随便写
                "platforVersion": "11",  # 手机对应的系统版本（Android 11）
                "appPackage": "com.tencent.mm",  # 被测APP的包名，乐活圈 Android
                "appActivity": ".ui.LauncherUI",  # 启动的Activity名
                "autoGrantPermissions": "true",  # 让 appium 自动授权 base 权限，
                # 如果 noReset 为 True，则该条不生效（该参数为 Android 独有），对应的值为 True 或 False
                "unicodekeyboard": True,  # 使用自带输入法，输入中文时填True
                "resetkeyboard": True,  # 执行完程序恢复原来输入法
                "noReset": True,  # 不重置APP
                "printPageSourceOnFailure": True,  # 找不到元素时，appium log 会完整记录当前页面的 pagesource
                "newCommandTimeout": 6000,  # 初始等待时间
                "automationName": "UiAutomator2"  # 使用引擎，默认为 Appium，
                # 其中 Appium、UiAutomator2、Selendroid、Espresso 用于 Android，XCUITest 用于 iOS
            }
            driver = webdriver.Remote("http://localhost:4723/wd/hub", caps)
            driver.implicitly_wait(10)

            Common.logger(log_type).info('点击发现TAB')
            driver.find_elements(By.ID, 'com.tencent.mm:id/f2s')[2].click()

            Common.logger(log_type).info('点击视频号\n')
            driver.find_elements(By.ID, 'com.tencent.mm:id/gv6')[1].click()
            time.sleep(5)

            cls.get_feeds(log_type, driver, env)

            Common.logger(log_type).info('休眠 3s')
            time.sleep(3)
            cls.quit(log_type, driver)

        except Exception as e:
            Common.logger(log_type).error('start_wechat异常:{}\n', e)

    # 退出 APP
    @classmethod
    def quit(cls, log_type, driver: WebDriver):
        driver.quit()
        Common.logger(log_type).info('退出 APP 成功\n')

    # 下载规则
    @staticmethod
    def download_rule(duration, like_cnt, share_cnt, favorite_cnt, comment_cnt):
        if int(duration) >= 30:
            if int(like_cnt) >= 1000:
                if int(share_cnt) >= 0:
                    if int(favorite_cnt) >= 0:
                        if int(comment_cnt) >= 0:
                            return True
                        else:
                            return False
                    else:
                        return False
                else:
                    return False
            else:
                return False
        else:
            return False

    # 操作安卓手机，自己滑动首页视频，并获取视频信息
    @classmethod
    def get_feeds(cls, log_type, driver: WebDriver, env):
        try:
            for i in range(5):
                driver.implicitly_wait(10)

                # 视频标题
                try:
                    title_id = driver.find_element(By.ID, 'com.tencent.mm:id/ki5')
                    video_title = title_id.get_attribute('name').split('\n')[0].replace('#', '').strip()
                except NoSuchElementException:
                    video_title = ''
                    driver.swipe(10, 1600, 10, 300, 200)

                # 点击播放器，获取视频时长
                # Common.logger(log_type).info('暂停播放')
                pause_btn = driver.find_element(By.ID, 'com.tencent.mm:id/eh4')
                pause_btn.click()
                start_time = driver.find_element(By.ID, 'com.tencent.mm:id/l59').get_attribute('name')
                start_time = int(start_time.split(':')[0]) * 60 + int(start_time.split(':')[-1])
                try:
                    end_time = driver.find_element(By.ID, 'com.tencent.mm:id/l7i').get_attribute('name')
                except NoSuchElementException:
                    end_time = driver.find_element(By.ID, 'com.tencent.mm:id/g73').get_attribute('name')
                end_time = int(end_time.split(':')[0]) * 60 + int(end_time.split(':')[-1])
                duration = start_time + end_time

                # 点赞
                like_id = driver.find_element(By.ID, 'com.tencent.mm:id/k04')
                like_cnt = like_id.get_attribute('name')
                if like_cnt == "" or like_cnt == "喜欢":
                    like_cnt = 0
                elif '万' in like_cnt:
                    like_cnt = float(like_cnt.split('万')[0]) * 10000
                elif '万+' in like_cnt:
                    like_cnt = float(like_cnt.split('万+')[0]) * 10000
                else:
                    like_cnt = float(like_cnt)

                # 分享
                share_id = driver.find_element(By.ID, 'com.tencent.mm:id/jhv')
                share_cnt = share_id.get_attribute('name')
                if share_cnt == "" or share_cnt == "转发":
                    share_cnt = 0
                elif '万' in share_cnt:
                    share_cnt = float(share_cnt.split('万')[0]) * 10000
                elif '万+' in share_cnt:
                    share_cnt = float(share_cnt.split('万+')[0]) * 10000
                else:
                    share_cnt = float(share_cnt)

                # 收藏
                favorite_id = driver.find_element(By.ID, 'com.tencent.mm:id/fnp')
                favorite_cnt = favorite_id.get_attribute('name')
                if favorite_cnt == "" or favorite_cnt == "收藏":
                    favorite_cnt = 0
                elif '万' in favorite_cnt:
                    favorite_cnt = float(favorite_cnt.split('万')[0]) * 10000
                elif '万+' in favorite_cnt:
                    favorite_cnt = float(favorite_cnt.split('万+')[0]) * 10000
                else:
                    favorite_cnt = float(favorite_cnt)

                # 评论
                comment_id = driver.find_element(By.ID, 'com.tencent.mm:id/bje')
                comment_cnt = comment_id.get_attribute('name')
                if comment_cnt == "" or comment_cnt == "评论":
                    comment_cnt = 0
                elif '万' in comment_cnt:
                    comment_cnt = float(comment_cnt.split('万')[0]) * 10000
                elif '万+' in comment_cnt:
                    comment_cnt = float(comment_cnt.split('万+')[0]) * 10000
                else:
                    comment_cnt = float(comment_cnt)

                # 用户名
                username_id = driver.find_element(By.ID, 'com.tencent.mm:id/hft')
                user_name = username_id.get_attribute('name')

                Common.logger(log_type).info('video_title:{}', video_title)
                Common.logger(log_type).info('duration:{}', duration)
                Common.logger(log_type).info('like_cnt:{}', like_cnt)
                Common.logger(log_type).info('share_cnt:{}', share_cnt)
                Common.logger(log_type).info('favorite_cnt:{}', favorite_cnt)
                Common.logger(log_type).info('comment_cnt:{}', comment_cnt)
                Common.logger(log_type).info('user_name:{}', user_name)

                # 判断无效视频
                if video_title == '' or user_name == '':
                    Common.logger(log_type).info('无效视频，滑动到下一个视频\n')
                    driver.swipe(10, 1600, 10, 300, 200)

                # 判断下载规则
                elif cls.download_rule(duration, like_cnt, share_cnt, favorite_cnt, comment_cnt) is False:
                    Common.logger(log_type).info('不满足抓取规则，滑动到下一个视频\n')
                    driver.swipe(10, 1600, 10, 300, 200)

                # 过滤词库(视频标题)
                elif any(word if word in video_title else False for word in cls.filter_words(log_type)) is True:
                    Common.logger(log_type).info('视频已中过滤词:{}\n', video_title)
                    driver.swipe(10, 1600, 10, 300, 200)

                # 黑名单(用户名)
                elif any(word if word in user_name else False for word in cls.username_blacklist(log_type)) is True:
                    Common.logger(log_type).info('用户名已中过滤词:{}\n', user_name)
                    driver.swipe(10, 1600, 10, 300, 200)

                # 连续下载判断
                elif user_name in Feishu.get_values_batch('recommend', 'shipinhao', 'c77cf9')[1][14] \
                        and user_name in Feishu.get_values_batch('recommend', 'shipinhao', 'c77cf9')[2][14]:
                    Common.logger(log_type).info('该用户已连续下载2条视频，滑动到下一个视频\n')
                    driver.swipe(10, 1600, 10, 300, 200)

                # 视频号推荐_已下载表
                elif str(video_title) in [x for y in Feishu.get_values_batch(log_type, 'shipinhao', 'c77cf9') for x in
                                          y]:
                    Common.logger(log_type).info('视频已下载，滑动到下一个视频\n')
                    driver.swipe(10, 1600, 10, 300, 200)
                # 新视推荐_已下载表
                elif str(video_title) in [x for y in Feishu.get_values_batch(log_type, 'shipinhao', 'WAG7Dq') for x in
                                          y]:
                    Common.logger(log_type).info('视频已下载，滑动到下一个视频\n')
                    driver.swipe(10, 1600, 10, 300, 200)
                # 新视热门_已下载表
                elif str(video_title) in [x for y in Feishu.get_values_batch(log_type, 'shipinhao', '0i4jmV') for x
                                          in y]:
                    Common.logger(log_type).info('视频已下载，滑动到下一个视频\n')
                    driver.swipe(10, 1600, 10, 300, 200)

                # feeds 表去重
                elif str(video_title) in [x for y in Feishu.get_values_batch(log_type, 'shipinhao', 'FSDlBy') for x in
                                          y]:
                    Common.logger(log_type).info('视频已存在，滑动到下一个视频\n')
                    driver.swipe(10, 1600, 10, 300, 200)

                # 分享给 windows 爬虫机
                else:
                    share_id.click()
                    driver.find_element(By.XPATH, '//*[@text="转发给朋友"]').click()
                    driver.find_element(By.XPATH, '//*[@text="爬虫群"]').click()
                    driver.find_element(By.ID, 'com.tencent.mm:id/guw').click()

                    # 把视频信息写入飞书feeds文档
                    Feishu.insert_columns(log_type, 'shipinhao', 'FSDlBy', 'ROWS', 1, 2)
                    get_feeds_time = int(time.time())
                    values = [[time.strftime('%Y/%m/%d %H:%M:%S', time.localtime(get_feeds_time)),
                               '推荐榜',
                               str(video_title),
                               duration,
                               like_cnt,
                               share_cnt,
                               favorite_cnt,
                               comment_cnt,
                               str(user_name)]]
                    time.sleep(1)
                    Feishu.update_values(log_type, 'shipinhao', 'FSDlBy', 'A2:Z2', values)
                    Common.logger(log_type).info('视频信息写入飞书文档成功\n')

                    while True:
                        if Feishu.get_values_batch(log_type, 'shipinhao', 'FSDlBy')[1][11] is None:
                            Common.logger(log_type).info('等待更新 URL 信息')
                            time.sleep(10)
                        else:
                            Common.logger(log_type).info('URL 信息已更新\n')
                            break

                    cls.download_publish(log_type, env)
                    driver.swipe(10, 1600, 10, 300, 200)

        except Exception as e:
            Common.logger(log_type).error('get_feeds异常，滑动到下一个视频\n', e)
            Feishu.dimension_range(log_type, "shipinhao", "FSDlBy", "ROWS", 2, 2)
            driver.swipe(10, 1600, 10, 300, 200)

    # 下载 、上传
    @classmethod
    def download_publish(cls, log_type, env):
        try:
            recommend_feeds_sheet = Feishu.get_values_batch(log_type, 'shipinhao', 'FSDlBy')
            for i in range(1, len(recommend_feeds_sheet)):
                download_title = recommend_feeds_sheet[i][2].strip().replace('"', '') \
                    .replace('“', '').replace('“', '…').replace("\n", "") \
                    .replace("/", "").replace("\r", "").replace("#", "") \
                    .replace(".", "。").replace("\\", "").replace("&NBSP", "") \
                    .replace(":", "").replace("*", "").replace("？", "") \
                    .replace("?", "").replace('"', "").replace("<", "") \
                    .replace(">", "").replace("|", "").replace(" ", "")
                download_duration = recommend_feeds_sheet[i][3]
                download_like_cnt = recommend_feeds_sheet[i][4]
                download_share_cnt = recommend_feeds_sheet[i][5]
                download_favorite_cnt = recommend_feeds_sheet[i][6]
                download_comment_cnt = recommend_feeds_sheet[i][7]
                download_username = recommend_feeds_sheet[i][8]
                download_head_url = recommend_feeds_sheet[i][9]
                download_cover_url = recommend_feeds_sheet[i][10]
                download_video_url = recommend_feeds_sheet[i][11]

                Common.logger(log_type).info("download_title:{}", download_title)
                Common.logger(log_type).info("download_username:{}", download_username)
                Common.logger(log_type).info("download_video_url:{}", download_video_url)

                if download_title is None or download_duration is None or download_video_url is None:
                    Feishu.dimension_range(log_type, 'shipinhao', 'FSDlBy', 'ROWS', i + 1, i + 1)
                    Common.logger(log_type).info('空行，删除成功\n')
                    return
                elif str(download_title) in [x for y in Feishu.get_values_batch(log_type, 'shipinhao', 'c77cf9') for x
                                             in y]:
                    Feishu.dimension_range(log_type, 'shipinhao', 'FSDlBy', 'ROWS', i + 1, i + 1)
                    Common.logger(log_type).info('视频已下载，删除成功\n')
                    return
                else:
                    # 下载封面
                    Common.download_method(log_type=log_type, text="cover",
                                           d_name=str(download_title), d_url=str(download_cover_url))
                    # 下载视频
                    Common.download_method(log_type=log_type, text="video",
                                           d_name=str(download_title), d_url=str(download_video_url))
                    # 保存视频信息至 "./videos/{download_video_title}/info.txt"
                    with open("./videos/" + download_title
                              + "/" + "info.txt", "a", encoding="UTF-8") as f_a:
                        f_a.write('shipinhao' + str(int(time.time())) + "\n" +
                                  str(download_title) + "\n" +
                                  str(download_duration) + "\n" +
                                  str(download_favorite_cnt) + "\n" +
                                  str(download_comment_cnt) + "\n" +
                                  str(download_like_cnt) + "\n" +
                                  str(download_share_cnt) + "\n" +
                                  str(1920 * 1080) + "\n" +
                                  str(int(time.time())) + "\n" +
                                  str(download_username) + "\n" +
                                  str(download_head_url) + "\n" +
                                  str(download_video_url) + "\n" +
                                  str(download_cover_url) + "\n" +
                                  "shipinhao")
                    Common.logger(log_type).info("==========视频信息已保存至info.txt==========")

                    Common.logger(log_type).info("开始上传视频:{}".format(download_title))
                    our_video_id = Publish.upload_and_publish(log_type, env, "play")
                    if env == 'dev':
                        our_video_link = "https://testadmin.piaoquantv.com/cms/post-detail/" + str(
                            our_video_id) + "/info"
                    else:
                        our_video_link = "https://admin.piaoquantv.com/cms/post-detail/" + str(our_video_id) + "/info"
                    Common.logger(log_type).info("视频上传完成:{}", our_video_link)

                    # 视频ID工作表，插入首行
                    Feishu.insert_columns(log_type, "shipinhao", "c77cf9", "ROWS", 1, 2)
                    # 视频ID工作表，首行写入数据
                    upload_time = int(time.time())
                    values = [[time.strftime("%Y/%m/%d %H:%M:%S", time.localtime(upload_time)),
                               "推荐榜",
                               str(download_title),
                               our_video_link,
                               download_duration,
                               download_like_cnt,
                               download_share_cnt,
                               download_favorite_cnt,
                               download_comment_cnt,
                               download_username,
                               str(download_head_url),
                               str(download_cover_url),
                               str(download_video_url)]]
                    time.sleep(1)
                    Feishu.update_values(log_type, "shipinhao", "c77cf9", "F2:V2", values)

                    cls.download_cnt.append(download_title)

                    # 删除行或列，可选 ROWS、COLUMNS
                    time.sleep(1)
                    Feishu.dimension_range(log_type, "shipinhao", "FSDlBy", "ROWS", i + 1, i + 1)
                    Common.logger(log_type).info("下载/上传成功，滑动到下一个视频")
                    return
        except Exception as e:
            Feishu.dimension_range(log_type, "shipinhao", "FSDlBy", "ROWS", 2, 2)
            Common.logger(log_type).error('download_publish异常，删除视频信息成功:{}\n', e)

    # 过滤词库
    @classmethod
    def filter_words(cls, log_type):
        try:
            filter_words_sheet = Feishu.get_values_batch(log_type, 'shipinhao', 'PYennl')
            filter_words_list = []
            for x in filter_words_sheet:
                for y in x:
                    if y is None:
                        pass
                    else:
                        filter_words_list.append(y)
            return filter_words_list
        except Exception as e:
            Common.logger(log_type).error('filter_words异常:{}\n', e)

    # 用户名黑名单
    @classmethod
    def username_blacklist(cls, log_type):
        try:
            username_blacklist_sheet = Feishu.get_values_batch(log_type, 'shipinhao', 'nkKrdl')
            username_blacklist_list = []
            for x in username_blacklist_sheet:
                for y in x:
                    if y is None:
                        pass
                    else:
                        username_blacklist_list.append(y)
            return username_blacklist_list
        except Exception as e:
            Common.logger(log_type).error('filter_words异常:{}\n', e)


if __name__ == '__main__':
    # print(Recommend.username_blacklist('recommend'))
    Recommend.run_recommend('recommend', 'prod')

    pass