crawler_haokan/main/haokan_play.py

# -*- coding: utf-8 -*-
# @Author: wangkun
# @Time: 2022/11/23
import os
import sys
import time

import requests

from main.haokan_publish import Publish

sys.path.append(os.getcwd())
from main.common import Common
from main.feishu_lib import Feishu


class Play:
    @classmethod
    def get_tab_from_feishu(cls, log_type):
        try:
            user_sheet = Feishu.get_values_batch(log_type, 'haokan', 'TaQXk3')
            user_dict = {}
            for i in range(1, len(user_sheet)):
                user_name = user_sheet[i][0]
                out_id = user_sheet[i][1]
                our_id = user_sheet[i][3]
                if user_name is None or out_id is None or our_id is None or i == 13:
                    pass
                else:
                    user_dict[user_name] = str(out_id) + ',' + str(our_id)
            return user_dict
        except Exception as e:
            Common.logger(log_type).error(f'get_tab_from_feishu异常:{e}\n')

    @classmethod
    def get_play_feeds(cls, log_type, tab):
        url = "https://haokan.baidu.com/web/video/feed?"
        params = {
            'tab': str(tab),
            'act': 'pcFeed',
            'pd': 'pc',
            'num': '20',
            'shuaxin_id': '1669367546078',
        }
        headers = {
            'Accept': '*/*',
            'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
            'Cache-Control': 'no-cache',
            'Connection': 'keep-alive',
            'Content-Type': 'application/x-www-form-urlencoded',
            'Cookie': 'BIDUPSID=0C817797C726E2312710D870ECDAE8A2; BDORZ=B490B5EBF6F3CD402E515D22BCDA1598; PSTM=1669001132; BAIDUID=AB7069CAF9ECB7AA43E400D164119733:FG=1; BDSFRCVID=H54OJexroG0GmHrjfnewtKf1EeKK0gOTDYLEJs2qYShnrsPVJeC6EG0PtoWQkz--EHtdogKK0mOTHv8F_2uxOjjg8UtVJeC6EG0Ptf8g0M5; H_BDCLCKID_SF=tR333R7oKRu_HRjYbb__-P4DePAttURZ56bHWh0M3b61qRcIh4ob5MPEDto-BMPj52OnKUT13lc5h4jX0P7_KRtr346-35543bRTLn76LRv0Kj6HybOfhP-UyN3LWh37bJblMKoaMp78jR093JO4y4Ldj4oxJpOJ5JbMopCafJOKHICGDTA-jMK; Hm_lvt_4aadd610dfd2f5972f1efee2653a2bc5=1669029805; PC_TAB_LOG=video_details_page; COMMON_LID=88bc9b0fbce964fbb6a76cfd7927d02b; hkpcvideolandquery=%u4E2D%u56FD%u7537%u513F%u5218%u5F3A%u6012%u70E7%u9756%u56FD%u795E%u793E%uFF0C%u518D%u70E7%u65E5%u672C%u9A7B%u97E9%u5927%u4F7F%u9986%uFF0C%u540E%u6765%u600E%u4E48%u6837%u4E86%uFF1F; BDSFRCVID_BFESS=H54OJexroG0GmHrjfnewtKf1EeKK0gOTDYLEJs2qYShnrsPVJeC6EG0PtoWQkz--EHtdogKK0mOTHv8F_2uxOjjg8UtVJeC6EG0Ptf8g0M5; H_BDCLCKID_SF_BFESS=tR333R7oKRu_HRjYbb__-P4DePAttURZ56bHWh0M3b61qRcIh4ob5MPEDto-BMPj52OnKUT13lc5h4jX0P7_KRtr346-35543bRTLn76LRv0Kj6HybOfhP-UyN3LWh37bJblMKoaMp78jR093JO4y4Ldj4oxJpOJ5JbMopCafJOKHICGDTA-jMK; H_PS_PSSID=36557_37769_34813_37778_37728_36806_37719_37743_26350_37787; delPer=0; BAIDUID_BFESS=AB7069CAF9ECB7AA43E400D164119733:FG=1; ZFY=e8fgayH:A:A8S2QAjzqo6RRIm7t3wLMhAdtluiJeCoSc4:C; PSINO=1; Hm_lpvt_4aadd610dfd2f5972f1efee2653a2bc5=1669367546; RT="z=1&dm=baidu.com&si=g2wkoutg3le&ss=lawabqi4&sl=1&tt=3v0&bcn=https%3A%2F%2Ffclog.baidu.com%2Flog%2Fweirwood%3Ftype%3Dperf"; ab_sr=1.0.1_ZDY1ZjNmZjA2MGVmZmFkNTMwN2Q5YTg4MTk5NTYwNGZlOWZhMDZmYTcwMDBhNWRlOWRjNWQwMjAzMzNhMjgyZWFhNTc0M2FmYjJmZTZkMzY4YTY0OTQ2MTUyYWM3MjBlZDFhODZkOGYzZTY3Zjk0YzQyMzE3NmQyMWM3YzZhODE0NTgzYmZhMzIzY2Y4NjQ3OTI0ZmYyYTY4OGM3ZDdjYw==; reptileData=%7B%22data%22%3A%22af081fa72566c0ffbb0f31576852da2f87dd6b701dfaf0db89f406ffbbf182dfb0f1ed7fbeee57d1bc0c40ee59eff348b237e865747fb09dd44b47bd4a3e0521c68ed4d71e17474b30aa210e7f59e7d9c63d673a4986ac3e00e398689a99cc8a%22%2C%22key_id%22%3A%2230%22%2C%22sign%22%3A%22dd49b147%22%7D; ariaDefaultTheme=undefined; PC_TAB_LOG=video_details_page',
            'Pragma': 'no-cache',
            'Referer': 'https://haokan.baidu.com/tab/recommend',
            'Sec-Fetch-Dest': 'empty',
            'Sec-Fetch-Mode': 'cors',
            'Sec-Fetch-Site': 'same-origin',
            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36 Edg/107.0.1418.52',
            'sec-ch-ua': '"Microsoft Edge";v="107", "Chromium";v="107", "Not=A?Brand";v="24"',
            'sec-ch-ua-mobile': '?0',
            'sec-ch-ua-platform': '"macOS"'
        }
        r = requests.get(url=url, headers=headers, params=params)
        if r.json()['errno'] != 0 or r.json()['errmsg'] != '成功':
            Common.logger(log_type).error(f'feeds_response:{r.text}\n')
        elif len(r.json()['data']['response']['videos']) == 0:
            Common.logger(log_type).warning(f'feeds_response:{r.text}\n')
        else:
            feeds = r.json()['data']['response']['videos']
            return feeds

    @classmethod
    def get_play_videos(cls, log_type, tab):
        feeds = cls.get_play_feeds(log_type, tab)
        for i in range(len(feeds)):
            # video_title
            if 'title' not in feeds[i]:
                video_title = 0
            else:
                video_title = feeds[i]['title']

            # video_id
            if 'id' not in feeds[i]:
                video_id = 0
            else:
                video_id = feeds[i]['id']

            # play_cnt
            if 'playcnt' not in feeds[i]:
                play_cnt = 0
            else:
                play_cnt = feeds[i]['playcnt']

            # duration
            if 'duration' not in feeds[i]:
                duration = 0
            else:
                duration = int(feeds[i]['duration'].split(':')[0])*60 + int(feeds[i]['duration'].split(':')[-1])

            # publish_time
            if 'publish_time' not in feeds[i]:
                publish_time = 0
            else:
                publish_time = feeds[i]['publish_time']

            # user_name
            if 'source_name' not in feeds[i]:
                user_name = 0
            else:
                user_name = feeds[i]['source_name']

            # head_url
            if 'author_avatar' not in feeds[i]:
                head_url = 0
            else:
                head_url = feeds[i]['author_avatar']

            # cover_url
            if 'poster_big' in feeds[i]:
                cover_url = feeds[i]['poster_big']
            elif 'poster_pc' in feeds[i]:
                cover_url = feeds[i]['poster_pc']
            elif 'poster_small' in feeds[i]:
                cover_url = feeds[i]['poster_small']
            else:
                cover_url = 0

            # video_url
            if 'play_url' not in feeds[i]:
                video_url = 0
            else:
                video_url = feeds[i]['play_url']

            Common.logger(log_type).info(f'video_title:{video_title}')
            Common.logger(log_type).info(f'video_id:{video_id}, type:{type(video_id)}')
            Common.logger(log_type).info(f'play_cnt:{play_cnt}')
            Common.logger(log_type).info(f'duration:{duration}')
            Common.logger(log_type).info(f'publish_time:{publish_time}')
            Common.logger(log_type).info(f'user_name:{user_name}')
            Common.logger(log_type).info(f'head_url:{head_url}')
            Common.logger(log_type).info(f'cover_url:{cover_url}')
            Common.logger(log_type).info(f'video_url:{video_url}\n')

            video_dict = {'video_title': video_title,
                          'video_id': video_id,
                          'play_cnt': play_cnt,
                          'duration': duration,
                          'publish_time': publish_time,
                          'user_name': user_name,
                          'head_url': head_url,
                          'cover_url': cover_url,
                          'video_url': video_url}

    @classmethod
    def download_publish(cls, log_type, our_id, video_dict, env):
        if video_dict['video_title'] == 0 or video_dict['video_url'] == 0:
            Common.logger(log_type).info('无效视频\n')
        elif video_dict['video_id'] in [x for y in Feishu.get_values_batch(log_type, 'haokan', '5pWipX') for x in y]:
            Common.logger(log_type).info('视频已下载\n')
        elif video_dict['video_id'] in [x for y in Feishu.get_values_batch(log_type, 'haokan', '7f05d8') for x in y]:
            Common.logger(log_type).info('视频已下载\n')
        elif video_dict['video_id'] in [x for y in Feishu.get_values_batch(log_type, 'haokan', 'A5VCbq') for x in y]:
            Common.logger(log_type).info('视频已下载\n')
        else:
            # 下载
            Common.download_method(log_type, 'cover', video_dict['video_title'], video_dict['cover_url'])
            Common.download_method(log_type, 'video', video_dict['video_title'], video_dict['video_url'])
            with open("./videos/" + video_dict['video_title']
                      + "/" + "info.txt", "a", encoding="UTF-8") as f_a:
                f_a.write(str(video_dict['video_id']) + "\n" +
                          str(video_dict['video_title']) + "\n" +
                          str(video_dict['duration']) + "\n" +
                          '0' + "\n" +
                          '0' + "\n" +
                          '0' + "\n" +
                          '0' + "\n" +
                          '1920*1080' + "\n" +
                          str(int(time.time())) + "\n" +
                          str(video_dict['user_name']) + "\n" +
                          str(video_dict['head_url']) + "\n" +
                          str(video_dict['video_url']) + "\n" +
                          str(video_dict['cover_url']) + "\n" +
                          "HAOKAN" + str(int(time.time())))
            Common.logger(log_type).info("==========视频信息已保存至info.txt==========")

            # 上传
            Common.logger(log_type).info(f"开始上传视频:{video_dict['video_title']}")
            if env == 'dev':
                our_video_id = Publish.upload_and_publish(log_type, our_id, env)
                our_video_link = "https://testadmin.piaoquantv.com/cms/post-detail/" + str(our_video_id) + "/info"
            else:
                our_video_id = Publish.upload_and_publish(log_type, our_id, env)
                our_video_link = "https://admin.piaoquantv.com/cms/post-detail/" + str(our_video_id) + "/info"
            Common.logger(log_type).info(f"视频上传完成:{video_dict['video_title']}\n")

            # 保存视频信息至云文档
            Common.logger(log_type).info(f"保存视频至已下载表:{video_dict['video_title']}")
            Feishu.insert_columns(log_type, "haokan", "5pWipX", "ROWS", 1, 2)
            upload_time = int(time.time())
            values = [[time.strftime("%Y/%m/%d %H:%M:%S", time.localtime(upload_time)),
                       "今日热播榜",
                       video_dict['video_title'],
                       video_dict['video_id'],
                       our_video_link,
                       video_dict['duration'],
                       video_dict['publish_time'],
                       video_dict['user_name'],
                       video_dict['head_url'],
                       video_dict['cover_url'],
                       video_dict['video_url']]]
            time.sleep(1)
            Feishu.update_values(log_type, "haokan", "5pWipX", "F2:Z2", values)
            Common.logger(log_type).info(f"视频:{video_dict['video_title']}，下载/上传成功\n")


if __name__ == '__main__':
    # print(Play.get_tab_from_feishu('play'))
    Play.get_play_videos('play', 'yinyue_new')

    pass