crawler_haokan/main/demo.py

# -*- coding: utf-8 -*-
# @Author: wangkun
# @Time: 2022/11/23
import datetime
import time
import requests

from main.common import Common
from main.feishu_lib import Feishu


class Demo:
    @classmethod
    def get_sheet(cls, log_type, crawler, sheetid):
        sheet = Feishu.get_values_batch(log_type, crawler, sheetid)
        print(sheet)
        # a = '2023/01/01 13:13:13'
        # Feishu.update_values(log_type, crawler, sheetid, 'F2:F2', [[a]])

    @classmethod
    def publish_time(cls, publish_time):
        today = datetime.date.today()
        if '刚刚' in publish_time:
            publish_time_stamp = int(time.time())
        elif '分钟前' in publish_time:
            publish_time_stamp = int(time.time()) - int(publish_time[0])*60
        elif '小时前' in publish_time:
            publish_time_stamp = int(time.time()) - int(publish_time[0])*3600
        elif '昨天' in publish_time:
            publish_time_str = (datetime.date.today() + datetime.timedelta(days=-1)).strftime("%Y/%m/%d")
            publish_time_stamp = int(time.mktime(time.strptime(publish_time_str, "%Y/%m/%d")))
        elif '天前' in publish_time:
            publish_time_str = today - datetime.timedelta(days=int(publish_time[0]))
            publish_time_stamp = int(time.mktime(publish_time_str.timetuple()))
        elif '年' in publish_time:
            publish_time_str = publish_time.replace('年', '/').replace('月', '/').replace('日', '')
            publish_time_stamp = int(time.mktime(time.strptime(publish_time_str, "%Y/%m/%d")))
        else:
            publish_time_str = publish_time.replace('月', '/').replace('日', '')
            this_year = datetime.datetime.now().year
            publish_time_stamp = int(time.mktime(time.strptime(f"{this_year}/{publish_time_str}", "%Y/%m/%d")))

        print(f'publish_time_stamp:{publish_time_stamp}')

    @classmethod
    def get_video_url(cls, log_type, video_id):
        url = 'https://haokan.hao123.com/v?'
        params = {
            'vid': video_id,
            '_format': 'json',
        }
        headers = {
            'Accept': '*/*',
            'Accept-Encoding': 'gzip, deflate, br',
            'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
            'Cache-Control': 'no-cache',
            'Connection': 'keep-alive',
            'Content-Type': 'application/x-www-form-urlencoded',
            'Cookie': 'PC_TAB_LOG=video_details_page; COMMON_LID=b0be69dd9fcae328d06935bd40f615cd; Hm_lvt_4aadd610dfd2f5972f1efee2653a2bc5=1669029953; hkpcvideolandquery=%u82CF%u5DDE%u6700%u5927%u7684%u4E8C%u624B%u8F66%u8D85%u5E02%uFF0C%u8F6C%u4E00%u8F6C%u91CC%u8FB9%u8C6A%u8F66%u592A%u591A%u4E86%uFF0C%u4EF7%u683C%u66F4%u8BA9%u6211%u5403%u60CA%uFF01; Hm_lpvt_4aadd610dfd2f5972f1efee2653a2bc5=1669875695; ariaDefaultTheme=undefined; reptileData=%7B%22data%22%3A%22636c55e0319da5169a60acec4a264a35c10862f8abfe2f2cc32c55eb6b0ab4de0efdfa115ea522d6d4d361dea07feae2831d3e2c16ed6b051c611ffe5aded6c9f852501759497b9fbd2132a2160e1e40e5845b41f78121ddcc3288bd077ae4e8%22%2C%22key_id%22%3A%2230%22%2C%22sign%22%3A%22f6752aac%22%7D; RT="z=1&dm=hao123.com&si=uc0q7wnm4w&ss=lb4otu71&sl=j&tt=av0&bcn=https%3A%2F%2Ffclog.baidu.com%2Flog%2Fweirwood%3Ftype%3Dperf&ld=1rdw&cl=7v6c"',
            'Pragma': 'no-cache',
            'Referer': 'https://haokan.hao123.com/v?vid=10623278258033022286&pd=pc&context=',
            'sec-ch-ua': '"Microsoft Edge";v="107", "Chromium";v="107", "Not=A?Brand";v="24"',
            'sec-ch-ua-mobile': '?0',
            'sec-ch-ua-platform': '"macOS"',
            'Sec-Fetch-Dest': 'empty',
            'Sec-Fetch-Mode': 'cors',
            'Sec-Fetch-Site': 'same-origin',
            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36 Edg/107.0.1418.62',
        }
        r = requests.get(url=url, headers=headers, params=params)
        if r.status_code != 200:
            Common.logger(log_type).info(f'get_video_url_response:{r.text}')
        elif r.json()['errno'] != 0 or len(r.json()['data']) == 0:
            Common.logger(log_type).info(f'get_video_url_response:{r.json()}')
        else:
            clarityUrl = r.json()['data']['apiData']['curVideoMeta']['clarityUrl']
            video_url = r.json()['data']['apiData']['curVideoMeta']['clarityUrl'][len(clarityUrl) - 1]['url']

            return video_url

    user_pcursor = ''
    user_cnt = []

    @classmethod
    def get_follow_users(cls):
        while True:
            url = "https://www.kuaishou.com/graphql"
            payload = {
                "operationName": "visionProfileUserList",
                "variables": {"ftype": 1, "pcursor": str(cls.user_pcursor)},
                "query": "query visionProfileUserList($pcursor: String, $ftype: Int) "
                         "{\n  visionProfileUserList(pcursor: $pcursor, ftype: $ftype) "
                         "{\n    result\n    fols {\n      user_name\n      headurl\n      "
                         "user_text\n      isFollowing\n      user_id\n      __typename\n    }\n"
                         "    hostName\n    pcursor\n    __typename\n  }\n}\n"
            }
            headers = {
                'accept': '*/*',
                'Accept-Encoding': 'gzip, deflate, br',
                'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
                'Cache-Control': 'no-cache',
                'Connection': 'keep-alive',
                'Content-Length': '382',
                'content-type': 'application/json',
                'Cookie': 'kpf=PC_WEB; '
                          'kpn=KUAISHOU_VISION; '
                          'clientid=3; '
                          'did=web_260fc6cc87668951b745d0aa536b6c45; '
                          'client_key=65890b29; '
                          'userId=1921947321; '
                          'kuaishou.server.web_st=ChZrdWFpc2hvdS5zZXJ2ZXIud2ViLnN0EqABeKWnZv84JvnYIdwhX6o5jaBjDblgvsn'
                          '7o8XEAvihFCj1P-69BK0qcjnCMbpUUmfT7pqlsn0KVsuK5789LXjySbqdee8yKhNhM-kttccnRokJM7BbSl7VzViTd'
                          'afHKDM3UEh1oJKy5s_iMUHbXo6wHSbMjdFYP6QsSRf7xU9KA4IC-uNX9ZcBj39OuCUCjlQFOXLjJV8mYAcBt1BqrIT'
                          'VhhoS6uws2LN-siMyPVYdMaXTUH7FIiCEI0MRi2vzOYp7_RSnHX_7--xNCDYQVUo05gjITkWaGCgFMAE; '
                          'kuaishou.server.web_ph=f2b6b9531d9ade009d5dccf1351ae7d4c7d2',
                'Pragma': 'no-cache',
                'Referer': 'https://www.kuaishou.com/profile/3xbp7922t5sgcpq',
                'sec-ch-ua': '"Not?A_Brand";v="8", "Chromium";v="108", "Google Chrome";v="108"',
                'sec-ch-ua-mobile': '?0',
                'sec-ch-ua-platform': '"macOS"',
                'Sec-Fetch-Dest': 'empty',
                'Sec-Fetch-Mode': 'cors',
                'Sec-Fetch-Site': 'same-origin',
                'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 '
                              '(KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36'
            }
            r = requests.post(url=url, headers=headers, json=payload)
            fols = r.json()['data']['visionProfileUserList']['fols']
            cls.user_pcursor = r.json()['data']['visionProfileUserList']['pcursor']
            if len(fols) == 0:
                print(f'到底啦~\n共关注 {len(cls.user_cnt)} 人\n')
                cls.user_pcursor = 0
                cls.user_cnt = []
                return
            for i in range(len(fols)):
                user_name = fols[i]['user_name']
                user_id = fols[i]['user_id']
                cls.user_cnt.append(user_id)

                print(f'user_name:{user_name}')
                print(f'user_id:{user_id}')
                print('\n')

    @classmethod
    def get_video_feeds(cls, log_type, out_id):
        url = 'https://www.kuaishou.com/graphql'
        headers = {
            'accept': '*/*',
            'Accept-Encoding': 'gzip, deflate, br',
            'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
            'Cache-Control': 'no-cache',
            'content-type': 'application/json',
            'Cookie': 'kpf=PC_WEB; kpn=KUAISHOU_VISION; clientid=3; did=web_260fc6cc87668951b745d0aa536b6c45; client_key=65890b29; userId=1921947321; kuaishou.server.web_st=ChZrdWFpc2hvdS5zZXJ2ZXIud2ViLnN0EqABwCuiLLMSOvJtdcmO-XD0RHf-lRaR--1o0MNMhnsdl1xRZOfkHgEnFM1GNwk-pNxGwIXupJiGfL6JC-dsw6pNVRfW9eiCMUiFfVSS8bbVuBhSnh4wlJc6e2wy6FeYXC4V2WUQ7CKlUc_1sESmPlYq1D2JV7eKyNJP9VVnZ1i_y31SnTnAx2DpncW8UNV2qv_bjTxokg4rZRPnBBRCfAAhShoStEyT9S95saEmiR8Dg-bb1DKRIiDJ8NOu7QSy1FkD3GwzpAURUTTYYTuENNqldrStQMfWxygFMAE; kuaishou.server.web_ph=f80aff104d1749fbb5b6760aee76d5b640e2',
            'Pragma': 'no-cache',
            'Referer': 'https://www.kuaishou.com/profile/'+str(out_id),
            'sec-ch-ua': '"Not?A_Brand";v="8", "Chromium";v="108", "Google Chrome";v="108"',
            'sec-ch-ua-mobile': '?0',
            'sec-ch-ua-platform': '"macOS"',
            'Sec-Fetch-Dest': 'empty',
            'Sec-Fetch-Mode': 'cors',
            'Sec-Fetch-Site': 'same-origin',
            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36',
        }
        payload = {
            "operationName": "visionProfilePhotoList",
            "variables": {
                "userId": str(out_id),
                "pcursor": "",
                "page": "profile"
            },
            "query": "fragment photoContent on PhotoEntity {\n  id\n  duration\n  caption\n  originCaption\n  likeCount\n  viewCount\n  realLikeCount\n  coverUrl\n  photoUrl\n  photoH265Url\n  manifest\n  manifestH265\n  videoResource\n  coverUrls {\n    url\n    __typename\n  }\n  timestamp\n  expTag\n  animatedCoverUrl\n  distance\n  videoRatio\n  liked\n  stereoType\n  profileUserTopPhoto\n  musicBlocked\n  __typename\n}\n\nfragment feedContent on Feed {\n  type\n  author {\n    id\n    name\n    headerUrl\n    following\n    headerUrls {\n      url\n      __typename\n    }\n    __typename\n  }\n  photo {\n    ...photoContent\n    __typename\n  }\n  canAddComment\n  llsid\n  status\n  currentPcursor\n  tags {\n    type\n    name\n    __typename\n  }\n  __typename\n}\n\nquery visionProfilePhotoList($pcursor: String, $userId: String, $page: String, $webPageArea: String) {\n  visionProfilePhotoList(pcursor: $pcursor, userId: $userId, page: $page, webPageArea: $webPageArea) {\n    result\n    llsid\n    webPageArea\n    feeds {\n      ...feedContent\n      __typename\n    }\n    hostName\n    pcursor\n    __typename\n  }\n}\n"
        }
        r = requests.post(url=url, headers=headers, json=payload)
        feeds = r.json()['data']['visionProfilePhotoList']['feeds']
        if len(feeds) == 0:
            Common.logger(log_type).info('到底啦~\n')
            return
        for i in range(len(feeds)):
            # 视频标题过滤话题及处理特殊字符
            kuaishou_title = feeds[i]['photo']["caption"]
            title_split1 = kuaishou_title.split(" #")
            if title_split1[0] != "":
                title1 = title_split1[0]
            else:
                title1 = title_split1[-1]

            title_split2 = title1.split(" #")
            if title_split2[0] != "":
                title2 = title_split2[0]
            else:
                title2 = title_split2[-1]

            title_split3 = title2.split("@")
            if title_split3[0] != "":
                title3 = title_split3[0]
            else:
                title3 = title_split3[-1]

            video_title = title3.strip()\
                .replace("\n", "").replace("/", "").replace("快手", "").replace(" ", "").replace(" ", "")\
                .replace("&NBSP", "").replace("\r", "").replace("#", "").replace(".", "。").replace("\\", "")\
                .replace(":", "").replace("*", "").replace("？", "").replace("?", "").replace('"', "")\
                .replace("<", "").replace(">", "").replace("|", "").replace("@", "")[:40]
            hevc = feeds[i]['photo']['videoResource']['hevc']
            if hevc == '':
                video_url = feeds[i]['photo']['videoResource']['h264']['adaptationSet'][0]['representation'][0]['url']
            else:
                video_url = feeds[i]['photo']['videoResource']['hevc']['adaptationSet'][0]['representation'][0]['url']

            print(f'video_title:{video_title}')
            print(f'video_url:{video_url}')
            print('\n')


if __name__ == '__main__':
    Demo.get_sheet('demo', 'haokan', 'kVaSjf')
    # Demo.publish_time()
    # Demo.get_video_url('demo', '10377041690614321392')
    # Demo.get_follow_users('demo')
    # Demo.get_video_feeds('demo', '3xfr3gqnxmk92y2')
    # print(Feishu.get_values_batch('log_type', 'haokan', '5LksMx')[0][0])
    # print(type(Feishu.get_values_batch('log_type', 'haokan', '5LksMx')[0][0]))

    # Demo.publish_time(publish_time='刚刚')
    # Demo.publish_time(publish_time='1分钟前')
    # Demo.publish_time(publish_time='1小时前')
    # Demo.publish_time(publish_time='昨天')
    # Demo.publish_time(publish_time='3天前')
    # Demo.publish_time(publish_time='2022年01月10日')
    # Demo.publish_time(publish_time='01月10日')

    pass