update

main/haokan_hot.py

@@ -5,6 +5,8 @@ import os
 import sys
 import time
 import requests
+import urllib3
+
 sys.path.append(os.getcwd())
 from main.common import Common
 from main.feishu_lib import Feishu
@@ -17,114 +19,118 @@ class Hot:
 
     @classmethod
     def get_hot_feeds(cls, log_type, our_id, env):
-        while True:
-            cls.page += 1
-            url = 'https://haokan.baidu.com/videoui/page/pc/toplist?'
-            headers = {
-                'Accept-Encoding': 'gzip, deflate, br',
-                'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
-                'Cache-Control': 'no-cache',
-                'Content-Type': 'application/x-www-form-urlencoded',
-                'Cookie': 'BIDUPSID=0C817797C726E2312710D870ECDAE8A2; BDORZ=B490B5EBF6F3CD402E515D22BCDA1598; PSTM=1669001132; BAIDUID=AB7069CAF9ECB7AA43E400D164119733:FG=1; BDSFRCVID=H54OJexroG0GmHrjfnewtKf1EeKK0gOTDYLEJs2qYShnrsPVJeC6EG0PtoWQkz--EHtdogKK0mOTHv8F_2uxOjjg8UtVJeC6EG0Ptf8g0M5; H_BDCLCKID_SF=tR333R7oKRu_HRjYbb__-P4DePAttURZ56bHWh0M3b61qRcIh4ob5MPEDto-BMPj52OnKUT13lc5h4jX0P7_KRtr346-35543bRTLn76LRv0Kj6HybOfhP-UyN3LWh37bJblMKoaMp78jR093JO4y4Ldj4oxJpOJ5JbMopCafJOKHICGDTA-jMK; Hm_lvt_4aadd610dfd2f5972f1efee2653a2bc5=1669029805; PC_TAB_LOG=video_details_page; COMMON_LID=88bc9b0fbce964fbb6a76cfd7927d02b; hkpcvideolandquery=%u4E2D%u56FD%u7537%u513F%u5218%u5F3A%u6012%u70E7%u9756%u56FD%u795E%u793E%uFF0C%u518D%u70E7%u65E5%u672C%u9A7B%u97E9%u5927%u4F7F%u9986%uFF0C%u540E%u6765%u600E%u4E48%u6837%u4E86%uFF1F; BDSFRCVID_BFESS=H54OJexroG0GmHrjfnewtKf1EeKK0gOTDYLEJs2qYShnrsPVJeC6EG0PtoWQkz--EHtdogKK0mOTHv8F_2uxOjjg8UtVJeC6EG0Ptf8g0M5; H_BDCLCKID_SF_BFESS=tR333R7oKRu_HRjYbb__-P4DePAttURZ56bHWh0M3b61qRcIh4ob5MPEDto-BMPj52OnKUT13lc5h4jX0P7_KRtr346-35543bRTLn76LRv0Kj6HybOfhP-UyN3LWh37bJblMKoaMp78jR093JO4y4Ldj4oxJpOJ5JbMopCafJOKHICGDTA-jMK; H_PS_PSSID=36557_37769_34813_37778_37728_36806_37719_37743_26350_37787; PSINO=2; delPer=0; BA_HECTOR=8la125a48ka00la4agal0he91hnre381e; BAIDUID_BFESS=AB7069CAF9ECB7AA43E400D164119733:FG=1; ZFY=e8fgayH:A:A8S2QAjzqo6RRIm7t3wLMhAdtluiJeCoSc4:C; Hm_lpvt_4aadd610dfd2f5972f1efee2653a2bc5=1669206505; ariaDefaultTheme=undefined; RT="z=1&dm=baidu.com&si=7jsviskg99&ss=latmgdwn&sl=1&tt=10y&bcn=https%3A%2F%2Ffclog.baidu.com%2Flog%2Fweirwood%3Ftype%3Dperf&ld=1tl"',
-                'Pragma': 'no-cache',
-                'Referer': 'https://haokan.baidu.com/',
-                'Sec-Fetch-Dest': 'empty',
-                'Sec-Fetch-Mode': 'cors',
-                'Sec-Fetch-Site': 'same-origin',
-                'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36 Edg/107.0.1418.52',
-                'sec-ch-ua': '"Microsoft Edge";v="107", "Chromium";v="107", "Not=A?Brand";v="24"',
-                'sec-ch-ua-mobile': '?0',
-                'sec-ch-ua-platform': '"macOS"'
-            }
-            params = {
-                'type': 'hotvideo',
-                'sfrom': 'haokan_web_banner',
-                'pageSize': '20',
-                '_format': 'json',
-                'page': str(cls.page)
-            }
-            r = requests.get(url=url, headers=headers, params=params)
-            if r.json()['errno'] != 0 or r.json()['error'] != '成功':
-                Common.logger(log_type).error(f'feeds_response:{r.text}\n')
-            elif len(r.json()['apiData']['response']['video']) == 0:
-                Common.logger(log_type).info(f'没有新数据了\n')
-                return
-            else:
-                feeds = r.json()['apiData']['response']['video']
-                Common.logger(log_type).info(f'正在抓取第{cls.page}页\n')
-                for i in range(len(feeds)):
-                    # video_title
-                    if 'title' not in feeds[i]:
-                        video_title = 0
-                    else:
-                        video_title = feeds[i]['title']
-
-                    # video_id
-                    if 'vid' not in feeds[i]:
-                        video_id = 0
-                    else:
-                        video_id = feeds[i]['vid']
-
-                    # duration
-                    if 'duration' not in feeds[i]:
-                        duration = 0
-                    else:
-                        duration = feeds[i]['duration']
-
-                    # publish_time
-                    if 'publish_time' not in feeds[i]:
-                        publish_time = 0
-                    else:
-                        publish_time = feeds[i]['publish_time']\
-                            .replace('发布时间：', '').replace('年', '/').replace('月', '/').replace('日', '')
-
-                    # user_name
-                    if 'author' not in feeds[i]:
-                        user_name = 0
-                    else:
-                        user_name = feeds[i]['author']
-
-                    # head_url
-                    if 'author_icon' not in feeds[i]:
-                        head_url = 0
-                    else:
-                        head_url = feeds[i]['author_icon']
-
-                    # cover_url
-                    if 'poster' not in feeds[i]:
-                        cover_url = 0
-                    else:
-                        cover_url = feeds[i]['poster']
-
-                    # video_url
-                    get_video_url = cls.get_video_url(log_type, video_id)
-                    if get_video_url is not None:
-                        video_url = get_video_url
-                    elif 'videoUrl' in feeds[i]:
-                        video_url = feeds[i]['videoUrl']
-                    else:
-                        video_url = 0
-
-                    Common.logger(log_type).info(f'video_title:{video_title}')
-                    Common.logger(log_type).info(f'video_id:{video_id}')
-                    Common.logger(log_type).info(f'duration:{duration}')
-                    Common.logger(log_type).info(f'publish_time:{publish_time}')
-                    Common.logger(log_type).info(f'user_name:{user_name}')
-                    Common.logger(log_type).info(f'head_url:{head_url}')
-                    Common.logger(log_type).info(f'cover_url:{cover_url}')
-                    Common.logger(log_type).info(f'video_url:{video_url}\n')
-
-                    video_dict = {'video_title': video_title,
-                                  'video_id': video_id,
-                                  'duration': duration,
-                                  'publish_time': publish_time,
-                                  'user_name': user_name,
-                                  'head_url': head_url,
-                                  'cover_url': cover_url,
-                                  'video_url': video_url}
-                    cls.download_publish(log_type, video_dict, our_id, env)
-
-                time.sleep(5)
+        try:
+            while True:
+                cls.page += 1
+                url = 'https://haokan.baidu.com/videoui/page/pc/toplist?'
+                headers = {
+                    'Accept-Encoding': 'gzip, deflate, br',
+                    'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
+                    'Cache-Control': 'no-cache',
+                    'Content-Type': 'application/x-www-form-urlencoded',
+                    'Cookie': 'BIDUPSID=0C817797C726E2312710D870ECDAE8A2; BDORZ=B490B5EBF6F3CD402E515D22BCDA1598; PSTM=1669001132; BAIDUID=AB7069CAF9ECB7AA43E400D164119733:FG=1; BDSFRCVID=H54OJexroG0GmHrjfnewtKf1EeKK0gOTDYLEJs2qYShnrsPVJeC6EG0PtoWQkz--EHtdogKK0mOTHv8F_2uxOjjg8UtVJeC6EG0Ptf8g0M5; H_BDCLCKID_SF=tR333R7oKRu_HRjYbb__-P4DePAttURZ56bHWh0M3b61qRcIh4ob5MPEDto-BMPj52OnKUT13lc5h4jX0P7_KRtr346-35543bRTLn76LRv0Kj6HybOfhP-UyN3LWh37bJblMKoaMp78jR093JO4y4Ldj4oxJpOJ5JbMopCafJOKHICGDTA-jMK; Hm_lvt_4aadd610dfd2f5972f1efee2653a2bc5=1669029805; PC_TAB_LOG=video_details_page; COMMON_LID=88bc9b0fbce964fbb6a76cfd7927d02b; hkpcvideolandquery=%u4E2D%u56FD%u7537%u513F%u5218%u5F3A%u6012%u70E7%u9756%u56FD%u795E%u793E%uFF0C%u518D%u70E7%u65E5%u672C%u9A7B%u97E9%u5927%u4F7F%u9986%uFF0C%u540E%u6765%u600E%u4E48%u6837%u4E86%uFF1F; BDSFRCVID_BFESS=H54OJexroG0GmHrjfnewtKf1EeKK0gOTDYLEJs2qYShnrsPVJeC6EG0PtoWQkz--EHtdogKK0mOTHv8F_2uxOjjg8UtVJeC6EG0Ptf8g0M5; H_BDCLCKID_SF_BFESS=tR333R7oKRu_HRjYbb__-P4DePAttURZ56bHWh0M3b61qRcIh4ob5MPEDto-BMPj52OnKUT13lc5h4jX0P7_KRtr346-35543bRTLn76LRv0Kj6HybOfhP-UyN3LWh37bJblMKoaMp78jR093JO4y4Ldj4oxJpOJ5JbMopCafJOKHICGDTA-jMK; H_PS_PSSID=36557_37769_34813_37778_37728_36806_37719_37743_26350_37787; PSINO=2; delPer=0; BA_HECTOR=8la125a48ka00la4agal0he91hnre381e; BAIDUID_BFESS=AB7069CAF9ECB7AA43E400D164119733:FG=1; ZFY=e8fgayH:A:A8S2QAjzqo6RRIm7t3wLMhAdtluiJeCoSc4:C; Hm_lpvt_4aadd610dfd2f5972f1efee2653a2bc5=1669206505; ariaDefaultTheme=undefined; RT="z=1&dm=baidu.com&si=7jsviskg99&ss=latmgdwn&sl=1&tt=10y&bcn=https%3A%2F%2Ffclog.baidu.com%2Flog%2Fweirwood%3Ftype%3Dperf&ld=1tl"',
+                    'Pragma': 'no-cache',
+                    'Referer': 'https://haokan.baidu.com/',
+                    'Sec-Fetch-Dest': 'empty',
+                    'Sec-Fetch-Mode': 'cors',
+                    'Sec-Fetch-Site': 'same-origin',
+                    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36 Edg/107.0.1418.52',
+                    'sec-ch-ua': '"Microsoft Edge";v="107", "Chromium";v="107", "Not=A?Brand";v="24"',
+                    'sec-ch-ua-mobile': '?0',
+                    'sec-ch-ua-platform': '"macOS"'
+                }
+                params = {
+                    'type': 'hotvideo',
+                    'sfrom': 'haokan_web_banner',
+                    'pageSize': '20',
+                    '_format': 'json',
+                    'page': str(cls.page)
+                }
+                urllib3.disable_warnings()
+                r = requests.get(url=url, headers=headers, params=params, verify=False)
+                if r.json()['errno'] != 0 or r.json()['error'] != '成功':
+                    Common.logger(log_type).error(f'feeds_response:{r.text}\n')
+                elif len(r.json()['apiData']['response']['video']) == 0:
+                    Common.logger(log_type).info(f'没有新数据了\n')
+                    return
+                else:
+                    feeds = r.json()['apiData']['response']['video']
+                    Common.logger(log_type).info(f'正在抓取第{cls.page}页\n')
+                    for i in range(len(feeds)):
+                        # video_title
+                        if 'title' not in feeds[i]:
+                            video_title = 0
+                        else:
+                            video_title = feeds[i]['title']
+
+                        # video_id
+                        if 'vid' not in feeds[i]:
+                            video_id = 0
+                        else:
+                            video_id = feeds[i]['vid']
+
+                        # duration
+                        if 'duration' not in feeds[i]:
+                            duration = 0
+                        else:
+                            duration = feeds[i]['duration']
+
+                        # publish_time
+                        if 'publish_time' not in feeds[i]:
+                            publish_time = 0
+                        else:
+                            publish_time = feeds[i]['publish_time']\
+                                .replace('发布时间：', '').replace('年', '/').replace('月', '/').replace('日', '')
+
+                        # user_name
+                        if 'author' not in feeds[i]:
+                            user_name = 0
+                        else:
+                            user_name = feeds[i]['author']
+
+                        # head_url
+                        if 'author_icon' not in feeds[i]:
+                            head_url = 0
+                        else:
+                            head_url = feeds[i]['author_icon']
+
+                        # cover_url
+                        if 'poster' not in feeds[i]:
+                            cover_url = 0
+                        else:
+                            cover_url = feeds[i]['poster']
+
+                        # video_url
+                        get_video_url = cls.get_video_url(log_type, video_id)
+                        if get_video_url is not None:
+                            video_url = get_video_url
+                        elif 'videoUrl' in feeds[i]:
+                            video_url = feeds[i]['videoUrl']
+                        else:
+                            video_url = 0
+
+                        Common.logger(log_type).info(f'video_title:{video_title}')
+                        Common.logger(log_type).info(f'video_id:{video_id}')
+                        Common.logger(log_type).info(f'duration:{duration}')
+                        Common.logger(log_type).info(f'publish_time:{publish_time}')
+                        Common.logger(log_type).info(f'user_name:{user_name}')
+                        Common.logger(log_type).info(f'head_url:{head_url}')
+                        Common.logger(log_type).info(f'cover_url:{cover_url}')
+                        Common.logger(log_type).info(f'video_url:{video_url}\n')
+
+                        video_dict = {'video_title': video_title,
+                                      'video_id': video_id,
+                                      'duration': duration,
+                                      'publish_time': publish_time,
+                                      'user_name': user_name,
+                                      'head_url': head_url,
+                                      'cover_url': cover_url,
+                                      'video_url': video_url}
+                        cls.download_publish(log_type, video_dict, our_id, env)
+
+                    time.sleep(5)
+        except Exception as e:
+            Common.logger(log_type).error(f'get_hot_feeds异常:{e}\n')
 
     @classmethod
     def get_video_url(cls, log_type, video_id):
@@ -152,7 +158,8 @@ class Hot:
                 'Sec-Fetch-Site': 'same-origin',
                 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36 Edg/107.0.1418.62',
             }
-            r = requests.get(url=url, headers=headers, params=params)
+            urllib3.disable_warnings()
+            r = requests.get(url=url, headers=headers, params=params, verify=False)
             if r.status_code != 200:
                 Common.logger(log_type).info(f'get_video_url_response:{r.text}')
             elif r.json()['errno'] != 0 or len(r.json()['data']) == 0: