抖音增加异常日志

video_capture/douyin/douyin_author/douyin_author.py

@@ -60,105 +60,107 @@ class douyinAuthor():
     """抖音读取数据 将数据存储到oss上"""
     @classmethod
     def get_videoList(cls):
-        cookie = Material.get_douyin_cookie()
-        # 读取飞书表格,更新数据库用户主页id
-        Material.insert_user()
-        # 获取 用户主页id
-        user_list = cls.get_videoUserId()
-        if len(user_list) == 0:
-            return
-        for i in user_list:
-            account_id = i[0].replace('(', '').replace(')', '').replace(',', '')
-            Common.logger().info(f"用户主页ID：{account_id}")
-            next_cursor = 0
-            while True:
-                if next_cursor == None:
-                    break
-                time.sleep(random.randint(5, 10))
-                url = 'https://www.douyin.com/aweme/v1/web/aweme/post/'
-                headers = {
-                    'Accept': 'application/json, text/plain, */*',
-                    'Accept-Language': 'zh-CN,zh;q=0.9',
-                    'Cache-Control': 'no-cache',
-                    'Cookie': cookie,
-                    'Pragma': 'no-cache',
-                    'Referer': f'https://www.douyin.com/user/{account_id}',
-                    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) '
-                                  'Chrome/118.0.0.0 Safari/537.36',
-                }
-                query = DouYinHelper.get_full_query(ua=headers['User-Agent'], extra_data={
-                    'sec_user_id': account_id,
-                    'max_cursor': next_cursor,
-                    'locate_query': 'false',
-                    'show_live_replay_strategy': '1',
-                    'need_time_list': '1',
-                    'time_list_query': '0',
-                    'whale_cut_token': '',
-                    'cut_version': '1',
-                    'count': '18',
-                    'publish_video_strategy_type': '2',
-                })
-                urllib3.disable_warnings()
-                s = requests.session()
-                s.mount('http://', HTTPAdapter(max_retries=3))
-                s.mount('https://', HTTPAdapter(max_retries=3))
-                response = requests.request(method='GET', url=url, headers=headers, params=query)
-                text = response.text
-                if len(text) == 0:
-                    time.sleep(60)
-                    continue
-                body = response.content.decode()
-                if body == "":
-                    Common.logger().info(f"请求获取body为空")
-                    continue
-                obj = json.loads(body)
-                has_more = True if obj.get('has_more', 0) == 1 else False
-                next_cursor = str(obj.get('max_cursor')) if has_more else None
-                data = obj.get('aweme_list', [])
-                response.close()
-                if response.status_code != 200:
-                    Common.logger().info(
-                        f"接口请求失败，请更换cookie，{response.status_code}")
-                    Feishu.bot('recommend', '抖音', '抖音cookie失效，请及时更换～')
-                    # 如果返回空信息，则随机睡眠 600, 1200 秒
-                    time.sleep(random.randint(600, 1200))
-                    continue
-                elif len(data) == 0:
-                    Common.logger().info(
-                        f"接口请求失败，请更换cookie")
-                    Feishu.bot('recommend', '抖音', '抖音cookie失效，请及时更换～')
-                    # 如果返回空信息，则随机睡眠 600, 1200 秒
-                    time.sleep(random.randint(600, 1200))
-                    continue
-                count = 0
-                for i in range(len(data)):
-                    try:
-                        entity_type = data[i].get('search_impr').get('entity_type')
+        try:
+            cookie = Material.get_douyin_cookie()
+            # 读取飞书表格,更新数据库用户主页id
+            Material.insert_user()
+            # 获取 用户主页id
+            user_list = cls.get_videoUserId()
+            if len(user_list) == 0:
+                return
+            for i in user_list:
+                account_id = i[0].replace('(', '').replace(')', '').replace(',', '')
+                Common.logger().info(f"用户主页ID：{account_id}")
+                next_cursor = 0
+                while True:
+                    if next_cursor == None:
+                        break
+                    time.sleep(random.randint(5, 10))
+
+                    url = 'https://www.douyin.com/aweme/v1/web/aweme/post/'
+                    headers = {
+                        'Accept': 'application/json, text/plain, */*',
+                        'Accept-Language': 'zh-CN,zh;q=0.9',
+                        'Cache-Control': 'no-cache',
+                        'Cookie': cookie,
+                        'Pragma': 'no-cache',
+                        'Referer': f'https://www.douyin.com/user/{account_id}',
+                        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) '
+                                      'Chrome/118.0.0.0 Safari/537.36',
+                    }
+                    query = DouYinHelper.get_full_query(ua=headers['User-Agent'], extra_data={
+                        'sec_user_id': account_id,
+                        'max_cursor': next_cursor,
+                        'locate_query': 'false',
+                        'show_live_replay_strategy': '1',
+                        'need_time_list': '1',
+                        'time_list_query': '0',
+                        'whale_cut_token': '',
+                        'cut_version': '1',
+                        'count': '18',
+                        'publish_video_strategy_type': '2',
+                    })
+                    urllib3.disable_warnings()
+                    s = requests.session()
+                    s.mount('http://', HTTPAdapter(max_retries=3))
+                    s.mount('https://', HTTPAdapter(max_retries=3))
+                    response = requests.request(method='GET', url=url, headers=headers, params=query)
+                    text = response.text
+                    if len(text) == 0:
+                        time.sleep(60)
+                        continue
+                    body = response.content.decode()
+                    obj = json.loads(body)
+                    has_more = True if obj.get('has_more', 0) == 1 else False
+                    next_cursor = str(obj.get('max_cursor')) if has_more else None
+                    data = obj.get('aweme_list', [])
+                    response.close()
+                    if response.status_code != 200:
                         Common.logger().info(
-                            f"非视频:{entity_type}")
-                        if entity_type == 'GENERAL':
-                            count += 1
-                            video_id = data[i].get('aweme_id')  # 文章id
-                            id = cls.select_videoUrl_id(video_id)
-                            if id:
-                                if count > 5:
-                                    Common.logger().info(
-                                        f"重复视频不在抓取该用户，用户主页id：{account_id}")
-                                    break
-                                continue
-                            video_url = data[i].get('video').get('play_addr').get('url_list')[0]  # 视频链接
-                            oss_object_key = Oss.video_sync_upload_oss(video_url, video_id, account_id)
-                            status = oss_object_key.get("status")
-                            # 发送 oss
-                            oss_object_key = oss_object_key.get("oss_object_key")
-                            Common.logger().info(f"抖音视频链接oss发送成功，oss地址：{oss_object_key}")
-                            # 发送成功 存入数据库
-                            if status == 200:
-                                cls.insert_videoUrl(video_id, account_id, oss_object_key)
-                                Common.logger().info(f"视频地址插入数据库成功，视频id：{video_id},用户主页id：{account_id},视频储存地址：{oss_object_key}")
-                    except Exception as e:
-                        Common.logger().warning(f"抓取单条视频异常:{e}\n")
+                            f"接口请求失败，请更换cookie，{response.status_code}")
+                        Feishu.bot('recommend', '抖音', '抖音cookie失效，请及时更换～')
+                        # 如果返回空信息，则随机睡眠 600, 1200 秒
+                        time.sleep(random.randint(600, 1200))
                         continue
+                    elif len(data) == 0:
+                        Common.logger().info(
+                            f"接口请求失败，请更换cookie")
+                        Feishu.bot('recommend', '抖音', '抖音cookie失效，请及时更换～')
+                        # 如果返回空信息，则随机睡眠 600, 1200 秒
+                        time.sleep(random.randint(600, 1200))
+                        continue
+                    count = 0
+                    for i in range(len(data)):
+                        try:
+                            entity_type = data[i].get('search_impr').get('entity_type')
+                            Common.logger().info(
+                                f"非视频:{entity_type}")
+                            if entity_type == 'GENERAL':
+                                count += 1
+                                video_id = data[i].get('aweme_id')  # 文章id
+                                id = cls.select_videoUrl_id(video_id)
+                                if id:
+                                    if count > 5:
+                                        Common.logger().info(
+                                            f"重复视频不在抓取该用户，用户主页id：{account_id}")
+                                        break
+                                    continue
+                                video_url = data[i].get('video').get('play_addr').get('url_list')[0]  # 视频链接
+                                oss_object_key = Oss.video_sync_upload_oss(video_url, video_id, account_id)
+                                status = oss_object_key.get("status")
+                                # 发送 oss
+                                oss_object_key = oss_object_key.get("oss_object_key")
+                                Common.logger().info(f"抖音视频链接oss发送成功，oss地址：{oss_object_key}")
+                                # 发送成功 存入数据库
+                                if status == 200:
+                                    cls.insert_videoUrl(video_id, account_id, oss_object_key)
+                                    Common.logger().info(f"视频地址插入数据库成功，视频id：{video_id},用户主页id：{account_id},视频储存地址：{oss_object_key}")
+                        except Exception as e:
+                            Common.logger().warning(f"抓取单条视频异常:{e}\n")
+                            continue
+        except Exception as e:
+            Common.logger().warning(f"抓取异常:{e}\n")
+            return
 
 if __name__ == '__main__':
     douyinAuthor.get_videoList()