增加抖音数据抓取、视频自动拼接代码

common/aliyun_oss_uploading.py

@@ -0,0 +1,74 @@
+# -*- coding: utf-8 -*-
+# @Time: 2023/12/26
+from datetime import datetime
+from typing import Dict, Any,  Optional
+
+import oss2
+import requests
+
+OSS_BUCKET_PATH = "douyin"
+OSS_ACCESS_KEY_ID = "LTAI5t9FaRSZZX2n6cnvhveb"
+OSS_ACCESS_KEY_SECRET = "VvMH6NkfByov57RWzBi4lKPdfrpdVu"
+OSS_BUCKET_ENDPOINT = "oss-cn-hangzhou.aliyuncs.com" # 外网地址上到服务器需要更改为内网地址
+OSS_BUCKET_NAME = "art-crawler"
+class Oss():
+    # 抓取视频上传到art-crawler
+    @classmethod
+    def video_sync_upload_oss(cls, src_url: str,
+                        video_id: str,
+                        account_id: str,
+                        referer: Optional[str] = None) -> Dict[str, Any]:
+        headers = {
+            'Accept': '*/*',
+            'Accept-Language': 'zh-CN,zh;q=0.9',
+            'Cache-Control': 'no-cache',
+            'Pragma': 'no-cache',
+            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) '
+                          'Chrome/117.0.0.0 Safari/537.36',
+        }
+        if referer:
+            headers.update({'Referer': referer})
+        response = requests.request(url=src_url, method='GET', headers=headers)
+        file_content = response.content
+        content_type = response.headers.get('Content-Type', 'application/octet-stream')
+
+        oss_object_key = f'{OSS_BUCKET_PATH}/{account_id}/{video_id}'
+        auth = oss2.Auth(OSS_ACCESS_KEY_ID, OSS_ACCESS_KEY_SECRET)
+        bucket = oss2.Bucket(auth, OSS_BUCKET_ENDPOINT, OSS_BUCKET_NAME)
+        response = bucket.put_object(oss_object_key, file_content, headers={'Content-Type': content_type})
+
+        if 'Content-Length' in response.headers:
+            return {
+                'status': response.status,
+                'oss_object_key': oss_object_key}
+        raise AssertionError(f'OSS上传失败，请求ID: \n{response.headers["x-oss-request-id"]}')
+
+    @classmethod
+    def stitching_sync_upload_oss(cls, src_url: str,
+                        video_id: str) -> Dict[str, Any]:
+        oss_object_key = f'videostitching/video_id/{video_id}'
+        auth = oss2.Auth(OSS_ACCESS_KEY_ID, OSS_ACCESS_KEY_SECRET)
+        bucket = oss2.Bucket(auth, OSS_BUCKET_ENDPOINT, "art-pubbucket")
+        response = bucket.put_object_from_file(oss_object_key, src_url)
+
+        if 'Content-Length' in response.headers:
+            return {
+                'status': response.status,
+                'oss_object_key': oss_object_key,
+                'save_oss_timestamp': int(datetime.now().timestamp() * 1000),
+            }
+        raise AssertionError(f'OSS上传失败，请求ID: \n{response.headers["x-oss-request-id"]}')
+
+
+    # 获取视频链接 将视频链接有效时间设置为1天
+    @classmethod
+    def get_oss_url(cls, path):
+        list = []
+        for i in path:
+            # 创建 OSS 客户端
+            auth = oss2.Auth(OSS_ACCESS_KEY_ID, OSS_ACCESS_KEY_SECRET)
+            bucket = oss2.Bucket(auth, OSS_BUCKET_ENDPOINT, OSS_BUCKET_NAME)
+            # 获取指定路径下的对象列表
+            url = bucket.sign_url('GET', i[2], 60 * 60 * 24)  # 设置链接有效期为1天（86400秒）
+            list.append([i[0], i[1], i[2], url])
+        return list

common/common.py

@@ -0,0 +1,44 @@
+# -*- coding: utf-8 -*-
+# @Time: 2023/12/26
+"""
+公共方法，包含：生成log / 删除log / 下载方法 / 删除 weixinzhishu_chlsfiles / 过滤词库 / 保存视频信息至本地 txt / 翻译 / ffmpeg
+"""
+from datetime import date, timedelta
+from datetime import datetime
+from loguru import logger
+import os
+
+proxies = {"http": None, "https": None}
+
+
+class Common:
+    # 统一获取当前时间 <class 'datetime.datetime'>  2022-04-14 20:13:51.244472
+    now = datetime.now()
+    # 昨天 <class 'str'>  2022-04-13
+    yesterday = (date.today() + timedelta(days=-1)).strftime("%Y-%m-%d")
+    # 今天 <class 'datetime.date'>  2022-04-14
+    today = date.today()
+    # 明天 <class 'str'>  2022-04-15
+    tomorrow = (date.today() + timedelta(days=1)).strftime("%Y-%m-%d")
+
+    # 使用 logger 模块生成日志
+    @staticmethod
+    def logger():
+        """
+        使用 logger 模块生成日志
+        """
+        # 日志路径
+        log_dir = f"./logs/"
+        log_path = os.getcwd() + os.sep + log_dir
+        if not os.path.isdir(log_path):
+            os.makedirs(log_path)
+        # 日志文件名
+        log_name = f"{datetime.now().date().strftime('%Y-%m-%d')}.log"
+        # 日志不打印到控制台
+        logger.remove(handler_id=None)
+        # 初始化日志
+        logger.add(os.path.join(log_dir, log_name), level="INFO", rotation="00:00", retention="10 days", enqueue=True)
+
+        return logger
+
+

common/db.py

@@ -0,0 +1,148 @@
+# -*- coding: utf-8 -*-
+# @Time: 2023/12/26
+"""
+数据库连接及操作
+"""
+import redis
+import pymysql
+from common.common import Common
+# from common import Common
+
+class MysqlHelper:
+    @classmethod
+    def connect_mysql(cls, env, machine):
+        if machine == 'aliyun_hk':
+            # 创建一个 Connection 对象，代表了一个数据库连接
+            connection = pymysql.connect(
+                host="rm-j6cz4c6pt96000xi3.mysql.rds.aliyuncs.com",# 数据库IP地址，内网地址
+                # host="rm-j6cz4c6pt96000xi3lo.mysql.rds.aliyuncs.com",# 数据库IP地址，外网地址
+                port=3306,                      # 端口号
+                user="crawler",                 #  mysql用户名
+                passwd="crawler123456@",        # mysql用户登录密码
+                db="piaoquan-crawler" ,         # 数据库名
+                # 如果数据库里面的文本是utf8编码的，charset指定是utf8
+                charset = "utf8")
+        elif env == 'prod':
+            # 创建一个 Connection 对象，代表了一个数据库连接
+            connection = pymysql.connect(
+                host="rm-bp1159bu17li9hi94.mysql.rds.aliyuncs.com",# 数据库IP地址，内网地址
+                # host="rm-bp1159bu17li9hi94ro.mysql.rds.aliyuncs.com",# 数据库IP地址，外网地址
+                port=3306,                      # 端口号
+                user="crawler",                 #  mysql用户名
+                passwd="crawler123456@",        # mysql用户登录密码
+                db="piaoquan-crawler" ,         # 数据库名
+                # 如果数据库里面的文本是utf8编码的，charset指定是utf8
+                charset = "utf8")
+        else:
+            # 创建一个 Connection 对象，代表了一个数据库连接
+            connection = pymysql.connect(
+                host="rm-bp1k5853td1r25g3n690.mysql.rds.aliyuncs.com",# 数据库IP地址，内网地址
+                # host="rm-bp1k5853td1r25g3ndo.mysql.rds.aliyuncs.com",  # 数据库IP地址，外网地址
+                port=3306,  # 端口号
+                user="crawler",  # mysql用户名
+                passwd="crawler123456@",  # mysql用户登录密码
+                db="piaoquan-crawler",  # 数据库名
+                # 如果数据库里面的文本是utf8编码的，charset指定是utf8
+                charset="utf8")
+
+        return connection
+
+    @classmethod
+    def get_values(cls, sql, env):
+        try:
+            machine = ""
+            # 连接数据库
+            connect = cls.connect_mysql(env, machine)
+            # 返回一个 Cursor对象
+            mysql = connect.cursor()
+
+            # 执行 sql 语句
+            mysql.execute(sql)
+
+            # fetchall方法返回的是一个元组，里面每个元素也是元组，代表一行记录
+            data = mysql.fetchall()
+
+            # 关闭数据库连接
+            connect.close()
+
+            # 返回查询结果，元组
+            return data
+        except Exception as e:
+            print(f"get_values异常:{e}\n")
+            # Common.logger(log_type, crawler).error(f"get_values异常:{e}\n")
+
+    @classmethod
+    def update_values(cls, sql, env, machine):
+        # 连接数据库
+        connect = cls.connect_mysql(env, machine)
+        # 返回一个 Cursor对象
+        mysql = connect.cursor()
+
+        try:
+            # 执行 sql 语句
+            res = mysql.execute(sql)
+            # 注意 一定要commit，否则添加数据不生效
+            connect.commit()
+            return res
+        except Exception as e:
+            # print(f"update_values异常，进行回滚操作:{e}\n")
+            # 发生错误时回滚
+            connect.rollback()
+
+        # 关闭数据库连接
+        connect.close()
+
+class RedisHelper:
+    @classmethod
+    def connect_redis(cls, env, machine):
+        if machine == 'aliyun_hk':
+            redis_pool = redis.ConnectionPool(
+                # host='r-bp154bpw97gptefiqk.redis.rds.aliyuncs.com',  # 内网地址
+                host='r-bp154bpw97gptefiqkpd.redis.rds.aliyuncs.com',  # 外网地址
+                port=6379,
+                db=2,
+                password='Qingqu2019'
+            )
+            redis_conn = redis.Redis(connection_pool=redis_pool)
+        elif env == 'prod':
+            redis_pool = redis.ConnectionPool(
+                host='r-bp1mb0v08fqi4hjffu.redis.rds.aliyuncs.com',  # 内网地址
+                # host='r-bp1mb0v08fqi4hjffupd.redis.rds.aliyuncs.com',  # 外网地址
+                port=6379,
+                db=2,
+                password='Qingqu2019'
+            )
+            redis_conn = redis.Redis(connection_pool=redis_pool)
+        else:
+            redis_pool = redis.ConnectionPool(
+                # host='r-bp154bpw97gptefiqk.redis.rds.aliyuncs.com',  # 内网地址
+                host='r-bp154bpw97gptefiqkpd.redis.rds.aliyuncs.com',  # 外网地址
+                port=6379,
+                db=2,
+                password='Qingqu2019'
+            )
+            redis_conn = redis.Redis(connection_pool=redis_pool)
+        return redis_conn
+
+    @classmethod
+    def redis_push(cls, env, machine, data):
+        redis_conn = cls.connect_redis(env, machine)
+        redis_conn.lpush(machine, data)
+
+    @classmethod
+    def redis_pop(cls, env, machine):
+        redis_conn = cls.connect_redis(env, machine)
+        if redis_conn.llen(machine) == 0:
+            return None
+        else:
+            return redis_conn.rpop(machine)
+
+
+
+if __name__ == "__main__":
+
+    print(RedisHelper.connect_redis("prod", "aliyun"))
+
+
+    pass
+

common/feishu.py

@@ -0,0 +1,379 @@
+# -*- coding: utf-8 -*-
+# @Time: 2023/12/26
+"""
+飞书表配置: token 鉴权 / 增删改查 / 机器人报警
+"""
+import json
+import os
+import sys
+import requests
+import urllib3
+
+sys.path.append(os.getcwd())
+from common.common import Common
+
+proxies = {"http": None, "https": None}
+
+
+class Feishu:
+    """
+    编辑飞书云文档
+    """
+    succinct_url = "https://w42nne6hzg.feishu.cn/sheets/CPDNs06R2hux6SthZ1wcQmkAnYg?"
+
+    # 飞书路径token
+    @classmethod
+    def spreadsheettoken(cls, crawler):
+        if crawler == "succinct":
+            return "CPDNs06R2hux6SthZ1wcQmkAnYg"
+
+    # 获取飞书api token
+    @classmethod
+    def get_token(cls, log_type, crawler, action=""):
+        """
+        获取飞书api token
+        :return:
+        """
+        url = "https://open.feishu.cn/open-apis/auth/v3/tenant_access_token/internal/"
+        post_data = {"app_id": "cli_a13ad2afa438d00b",  # 这里账号密码是发布应用的后台账号及密码
+                     "app_secret": "4tK9LY9VbiQlY5umhE42dclBFo6t4p5O"}
+
+        try:
+            urllib3.disable_warnings()
+            response = requests.post(url=url, data=post_data, proxies=proxies, verify=False)
+            tenant_access_token = response.json()["tenant_access_token"]
+            return tenant_access_token
+        except Exception as e:
+            Common.logger().error("获取飞书 api token 异常:{}", e)
+
+    # 获取表格元数据
+    @classmethod
+    def get_metainfo(cls, log_type, crawler):
+        """
+        获取表格元数据
+        :return:
+        """
+        try:
+            get_metainfo_url = "https://open.feishu.cn/open-apis/sheets/v2/spreadsheets/" \
+                               + cls.spreadsheettoken(crawler) + "/metainfo"
+
+            headers = {
+                "Authorization": "Bearer " + cls.get_token(log_type, crawler),
+                "Content-Type": "application/json; charset=utf-8"
+            }
+            params = {
+                "extFields": "protectedRange",  # 额外返回的字段，extFields=protectedRange时返回保护行列信息
+                "user_id_type": "open_id"  # 返回的用户id类型，可选open_id,union_id
+            }
+            urllib3.disable_warnings()
+            r = requests.get(url=get_metainfo_url, headers=headers, params=params, proxies=proxies, verify=False)
+            response = json.loads(r.content.decode("utf8"))
+            return response
+        except Exception as e:
+            Common.logger().error("获取表格元数据异常:{}", e)
+
+    # 读取工作表中所有数据
+    @classmethod
+    def get_values_batch(cls, log_type, crawler, sheetid):
+        """
+        读取工作表中所有数据
+        :param log_type: 启用哪个 log
+        :param crawler: 哪个爬虫
+        :param sheetid: 哪张表
+        :return: 所有数据
+        """
+        try:
+            get_values_batch_url = "https://open.feishu.cn/open-apis/sheets/v2/spreadsheets/" \
+                                   + cls.spreadsheettoken(crawler) + "/values_batch_get"
+            headers = {
+                "Authorization": "Bearer " + cls.get_token(log_type, crawler),
+                "Content-Type": "application/json; charset=utf-8"
+            }
+            params = {
+                # 多个查询范围 如 url?ranges=range1,range2 ，其中 range 包含 sheetId 与单元格范围两部分
+                "ranges": sheetid,
+
+                # valueRenderOption=ToString 可返回纯文本的值(数值类型除外)；
+                # valueRenderOption=FormattedValue 计算并格式化单元格；
+                # valueRenderOption=Formula单元格中含有公式时返回公式本身；
+                # valueRenderOption=UnformattedValue计算但不对单元格进行格式化
+                "valueRenderOption": "ToString",
+
+                # dateTimeRenderOption=FormattedString 计算并将时间日期按照其格式进行格式化，但不会对数字进行格式化，返回格式化后的字符串。
+                "dateTimeRenderOption": "",
+
+                # 返回的用户id类型，可选open_id,union_id
+                "user_id_type": "open_id"
+            }
+            urllib3.disable_warnings()
+            r = requests.get(url=get_values_batch_url, headers=headers, params=params, proxies=proxies, verify=False)
+            # print(r.text)
+            response = json.loads(r.content.decode("utf8"))
+            values = response["data"]["valueRanges"][0]["values"]
+            return values
+        except Exception as e:
+            Common.logger().error("读取工作表所有数据异常:{}", e)
+
+    # 工作表，插入行或列
+    @classmethod
+    def insert_columns(cls, log_type, crawler, sheetid, majordimension, startindex, endindex):
+        """
+        工作表插入行或列
+        :param log_type: 日志路径
+        :param crawler: 哪个爬虫的云文档
+        :param sheetid:哪张工作表
+        :param majordimension:行或者列, ROWS、COLUMNS
+        :param startindex:开始位置
+        :param endindex:结束位置
+        """
+        try:
+            insert_columns_url = "https://open.feishu.cn/open-apis/sheets/v2/spreadsheets/" \
+                                 + cls.spreadsheettoken(crawler) + "/insert_dimension_range"
+            headers = {
+                "Authorization": "Bearer " + cls.get_token(log_type, crawler),
+                "Content-Type": "application/json; charset=utf-8"
+            }
+            body = {
+                "dimension": {
+                    "sheetId": sheetid,
+                    "majorDimension": majordimension,  # 默认 ROWS ，可选 ROWS、COLUMNS
+                    "startIndex": startindex,  # 开始的位置
+                    "endIndex": endindex  # 结束的位置
+                },
+                "inheritStyle": "AFTER"  # BEFORE 或 AFTER，不填为不继承 style
+            }
+
+            urllib3.disable_warnings()
+            r = requests.post(url=insert_columns_url, headers=headers, json=body, proxies=proxies, verify=False)
+            Common.logger().info("插入行或列:{}", r.json()["msg"])
+        except Exception as e:
+            Common.logger().error("插入行或列异常:{}", e)
+
+    # 写入数据
+    @classmethod
+    def update_values(cls, log_type, crawler, sheetid, ranges, values):
+        """
+        写入数据
+        :param log_type: 日志路径
+        :param crawler: 哪个爬虫的云文档
+        :param sheetid:哪张工作表
+        :param ranges:单元格范围
+        :param values:写入的具体数据，list
+        """
+        try:
+            update_values_url = "https://open.feishu.cn/open-apis/sheets/v2/spreadsheets/" \
+                                + cls.spreadsheettoken(crawler) + "/values_batch_update"
+            headers = {
+                "Authorization": "Bearer " + cls.get_token(log_type, crawler),
+                "Content-Type": "application/json; charset=utf-8"
+            }
+            body = {
+                "valueRanges": [
+                    {
+                        "range": sheetid + "!" + ranges,
+                        "values": values
+                    },
+                ],
+            }
+            urllib3.disable_warnings()
+            r = requests.post(url=update_values_url, headers=headers, json=body, proxies=proxies, verify=False)
+            Common.logger().info("写入数据:{}", r.json()["msg"])
+        except Exception as e:
+            Common.logger().error("写入数据异常:{}", e)
+
+    # 合并单元格
+    @classmethod
+    def merge_cells(cls, log_type, crawler, sheetid, ranges):
+        """
+        合并单元格
+        :param log_type: 日志路径
+        :param crawler: 哪个爬虫
+        :param sheetid:哪张工作表
+        :param ranges:需要合并的单元格范围
+        """
+        try:
+            merge_cells_url = "https://open.feishu.cn/open-apis/sheets/v2/spreadsheets/" \
+                              + cls.spreadsheettoken(crawler) + "/merge_cells"
+            headers = {
+                "Authorization": "Bearer " + cls.get_token(log_type, crawler),
+                "Content-Type": "application/json; charset=utf-8"
+            }
+
+            body = {
+                "range": sheetid + "!" + ranges,
+                "mergeType": "MERGE_ROWS"
+            }
+            urllib3.disable_warnings()
+            r = requests.post(url=merge_cells_url, headers=headers, json=body, proxies=proxies, verify=False)
+            Common.logger().info("合并单元格:{}", r.json()["msg"])
+        except Exception as e:
+            Common.logger().error("合并单元格异常:{}", e)
+
+    # 读取单元格数据
+    @classmethod
+    def get_range_value(cls, log_type, crawler, sheetid, cell):
+        """
+        读取单元格内容
+        :param log_type: 日志路径
+        :param crawler: 哪个爬虫
+        :param sheetid: 哪张工作表
+        :param cell: 哪个单元格
+        :return: 单元格内容
+        """
+        try:
+            get_range_value_url = "https://open.feishu.cn/open-apis/sheets/v2/spreadsheets/" \
+                                  + cls.spreadsheettoken(crawler) + "/values/" + sheetid + "!" + cell
+            headers = {
+                "Authorization": "Bearer " + cls.get_token(log_type, crawler),
+                "Content-Type": "application/json; charset=utf-8"
+            }
+            params = {
+                "valueRenderOption": "FormattedValue",
+
+                # dateTimeRenderOption=FormattedString 计算并对时间日期按照其格式进行格式化，但不会对数字进行格式化，返回格式化后的字符串。
+                "dateTimeRenderOption": "",
+
+                # 返回的用户id类型，可选open_id,union_id
+                "user_id_type": "open_id"
+            }
+            urllib3.disable_warnings()
+            r = requests.get(url=get_range_value_url, headers=headers, params=params, proxies=proxies, verify=False)
+            # print(r.text)
+            return r.json()["data"]["valueRange"]["values"][0]
+        except Exception as e:
+            Common.logger().error("读取单元格数据异常:{}", e)
+    # 获取表内容
+    @classmethod
+    def get_sheet_content(cls, log_type, crawler, sheet_id):
+        try:
+            sheet = Feishu.get_values_batch(log_type, crawler, sheet_id)
+            content_list = []
+            for x in sheet:
+                for y in x:
+                    if y is None:
+                        pass
+                    else:
+                        content_list.append(y)
+            return content_list
+        except Exception as e:
+            Common.logger().error(f'get_sheet_content:{e}\n')
+
+    # 删除行或列，可选 ROWS、COLUMNS
+    @classmethod
+    def dimension_range(cls, log_type, crawler, sheetid, major_dimension, startindex, endindex):
+        """
+        删除行或列
+        :param log_type: 日志路径
+        :param crawler: 哪个爬虫
+        :param sheetid:工作表
+        :param major_dimension:默认 ROWS ，可选 ROWS、COLUMNS
+        :param startindex:开始的位置
+        :param endindex:结束的位置
+        :return:
+        """
+        try:
+            dimension_range_url = "https://open.feishu.cn/open-apis/sheets/v2/spreadsheets/" \
+                                  + cls.spreadsheettoken(crawler) + "/dimension_range"
+            headers = {
+                "Authorization": "Bearer " + cls.get_token(log_type, crawler),
+                "Content-Type": "application/json; charset=utf-8"
+            }
+            body = {
+                "dimension": {
+                    "sheetId": sheetid,
+                    "majorDimension": major_dimension,
+                    "startIndex": startindex,
+                    "endIndex": endindex
+                }
+            }
+            urllib3.disable_warnings()
+            r = requests.delete(url=dimension_range_url, headers=headers, json=body, proxies=proxies, verify=False)
+            Common.logger().info("删除视频数据:{}", r.json()["msg"])
+        except Exception as e:
+            Common.logger().error("删除视频数据异常:{}", e)
+
+    # 获取用户 ID
+    @classmethod
+    def get_userid(cls, log_type, crawler, username):
+        try:
+            url = "https://open.feishu.cn/open-apis/user/v1/batch_get_id?"
+            headers = {
+                "Authorization": "Bearer " + cls.get_token(log_type, crawler),
+                "Content-Type": "application/json; charset=utf-8"
+            }
+            name_phone_dict = {
+                "xinxin": "15546206651",
+                "muxinyi": "13699208058",
+                "wangxueke": "13513479926",
+                "yuzhuoyi": "18624010360",
+                "luojunhui": "18801281360",
+                "fanjun": "15200827642",
+                "zhangyong": "17600025055"
+            }
+            username = name_phone_dict.get(username)
+
+            data = {"mobiles": [username]}
+            urllib3.disable_warnings()
+            r = requests.get(url=url, headers=headers, params=data, verify=False, proxies=proxies)
+            open_id = r.json()["data"]["mobile_users"][username][0]["open_id"]
+
+            return open_id
+        except Exception as e:
+            Common.logger().error(f"get_userid异常:{e}\n")
+
+    # 飞书机器人
+    @classmethod
+    def bot(cls, log_type, crawler, text):
+        try:
+            url = "https://open.feishu.cn/open-apis/bot/v2/hook/96989577-50e7-4653-9ec2-308fe3f2c5fe"
+            headers = {'Content-Type': 'application/json'}
+            if crawler == "抖音":
+                content = "抖音cookie过期"
+                sheet_url = "https://w42nne6hzg.feishu.cn/sheets/CPDNs06R2hux6SthZ1wcQmkAnYg?sheet=OpE35G"
+                users = "\n<at id=" + str(cls.get_userid(log_type, crawler, "wangxueke")) + "></at> <at id=" + str(
+                    cls.get_userid(log_type, crawler, "muxinyi")) + "></at>\n"
+
+            data = json.dumps({
+                "msg_type": "interactive",
+                "card": {
+                    "config": {
+                        "wide_screen_mode": True,
+                        "enable_forward": True
+                    },
+                    "elements": [{
+                        "tag": "div",
+                        "text": {
+                            "content": users + text,
+                            "tag": "lark_md"
+                        }
+                    }, {
+                        "actions": [{
+                            "tag": "button",
+                            "text": {
+                                "content": content,
+                                "tag": "lark_md"
+                            },
+                            "url": sheet_url,
+                            "type": "default",
+                            "value": {}
+                        }],
+                        "tag": "action"
+                    }],
+                    "header": {
+                        "title": {
+                            "content": "📣您有新的信息，请注意查收",
+                            "tag": "plain_text"
+                        }
+                    }
+                }
+            })
+            urllib3.disable_warnings()
+            r = requests.post(url, headers=headers, data=data, verify=False, proxies=proxies)
+            Common.logger().info(f'触发机器人消息:{r.status_code}, {text}')
+        except Exception as e:
+            Common.logger().error(f"bot异常:{e}\n")
+
+
+if __name__ == "__main__":
+    Feishu.bot('recommend', '抖音', '测试: 抖音cookie失效，请及时更换')
+

common/material.py

@@ -0,0 +1,74 @@
+# -*- coding: utf-8 -*-
+# @Time: 2023/12/26
+import os
+import random
+import sys
+
+sys.path.append(os.getcwd())
+
+from common.db import MysqlHelper
+from common.feishu import Feishu
+
+
+class Material():
+
+    # 获取视频链接 存入数据库
+    @classmethod
+    def insert_user(cls):
+        # 获取抖音视频链接
+        douyin = Feishu.get_values_batch("prod", "succinct", "iYbVis")
+        # 提取账号昵称和账号主页链接
+        channel = '抖音'
+        for row in douyin[2:]:
+            platform = row[0]
+            if platform == channel:
+                account_name = row[2]
+                account_link = row[3]
+                user_id = account_link.split("user/")[1]
+                insert_sql = f"""INSERT INTO video_user_id (name, user_id, channel) values ("{account_name}", "{user_id}", "{channel}")"""
+                MysqlHelper.update_values(
+                    sql=insert_sql,
+                    env="prod",
+                    machine="",
+                )
+
+    # 随机获取标题
+    @classmethod
+    def get_title(cls):
+        title = Feishu.get_values_batch("prod", "succinct", "meGnsz")
+        random_item = random.choice(title)
+        cleaned_item = random_item[0].strip("[]'")
+        return cleaned_item
+
+    # 获取所有音频
+    @classmethod
+    def get_audio(cls):
+        audio = Feishu.get_values_batch("prod", "succinct", "zucQUM")
+        list = []
+        for row in audio[1:]:
+            account_name = row[0]
+            list.append(account_name)
+            srt = row[2]
+            if srt == None:
+                continue
+            file_path = f"video/{account_name}.srt"
+            if os.path.isfile(file_path):
+                continue
+            else:
+                with open(file_path, 'w') as file:
+                    # 写入内容到文件中
+                    file.write(srt)
+
+        return list
+
+    # 获取抖音 cookie
+    @classmethod
+    def get_douyin_cookie(cls):
+        douyin_token = Feishu.get_values_batch("prod", "succinct", "OpE35G")
+        for item in douyin_token:
+            if item[0] == '抖音':
+                return item[1]
+
+
+
+

common/scheduling_db.py

@@ -0,0 +1,157 @@
+# -*- coding: utf-8 -*-
+# @Time: 2023/12/26
+"""
+数据库连接及操作
+"""
+import redis
+import pymysql
+from common.common import Common
+# from common import Common
+
+
+class MysqlHelper:
+    @classmethod
+    def connect_mysql(cls, env, action):
+        if env == 'hk':
+            if action == 'get_author_map':
+                # 创建一个 Connection 对象，代表了一个数据库连接
+                connection = pymysql.connect(
+                    host="rm-bp1159bu17li9hi94ro.mysql.rds.aliyuncs.com",  # 数据库IP地址，内网地址
+                    port=3306,  # 端口号
+                    user="crawler",  # mysql用户名
+                    passwd="crawler123456@",  # mysql用户登录密码
+                    db="piaoquan-crawler",  # 数据库名
+                    # 如果数据库里面的文本是utf8编码的，charset指定是utf8
+                    charset="utf8mb4")
+            else:
+                # 创建一个 Connection 对象，代表了一个数据库连接
+                connection = pymysql.connect(
+                    host="rm-j6cz4c6pt96000xi3.mysql.rds.aliyuncs.com",  # 数据库IP地址，内网地址
+                    # host="rm-j6cz4c6pt96000xi3lo.mysql.rds.aliyuncs.com",# 数据库IP地址，外网地址
+                    port=3306,  # 端口号
+                    user="crawler",  # mysql用户名
+                    passwd="crawler123456@",  # mysql用户登录密码
+                    db="piaoquan-crawler",  # 数据库名
+                    # 如果数据库里面的文本是utf8编码的，charset指定是utf8
+                    charset="utf8mb4")
+        elif env == 'prod':
+            # 创建一个 Connection 对象，代表了一个数据库连接
+            connection = pymysql.connect(
+                host="rm-bp1159bu17li9hi94.mysql.rds.aliyuncs.com",  # 数据库IP地址，内网地址
+                # host="rm-bp1159bu17li9hi94ro.mysql.rds.aliyuncs.com",# 数据库IP地址，外网地址
+                port=3306,  # 端口号
+                user="crawler",  # mysql用户名
+                passwd="crawler123456@",  # mysql用户登录密码
+                db="piaoquan-crawler",  # 数据库名
+                # 如果数据库里面的文本是utf8编码的，charset指定是utf8
+                charset="utf8mb4")
+        else:
+            # 创建一个 Connection 对象，代表了一个数据库连接
+            connection = pymysql.connect(
+                host="rm-bp1k5853td1r25g3n690.mysql.rds.aliyuncs.com",  # 数据库IP地址，内网地址
+                # host="rm-bp1k5853td1r25g3ndo.mysql.rds.aliyuncs.com",  # 数据库IP地址，外网地址
+                port=3306,  # 端口号
+                user="crawler",  # mysql用户名
+                passwd="crawler123456@",  # mysql用户登录密码
+                db="piaoquan-crawler",  # 数据库名
+                # 如果数据库里面的文本是utf8编码的，charset指定是utf8
+                charset="utf8mb4")
+
+        return connection
+
+    @classmethod
+    def get_values(cls, log_type, crawler, sql, env, action=''):
+        try:
+            # 连接数据库
+            connect = cls.connect_mysql(env, action)
+            # 返回一个 Cursor对象
+            mysql = connect.cursor(cursor=pymysql.cursors.DictCursor)
+
+            # 执行 sql 语句
+            mysql.execute(sql)
+
+            # fetchall方法返回的是一个元组，里面每个元素也是元组，代表一行记录
+            data = mysql.fetchall()
+
+            # 关闭数据库连接
+            connect.close()
+
+            # 返回查询结果，元组
+            return data
+        except Exception as e:
+            Common.logger(log_type, crawler).error(f"get_values异常:{e}\n")
+
+    @classmethod
+    def update_values(cls, log_type, crawler, sql, env, action=''):
+        # 连接数据库
+        connect = cls.connect_mysql(env, action)
+        # 返回一个 Cursor对象
+        mysql = connect.cursor()
+
+        try:
+            # 执行 sql 语句
+            res = mysql.execute(sql)
+            # 注意 一定要commit，否则添加数据不生效
+            connect.commit()
+            return res
+        except Exception as e:
+            Common.logger(log_type, crawler).error(f"update_values异常，进行回滚操作:{e}\n")
+            # 发生错误时回滚
+            connect.rollback()
+
+        # 关闭数据库连接
+        connect.close()
+
+
+class RedisHelper:
+    @classmethod
+    def connect_redis(cls, env):
+        if env == 'hk':
+            redis_pool = redis.ConnectionPool(
+                # host='r-bp154bpw97gptefiqk.redis.rds.aliyuncs.com',  # 内网地址
+                # host='r-bp154bpw97gptefiqkpd.redis.rds.aliyuncs.com',  # 测试地址
+                host='r-bp1mb0v08fqi4hjffupd.redis.rds.aliyuncs.com',  # 外网地址
+                port=6379,
+                db=2,
+                password='Wqsd@2019'
+            )
+            redis_conn = redis.Redis(connection_pool=redis_pool)
+        elif env == 'prod':
+            redis_pool = redis.ConnectionPool(
+                host='r-bp1mb0v08fqi4hjffu.redis.rds.aliyuncs.com',  # 内网地址
+                # host='r-bp1mb0v08fqi4hjffupd.redis.rds.aliyuncs.com',  # 外网地址
+                port=6379,
+                db=2,
+                password='Wqsd@2019'
+            )
+            redis_conn = redis.Redis(connection_pool=redis_pool)
+        else:
+            redis_pool = redis.ConnectionPool(
+                # host='r-bp154bpw97gptefiqk.redis.rds.aliyuncs.com',  # 内网地址
+                host='r-bp154bpw97gptefiqkpd.redis.rds.aliyuncs.com',  # 外网地址
+                port=6379,
+                db=2,
+                password='Qingqu2019'
+            )
+            redis_conn = redis.Redis(connection_pool=redis_pool)
+        return redis_conn
+
+    @classmethod
+    def redis_push(cls, env, task_key, data):
+        redis_conn = cls.connect_redis(env)
+        # print("开始写入数据")
+        redis_conn.lpush(task_key, data)
+        # print("数据写入完成")
+
+    @classmethod
+    def redis_pop(cls, env, task_key):
+        redis_conn = cls.connect_redis(env)
+        if redis_conn.llen(task_key) == 0:
+            return None
+        else:
+            return redis_conn.rpop(task_key)
+
+
+if __name__ == "__main__":
+    print(RedisHelper.redis_pop("dev", "local"))
+    pass

video_capture/douyin/douyin_author/__init__.py

@@ -0,0 +1,2 @@
+# -*- coding: utf-8 -*-
+# @Time: 2023/12/25

video_capture/douyin/douyin_author/douyin_author.py

@@ -0,0 +1,157 @@
+# -*- coding: utf-8 -*-
+# @Time: 2023/12/22
+import os
+import random
+import sys
+import time
+import requests
+import json
+import urllib3
+sys.path.append(os.getcwd())
+from common.aliyun_oss_uploading import Oss
+from common.common import Common
+from common.material import Material
+from common.feishu import Feishu
+
+from common.db import MysqlHelper
+from requests.adapters import HTTPAdapter
+
+from video_capture.douyin.douyin_author.douyin_author_help import DouYinHelper
+
+class douyinAuthor():
+    """
+    获取用户主页id
+    """
+    @classmethod
+    def get_videoUserId(cls):
+        select_user_sql = f"""select user_id from video_user_id;"""
+        user_list = MysqlHelper.get_values(select_user_sql, "prod")
+        return user_list
+
+    """
+    oss视频地址 存入数据库
+    """
+    @classmethod
+    def insert_videoUrl(cls, video_id, account_id, oss_object_key):
+        insert_sql = f"""INSERT INTO video_url (video_id, account_id, oss_object_key) values ("{video_id}", "{account_id}", "{oss_object_key}")"""
+        MysqlHelper.update_values(
+            sql=insert_sql,
+            env="prod",
+            machine="",
+        )
+
+    """
+    查询该video_id是否在数据库存在
+    """
+    @classmethod
+    def select_videoUrl_id(cls, video_id):
+        select_user_sql = f"""select video_id from video_url where video_id={video_id} ;"""
+        user_list = MysqlHelper.get_values(select_user_sql, "prod")
+        if user_list:
+            return True
+        else:
+            return False
+
+
+    """抖音读取数据 将数据存储到oss上"""
+    @classmethod
+    def get_videoList(cls, cookie):
+        # 读取飞书表格,更新数据库用户主页id
+        Material.insert_user()
+        # 获取 用户主页id
+        user_list = cls.get_videoUserId()
+        if len(user_list) == 0:
+            return
+        for i in user_list:
+            account_id = i[0].replace('(', '').replace(')', '').replace(',', '')
+            Common.logger().info(f"用户主页ID：{account_id}")
+            next_cursor = 0
+            while True:
+                if next_cursor == None:
+                    break
+                time.sleep(random.randint(5, 10))
+                url = 'https://www.douyin.com/aweme/v1/web/aweme/post/'
+                headers = {
+                    'Accept': 'application/json, text/plain, */*',
+                    'Accept-Language': 'zh-CN,zh;q=0.9',
+                    'Cache-Control': 'no-cache',
+                    'Cookie': cookie,
+                    'Pragma': 'no-cache',
+                    'Referer': f'https://www.douyin.com/user/{account_id}',
+                    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) '
+                                  'Chrome/118.0.0.0 Safari/537.36',
+                }
+                query = DouYinHelper.get_full_query(ua=headers['User-Agent'], extra_data={
+                    'sec_user_id': account_id,
+                    'max_cursor': next_cursor,
+                    'locate_query': 'false',
+                    'show_live_replay_strategy': '1',
+                    'need_time_list': '1',
+                    'time_list_query': '0',
+                    'whale_cut_token': '',
+                    'cut_version': '1',
+                    'count': '18',
+                    'publish_video_strategy_type': '2',
+                })
+                urllib3.disable_warnings()
+                s = requests.session()
+                s.mount('http://', HTTPAdapter(max_retries=3))
+                s.mount('https://', HTTPAdapter(max_retries=3))
+                response = requests.request(method='GET', url=url, headers=headers, params=query)
+                body = response.content.decode()
+                obj = json.loads(body)
+
+                has_more = True if obj.get('has_more', 0) == 1 else False
+                next_cursor = str(obj.get('max_cursor')) if has_more else None
+                data = obj.get('aweme_list', [])
+                response.close()
+                if response.status_code != 200:
+                    Common.logger().info(
+                        f"接口请求失败，，请更换cookie，{response.status_code}")
+                    Feishu.bot('recommend', '抖音', '抖音cookie失效，请及时更换～')
+                    # 如果返回空信息，则随机睡眠 600, 1200 秒
+                    time.sleep(random.randint(600, 1200))
+                    # 重新获取cookie
+                    cookie = Material.get_douyin_cookie()
+                    douyinAuthor.get_videoList(cls, cookie)
+                elif len(data) == 0:
+                    Common.logger().info(
+                        f"接口请求失败，请更换cookie")
+                    Feishu.bot('recommend', '抖音', '抖音cookie失效，请及时更换～')
+                    # 如果返回空信息，则随机睡眠 600, 1200 秒
+                    time.sleep(random.randint(600, 1200))
+                    # 重新获取cookie
+                    cookie = Material.get_douyin_cookie()
+                    douyinAuthor.get_videoList(cls, cookie)
+                count = 0
+                for i in range(len(data)):
+                    try:
+                        entity_type = data[i].get('search_impr').get('entity_type')
+                        if entity_type == 'GENERAL':
+                            count += 1
+                            video_id = data[i].get('aweme_id')  # 文章id
+                            id = cls.select_videoUrl_id(video_id)
+                            if id:
+                                if count > 5:
+                                    Common.logger().info(
+                                        f"重复视频不在抓取该用户，用户主页id：{account_id}")
+                                    break
+                                continue
+                            video_url = data[i].get('video').get('play_addr').get('url_list')[0]  # 视频链接
+                            oss_object_key = Oss.video_sync_upload_oss(video_url, video_id, account_id)
+                            status = oss_object_key.get("status")
+                            # 发送 oss
+                            oss_object_key = oss_object_key.get("oss_object_key")
+                            Common.logger().info(f"抖音视频链接oss发送成功，oss地址：{oss_object_key}")
+                            # 发送成功 存入数据库
+                            if status == 200:
+                                cls.insert_videoUrl(video_id, account_id, oss_object_key)
+                                Common.logger().info(f"视频地址插入数据库成功，视频id：{video_id},用户主页id：{account_id},视频储存地址：{oss_object_key}")
+                    except Exception as e:
+                        Common.logger().warning(f"抓取单条视频异常:{e}\n")
+                        continue
+
+    @classmethod
+    def get_token(cls):
+        cookie = Material.get_douyin_cookie()
+        douyinAuthor.get_videoList(cookie)

video_capture/douyin/douyin_author/douyin_author_help.py

@@ -0,0 +1,140 @@
+# -*- coding: utf-8 -*-
+# @Time: 2023/12/26
+import json
+import time
+from base64 import b64encode
+from functools import reduce
+from hashlib import md5
+from random import choice, randint
+from typing import Any, Dict, List, Optional
+from urllib.parse import urlencode
+
+
+class DouYinHelper(object):
+    ttwid_list = [
+        '1|G3wy_-RdLJnfG5P9zAcP54OM8_nTLZVrJxNi1lPzdmg|1693558867|5e43c47a424e939aaf7193b096e3c6f2274982ee64e9608c99c54d2a43982aca'
+    ]
+
+    @classmethod
+    def _0x30492c(cls, x: bytes, y: bytes, f: Optional[List[int]] = None) -> bytes:
+        """RC4加密, 可以用Crypto.Cipher.ARC4替代"""
+        c = 0
+        d = [i for i in range(256)]
+        for b in range(256):
+            c = (c + d[b] + x[b % len(x)]) % 256
+            e = d[b]
+            d[b] = d[c]
+            d[c] = e
+        t, c = 0, 0
+
+        if not f:
+            f = []
+        for i in range(len(y)):
+            t = (t + 1) % 256
+            c = (c + d[t]) % 256
+            e = d[t]
+            d[t] = d[c]
+            d[c] = e
+            f.append(y[i] ^ d[(d[t] + d[c]) % 256])
+        return bytes(f)
+
+    @classmethod
+    def _0x485470(cls, a: str) -> List[int]:
+        _0x583e81 = [0] * 103
+        for i in range(10):
+            _0x583e81[i + 48] = i
+        for j in range(10, 16):
+            _0x583e81[j + 87] = j
+
+        b = len(a) >> 1
+        e = b << 1
+        d = [0] * b
+        c = 0
+        for f in range(0, e, 2):
+            d[c] = _0x583e81[ord(a[f])] << 4 | _0x583e81[ord(a[f + 1])]
+            c += 1
+        return d
+
+    @classmethod
+    def calc_x_bogus(cls, ua: str, query: str, data: Optional[Dict[str, Any]] = None) -> str:
+        """计算X_Bogus参数"""
+        query = query.encode()
+        for _ in range(2):
+            query = md5(query).hexdigest()
+            query = bytes([int(query[i:i + 2], 16) for i in range(0, len(query), 2)])
+
+        data = json.dumps(data, separators=(',', ':'), ensure_ascii=False).encode() if data else b''
+        for _ in range(2):
+            data = md5(data).hexdigest()
+            data = bytes([int(data[i:i + 2], 16) for i in range(0, len(data), 2)])
+
+        a = b'\x00\x01\x0e'
+        ua = b64encode(cls._0x30492c(a, ua.encode())).decode()
+        ua = md5(ua.encode()).hexdigest()
+        ua = cls._0x485470(ua)
+
+        t = int(time.time())
+        fp = 2421646185  # 真实的canvas指纹
+        arr1 = [
+            64,
+            1 / 256,
+            1 % 256,
+            14,
+            query[14],
+            query[15],
+            data[14],
+            data[15],
+            ua[14],
+            ua[15],
+            t >> 24 & 255,
+            t >> 16 & 255,
+            t >> 8 & 255,
+            t >> 0 & 255,
+            fp >> 24 & 255,
+            fp >> 16 & 255,
+            fp >> 8 & 255,
+            fp >> 0 & 255,
+        ]
+        reduce_num = reduce(lambda x, y: int(x) ^ int(y), arr1)
+        arr1.append(reduce_num)
+        arr2 = [int(arr1[i]) for i in range(len(arr1))]
+
+        garble = cls._0x30492c(b'\xff', bytes(arr2), [2, 255])
+        m = 'Dkdpgh4ZKsQB80/Mfvw36XI1R25-WUAlEi7NLboqYTOPuzmFjJnryx9HVGcaStCe='
+        xb = ''
+        for i in range(0, len(garble), 3):
+            a, b, c = garble[i], garble[i + 1], garble[i + 2]
+            base_num = c | b << 8 | a << 16
+            c1 = m[(base_num & 16515072) >> 18]
+            c2 = m[(base_num & 258048) >> 12]
+            c3 = m[(base_num & 4032) >> 6]
+            c4 = m[(base_num & 63)]
+            xb += ''.join([c1, c2, c3, c4])
+        return xb
+
+    @classmethod
+    def get_full_query(cls, ua: str, extra_data: Dict[str, Any]) -> Dict[str, Any]:
+        ms_token = b64encode(bytes([randint(0, 255) for _ in range(94)])).decode()
+        ms_token = ms_token.replace('+', '-').replace('/', '_').rstrip('=')
+
+        data = {
+            'device_platform': 'webapp',
+            'aid': '6383',
+            'channel': 'channel_pc_web',
+            'pc_client_type': '1',
+            'version_code': '190500',
+            'version_name': '19.5.0',
+            'cookie_enabled': 'true',
+            'platform': 'PC',
+            'msToken': ms_token,
+        }
+        data.update(extra_data)
+        query = urlencode(data, safe='=')
+        x_bogus = cls.calc_x_bogus(ua=ua, query=query, data=None)
+        data.update({'X-Bogus': x_bogus})
+        return data
+
+    @classmethod
+    def get_cookie(cls):
+        ttwid = choice(cls.ttwid_list)
+        return f'ttwid={ttwid}'

video_stitching/video_stitching.py

@@ -0,0 +1,300 @@
+# -*- coding: utf-8 -*-
+# @Time: 2023/12/26
+import datetime
+import random
+import os
+import sys
+import time
+
+import requests
+import urllib.parse
+
+
+sys.path.append(os.getcwd())
+from common.aliyun_oss_uploading import Oss
+from common.common import Common
+from common.db import MysqlHelper
+from common.material import Material
+from moviepy.editor import VideoFileClip, concatenate_videoclips
+from moviepy import editor
+
+
+output_path = "video/new_video.mp4"
+class VideoStitching():
+    @classmethod
+    def split_text(cls, text, max_length):
+        words = text.split(' ')
+        lines = []
+        current_line = ''
+        for word in words:
+            if len(current_line) + len(word) <= max_length:
+                current_line += word + ' '
+            else:
+                lines.append(current_line.strip())
+                current_line = word + ' '
+        lines.append(current_line.strip())
+        result = ''.join(lines)
+        result = result[:11] + '\n' + result[11:]  # 在第10个字后面增加换行
+        return result
+
+    @classmethod
+    def srt_to_seconds(cls,srt_time):
+        hours, minutes, seconds = map(float, srt_time.replace(',', '.').split(':'))
+        return hours * 3600 + minutes * 60 + seconds
+
+    @classmethod
+    def insert_videoAudio(cls, audio_url, i):
+        for j in audio_url:
+            insert_sql = f"""INSERT INTO video_audio (audio, video_id, account_id, oss_object_key) values ("{i}", "{j[0]}", "{j[1]}", "{j[2]}")"""
+            MysqlHelper.update_values(
+                sql=insert_sql,
+                env="prod",
+                machine="",
+            )
+
+    # 随机生成id
+    @classmethod
+    def random_id(cls):
+        now = datetime.datetime.now()
+        rand_num = random.randint(10000, 99999)
+
+        id = "{}{}".format(now.strftime("%Y%m%d%H%M%S"), rand_num)
+        return id
+
+    @classmethod
+    def get_account_id(cls):
+        account_id = f"""select account_id from video_url group by account_id;"""
+        account_id = MysqlHelper.get_values(account_id, "prod")
+        return account_id
+
+
+    @classmethod
+    def get_url_list(cls, i, account):
+        url_list = f"""SELECT a.video_id,a.account_id,a.oss_object_key FROM video_url a WHERE NOT EXISTS (
+                SELECT video_id
+                FROM video_audio b
+                WHERE a.video_id = b.video_id AND b.audio = "{i}"
+            ) AND a.account_id = '{account}' ;"""
+        url_list = MysqlHelper.get_values(url_list, "prod")
+        return url_list
+
+    # 新生成视频上传到对应账号下
+    @classmethod
+    def insert_piaoquantv(cls, oss_object_key):
+        title = Material.get_title()
+        url = "https://vlogapi.piaoquantv.com/longvideoapi/crawler/video/send"
+        payload = dict(pageSource='vlog-pages/post/post-video-post', videoPath=oss_object_key, width='720',
+                       height='1280', fileExtensions='mp4', viewStatus='1', title=title, careModelStatus='1',
+                       token='f04f58d6e664cbc9902660a1e8d20ce6cd7fdb0f', loginUid='66425096', versionCode='719',
+                       machineCode='weixin_openid_o0w175aZ4FJtqVsA1tcozJDJHdDU', appId='wx89e7eb06478361d7',
+                       clientTimestamp='1703337579331',
+                       machineInfo='{"sdkVersion":"3.2.5","brand":"iPhone","language":"zh_CN","model":"iPhone 12 Pro<iPhone13,3>","platform":"ios","system":"iOS 15.6.1","weChatVersion":"8.0.44","screenHeight":844,"screenWidth":390,"pixelRatio":3,"windowHeight":762,"windowWidth":390,"softVersion":"4.1.719"}',
+                       sessionId='1703337560040-27bfe208-a389-f476-db1d-840681e04b32',
+                       subSessionId='1703337569952-8f56d53c-b36d-760e-8abe-0b4a027cd5bd', senceType='1089',
+                       hotSenceType='1089', id='1050', channel='pq')
+
+        payload['videoPath'] = oss_object_key
+        payload['title'] = title
+
+        data = urllib.parse.urlencode(payload)
+
+        headers = {
+            'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 15_6_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 MicroMessenger/8.0.44(0x18002c2d) NetType/WIFI Language/zh_CN',
+            'Accept-Encoding': 'gzip,compress,br,deflate',
+            'Referer': 'https://servicewechat.com/wx89e7eb06478361d7/726/page-frame.html',
+            'Content-Type': 'application/x-www-form-urlencoded',
+            'Cookie': 'JSESSIONID=A60D96E7A300A25EA05425B069C8B459'
+        }
+
+        response = requests.post(url, data=data, headers=headers)
+        data = response.json()
+        if data["code"] == 0:
+            return True
+        else:
+            return False
+
+
+
+    # 视频拼接
+    @classmethod
+    def concatenate_videos(cls, videos, audio, srt):
+        clips = []
+        total_duration = 0
+        included_videos = []
+        # 提取视频的音频
+        video1 = VideoFileClip(audio)
+        mp3 = video1.audio
+        # 获取音频时长（以秒为单位）
+        duration_limit = mp3.duration
+
+        # 遍历每个视频并计算总时长
+        for i, video in enumerate(videos):
+            clip = VideoFileClip(video[3])
+            clips.append(clip)
+            total_duration += clip.duration
+            if total_duration >= duration_limit:
+                break
+
+        # 如果总时长小于等于目标时长，则不做视频拼接
+        if total_duration <= duration_limit:
+            return ""
+        else:
+            remaining_time = duration_limit
+            final_clips = []
+
+            for clip, video in zip(clips, videos):
+                if remaining_time - clip.duration >= 0:
+                    final_clips.append(clip)
+                    included_videos.append(video)
+                    remaining_time -= clip.duration
+                else:
+                    # 如果剩余时间不足以加入下一个视频，则截断当前视频并返回已包含的URL
+                    final_clips.append(clip.subclip(0, remaining_time))
+                    included_videos.append(video)
+                    break
+            final_clip = concatenate_videoclips(final_clips)
+        final_clip = final_clip.set_audio(mp3)
+        # 统一设置视频分辨率
+        final_width = 480
+        final_height = 720
+        final_clip = final_clip.resize((final_width, final_height))
+        # 设置背景色
+        color_clip = editor.ColorClip(size=(final_width, 120),
+                                      color=(255, 255, 0)).set_duration(duration_limit)
+        final_clip = editor.CompositeVideoClip([final_clip, color_clip.set_position(("center", final_height - 100))])
+        # 处理SRT字幕文件
+        subtitle_file = f"video/{srt}.srt"
+        if os.path.isfile(subtitle_file):
+            with open(subtitle_file, 'r') as file:
+                subtitles = file.read().strip().split('\n\n')
+            # 从SRT字幕文件中获取字幕
+            subtitle_clips = []
+            for subtitle in subtitles:
+                # 按行分割字幕内容
+                subtitle_lines = subtitle.strip().split('\n')
+                # 提取时间轴信息和字幕文本
+                if len(subtitle_lines) >= 3:
+                    times, text = subtitle_lines[1], '\n'.join(subtitle_lines[2:])
+                    start, end = map(cls.srt_to_seconds, times.split(' --> '))
+                    start = editor.cvsecs(start)
+                    end = editor.cvsecs(end)
+                text = cls.split_text(text, 10)
+                sub = editor.TextClip(text, font="/System/Library/Fonts/Hiragino Sans GB.ttc",
+                                      fontsize=30, color="black").set_duration(end - start).set_start(
+                    start).set_position(
+                    ("center", final_height - 80)).set_opacity(0.8)
+                subtitle_clips.append(sub)
+
+            # 将字幕添加到视频上
+            video_with_subtitles = editor.CompositeVideoClip([final_clip] + subtitle_clips)
+        else:
+            text_clip = (
+                editor.TextClip("分享、转发给群友", font="/System/Library/Fonts/Hiragino Sans GB.ttc",
+                                fontsize=30, color="black").
+                    set_position(("center", final_height - 80)).
+                    set_duration(duration_limit).
+                    set_opacity(0.8)
+            )
+            # 把 `文本剪贴板` 贴在视频上
+            video_with_subtitles = editor.CompositeVideoClip([final_clip, text_clip])
+
+        # 生成视频
+        video_with_subtitles.write_videofile(output_path, codec='libx264', fps=30, threads=4)
+        if os.path.isfile(output_path):
+            Common.logger().info("视频生成成功！生成路径为:", output_path)
+            return included_videos
+        else:
+            Common.logger().info("视频生成失败，请检查代码和文件路径。")
+            return ""
+
+
+
+    @classmethod
+    def get_audio_url(cls, i):
+        url = f"https://admin.piaoquantv.com/manager/video/detail/{i}"
+
+        payload = {}
+        headers = {
+            'authority': 'admin.piaoquantv.com',
+            'accept': 'application/json, text/plain, */*',
+            'accept-language': 'zh-CN,zh;q=0.9',
+            'cache-control': 'no-cache',
+            'cookie': 'SESSION=MDJiNGM4YzgtY2RiMC00ZjRkLThlNzEtOThhZDJkMjE0Yzgx',
+            'pragma': 'no-cache',
+            'referer': f'https://admin.piaoquantv.com/cms/post-detail/{i}/detail',
+            'sec-ch-ua': '"Not_A Brand";v="8", "Chromium";v="120", "Google Chrome";v="120"',
+            'sec-ch-ua-mobile': '?0',
+            'sec-ch-ua-platform': '"macOS"',
+            'sec-fetch-dest': 'empty',
+            'sec-fetch-mode': 'cors',
+            'sec-fetch-site': 'same-origin',
+            'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
+        }
+
+        response = requests.request("GET", url, headers=headers, data=payload)
+        data = response.json()
+        audio_url = data["content"]["transedVideoPath"]
+        print(audio_url)
+        return audio_url
+
+    @classmethod
+    def video_stitching(cls):
+        count = 0
+        # 获取音频
+        audio = Material.get_audio()
+        for i in audio:
+            # 获取已入库的用户id
+            account_id = cls.get_account_id()
+            for j in account_id:
+                account = j[0].replace('(', '').replace(')', '').replace(',', '')
+                Common.logger().info(f"获取用户ID：{account}")
+                # 获取 未使用的视频链接
+                url_list = cls.get_url_list(i, account)
+                # 获取音频url
+                audio = cls.get_audio_url(i)
+                Common.logger().info(f"获取音频地址：{audio}")
+                videos = [list(item) for item in url_list]
+                videos = Oss.get_oss_url(videos)
+                # 视频截取
+                try:
+                    audio_url = cls.concatenate_videos(videos, str(audio), i)
+                    if len(audio_url) == 0:
+                        Common.logger().info(f"视频生成失败")
+                        continue
+                    # 随机生成视频id
+                    id = cls.random_id()
+                    Common.logger().info(f"生成视频id为：{id}")
+                    # 上传 oss
+                    oss_object_key = Oss.stitching_sync_upload_oss(output_path, id)
+                    status = oss_object_key.get("status")
+                    # 获取 oss 视频地址
+                    oss_object_key = oss_object_key.get("oss_object_key")
+                    Common.logger().info(f"新拼接视频，oss发送成功，oss地址：{oss_object_key}")
+                    if status == 200:
+                        time.sleep(10)
+                        # 发送成功 已使用视频存入数据库
+                        cls.insert_videoAudio(audio_url, i)
+                        Common.logger().info(f"发送成功 已使用视频存入数据库完成")
+                        if os.path.isfile(output_path):
+                            os.remove(output_path)
+                            Common.logger().info(f"文件删除成功{output_path}")
+                        else:
+                            Common.logger().info(f"文件不存在{output_path}")
+                        piaoquantv = cls.insert_piaoquantv(oss_object_key)
+                        if piaoquantv:
+                            count += 1
+                            Common.logger().info(f"视频添加到对应用户成功")
+                            if count >= 20:
+                                break
+                except Exception as e:
+                    Common.logger().warning(f"新拼接视频发送oss失败:{e}\n")
+                    continue
+            if count >= 20:
+                break
+
+
+
+
+
+if __name__ == '__main__':
+    VideoStitching.video_stitching()