调试

channel/ks_analyze_photo.py → channel_ks/ks_analyze_photo.py

@@ -6,16 +6,16 @@ from flask import Flask, request, jsonify
 
 app = Flask(__name__)
 
-def haiwai_tunnel_proxies():
-    tunnel = "c101.kdlfps.com:18866"
-    # 用户名密码方式
-    username = "f2801246645"
-    password = "q0i0ohnl"
-    proxies = {
-        "http": "http://%(user)s:%(pwd)s@%(proxy)s/" % {"user": username, "pwd": password, "proxy": tunnel},
-        "https": "http://%(user)s:%(pwd)s@%(proxy)s/" % {"user": username, "pwd": password, "proxy": tunnel}
-    }
-    return proxies
+# def haiwai_tunnel_proxies():
+#     tunnel = "c101.kdlfps.com:18866"
+#     # 用户名密码方式
+#     username = "f2801246645"
+#     password = "q0i0ohnl"
+#     proxies = {
+#         "http": "http://%(user)s:%(pwd)s@%(proxy)s/" % {"user": username, "pwd": password, "proxy": tunnel},
+#         "https": "http://%(user)s:%(pwd)s@%(proxy)s/" % {"user": username, "pwd": password, "proxy": tunnel}
+#     }
+#     return proxies
 
 def process_range_data(data, label_key, value_key, selected_key=None, tips_key=None):
     processed_data = []
@@ -54,7 +54,7 @@ def analyze_photo():
     salt = '08d8eece8e83'
     sig = md5(f'{src}{salt}'.encode()).hexdigest()
     query.update({'sig': sig})
-    response = requests.post(url=url, headers=headers, params=query, data=data, proxies=haiwai_tunnel_proxies())
+    response = requests.post(url=url, headers=headers, params=query, data=data)
 
     if response.status_code != 200:
         return jsonify({"error": "Failed to analyze photo"}), response.status_code
@@ -63,6 +63,8 @@ def analyze_photo():
     json_body = json.loads(body)
 
     user_range = json_body['data']['play']['userRange']
+    if len(user_range) == 0:
+        return jsonify({photo_id: "无观众画像"})
     age_range = user_range['ageRange']
     sex_range = user_range['sexRange']
     province_range = user_range['provinceRange']

channel_ks/ks_feed_list.py

@@ -0,0 +1,335 @@
+import json
+import os
+from hashlib import md5
+import requests
+import time
+from urllib.parse import urlencode
+from datetime import datetime, timedelta
+
+from common import Oss
+from common.sql_help import sqlCollect
+
+headers = {
+    'Accept-Language': 'zh-cn',
+    'Connection': 'keep-alive',
+    'Content-Type': 'application/x-www-form-urlencoded',
+    'Host': 'creator-app.kuaishou.com',
+    'User-Agent': 'kwai-android aegon/3.12.1',
+}
+class KsFeedVideo:
+    CATEGORY_IDS = {
+        1: "生活",
+        2: "才艺",
+        3: "时尚",
+        4: "宠物",
+        5: "读书",
+        6: "二次元",
+        7: "家居",
+        8: "数码",
+        9: "搞笑",
+        10: "健康",
+        11: "旅游",
+        12: "美食",
+        13: "美妆",
+        14: "汽车",
+        15: "亲子",
+        16: "情感",
+        17: "三农",
+        18: "摄影",
+        19: "舞蹈",
+        20: "颜值",
+        21: "音乐",
+        22: "影视",
+        23: "短剧",
+        24: "游戏",
+        25: "运动",
+        26: "资讯",
+        27: "人文"
+    }
+    current_category_index = 0
+
+    @staticmethod
+    def calculate_sig(data):
+        src = ''.join([f'{key}={data[key]}' for key in sorted(data.keys())])
+        salt = '08d8eece8e83'
+        return md5(f'{src}{salt}'.encode()).hexdigest()
+
+
+    """
+    切换品类
+    """
+    @classmethod
+    def switch_category(cls):
+        if cls.current_category_index >= len(cls.CATEGORY_IDS):
+            cls.current_category_index = 0
+        category_id = list(cls.CATEGORY_IDS.keys())[cls.current_category_index]
+        url = 'https://creator-app.kuaishou.com/rest/bamboo/inspiration/n/category/confirm/optimize'
+        data = {
+            'isRecommendChange': False,
+            'categoryId': category_id,
+            'kuaishou.api_st': "Cg9rdWFpc2hvdS5hcGkuc3QSkAGMQoIK2ZpwlQszYISTxSFxzugi58w2U5gpPqa6an0eU6MFcVsXq2rd_K16UTItZ_OzPV-4jmVN5rNXKXW9jL97JV79Y9PqxaR9xOIr1TEyDzpOq2GM-0W1QRW3M8Li_J6NZ5t1hRFCWHBlOESjiBWs7vq4m1bq_ml0dZ6pgEDfpsWNpBaLRzwZwOO1mD4LqO4aEokh6uHql0RmmtbfoBF25r7QOyIgqNv0TBf6mlwS3bjE0K6sl08M1mMPjW1PB9e0Qr494H8oBTAB",
+            'client_key': '214c9979',
+        }
+        sig = cls.calculate_sig(data)
+        data['sig'] = sig
+        response = requests.post(url=url, headers=headers, data=data)
+        body = response.content.decode()
+        cls.current_category_index += 1
+        return body
+
+    """
+    获取feed流信息
+    """
+    @classmethod
+    def get_feed_list(cls):
+        cls.switch_category()
+        url = 'https://creator-app.kuaishou.com/rest/bamboo/inspiration/n/feed'
+        data = {
+            'cs': False,
+            'kuaishou.api_st': "Cg9rdWFpc2hvdS5hcGkuc3QSkAGMQoIK2ZpwlQszYISTxSFxzugi58w2U5gpPqa6an0eU6MFcVsXq2rd_K16UTItZ_OzPV-4jmVN5rNXKXW9jL97JV79Y9PqxaR9xOIr1TEyDzpOq2GM-0W1QRW3M8Li_J6NZ5t1hRFCWHBlOESjiBWs7vq4m1bq_ml0dZ6pgEDfpsWNpBaLRzwZwOO1mD4LqO4aEokh6uHql0RmmtbfoBF25r7QOyIgqNv0TBf6mlwS3bjE0K6sl08M1mMPjW1PB9e0Qr494H8oBTAB",
+            'client_key': '214c9979',
+        }
+        sig = cls.calculate_sig(data)
+        data['sig'] = sig
+        response = requests.post(url=url, headers=headers, data=data)
+        body = response.content.decode()
+        return body
+
+    """
+    获取观众画像
+    """
+    @classmethod
+    def analyze_photo(cls, photo_id):
+        url = 'https://creator-app.kuaishou.com/rest/bamboo/inspiration/n/photo/analysis'
+
+        headers = {
+            'Accept-Language': 'zh-cn',
+            'Connection': 'keep-alive',
+            'Content-Type': 'application/x-www-form-urlencoded',
+            'Host': 'creator-app.kuaishou.com',
+        }
+        data = {
+            'photoId': photo_id,
+            'client_key': '214c9979',
+        }
+        sig = cls.calculate_sig(data)
+        data['sig'] = sig
+        response = requests.post(url=url, headers=headers, data=data)
+        body = response.content.decode()
+        json_body = json.loads(body)
+        user_range = json_body['data']['play']['userRange']
+        if len(user_range) == 0:
+            return False, "无画像"
+        age_range = user_range['ageRange']
+        value = age_range[5]['value']
+        value = int(value.strip('%'))
+        if value >= 50:
+            return True, value
+        else:
+            return False, value
+
+    """
+    视频时长转换成秒
+    """
+    @classmethod
+    def milliseconds_to_seconds(cls, milliseconds):
+        seconds = milliseconds / 1000
+        return int(seconds)
+
+    """
+    判断当前视频是否在90天内
+    """
+    @classmethod
+    def get_video_data(cls, timestamp_str):
+        timestamp = datetime.strptime(timestamp_str, "%Y-%m-%d %H:%M:%S")
+        # 获取当前时间
+        current_time = datetime.now()
+        difference = current_time - timestamp
+        if difference <= timedelta(days=90):
+            return False
+        else:
+            return True
+
+    """
+    生成目录
+    """
+    @classmethod
+    def create_folders(cls):
+        video_path_url = "/Users/tzld/Desktop/ks_automation/path_video/"
+        if not os.path.exists(video_path_url):
+            os.makedirs(video_path_url)
+        return video_path_url
+
+    """
+    删除文件
+    """
+    @classmethod
+    def remove_files(cls, video_path_url):
+        if os.path.exists(video_path_url) and os.path.isdir(video_path_url):
+            for root, dirs, files in os.walk(video_path_url):
+                for file in files:
+                    file_path = os.path.join(root, file)
+                    os.remove(file_path)
+                for dir in dirs:
+                    dir_path = os.path.join(root, dir)
+                    os.rmdir(dir_path)
+
+    """
+    视频下载
+    """
+    @classmethod
+    def download_video(cls, video_url, path_url, video_id):
+        for i in range(3):
+            payload = {}
+            headers = {}
+            response = requests.request("GET", video_url, headers=headers, data=payload)
+            if response.status_code == 200:
+                # 以二进制写入模式打开文件
+                video = path_url + str(video_id) + '.mp4'
+                with open(f"{video}", "wb") as file:
+                    # 将响应内容写入文件
+                    file.write(response.content)
+                time.sleep(5)
+                return video
+        return ''
+
+    """
+    获取票圈ID
+    """
+    @classmethod
+    def get_id_by_category(cls, category_name):
+        category_list = [
+            {"id": 71502003, "category": "生活"},
+            {"id": 71502004, "category": "才艺"},
+            {"id": 71502005, "category": "时尚"},
+            {"id": 71502006, "category": "宠物"},
+            {"id": 71502007, "category": "读书"},
+            {"id": 71502008, "category": "二次元"},
+            {"id": 71502009, "category": "家居"},
+            {"id": 71502010, "category": "数码"},
+            {"id": 71502011, "category": "搞笑"},
+            {"id": 71502012, "category": "健康"},
+            {"id": 71502013, "category": "旅游"},
+            {"id": 71502014, "category": "美食"},
+            {"id": 71502015, "category": "美妆"},
+            {"id": 71502016, "category": "汽车"},
+            {"id": 71502018, "category": "亲子"},
+            {"id": 71502019, "category": "情感"},
+            {"id": 71502020, "category": "三农"},
+            {"id": 71502021, "category": "摄影"},
+            {"id": 71502022, "category": "舞蹈"},
+            {"id": 71502023, "category": "颜值"},
+            {"id": 71502024, "category": "音乐"},
+            {"id": 71502025, "category": "影视"},
+            {"id": 71502026, "category": "短剧"},
+            {"id": 71502027, "category": "游戏"},
+            {"id": 71502028, "category": "运动"},
+            {"id": 71502029, "category": "资讯"},
+            {"id": 71502030, "category": "人文"}
+        ]
+        for category in category_list:
+            if category['category'] == category_name:
+                return category['id']
+        return None
+
+    """
+    新生成视频上传到对应账号下
+    """
+    @classmethod
+    def insert_piaoquantv(cls, new_video_path, new_title, n_id):
+
+        url = "https://vlogapi.piaoquantv.com/longvideoapi/crawler/video/send"
+        headers = {
+            'User-Agent': 'PQSpeed/486 CFNetwork/1410.1 Darwin/22.6.0',
+            'cookie': 'JSESSIONID=4DEA2B5173BB9A9E82DB772C0ACDBC9F; JSESSIONID=D02C334150025222A0B824A98B539B78',
+            'referer': 'http://appspeed.piaoquantv.com',
+            'token': '524a8bc871dbb0f4d4717895083172ab37c02d2f',
+            'accept-language': 'zh-CN,zh-Hans;q=0.9',
+            'Content-Type': 'application/x-www-form-urlencoded'
+        }
+        payload = {
+            'deviceToken': '9ef064f2f7869b3fd67d6141f8a899175dddc91240971172f1f2a662ef891408',
+            'fileExtensions': 'MP4',
+            'loginUid': n_id,
+            'networkType': 'Wi-Fi',
+            'platform': 'iOS',
+            'requestId': 'fb972cbd4f390afcfd3da1869cd7d001',
+            'sessionId': '362290597725ce1fa870d7be4f46dcc2',
+            'subSessionId': '362290597725ce1fa870d7be4f46dcc2',
+            'title': new_title,
+            'token': '524a8bc871dbb0f4d4717895083172ab37c02d2f',
+            'uid': n_id,
+            'versionCode': '486',
+            'versionName': '3.4.12',
+            'videoFromScene': '1',
+            'videoPath': new_video_path,
+            'viewStatus': '1'
+        }
+        encoded_payload = urlencode(payload)
+        response = requests.request("POST", url, headers=headers, data=encoded_payload)
+        data = response.json()
+        code = data["code"]
+        if code == 0:
+            new_video_id = data["data"]["id"]
+            return new_video_id
+        else:
+            return None
+
+    @classmethod
+    def get_data(cls):
+        for category_id, category_name in cls.CATEGORY_IDS.items():
+            feed_data = cls.get_feed_list()
+            feed_data = json.loads(feed_data)
+            feeds = feed_data['feeds']
+            for feed in feeds:
+                photo_id = feed["photo_id"]  # 视频ID
+                status = sqlCollect.is_used(photo_id)
+                if status:
+                    user_name = feed["user_name"]  # 用户名
+                    user_sex = feed["user_sex"]  # 性别 F为女，U为男
+                    time_data = feed["time"]  # 发布时间
+                    caption = feed["caption"]  # 标题
+                    view_count = feed["view_count"]  # 浏览数
+                    like_count = feed["like_count"]  # 点赞数
+                    share_count = feed["share_count"]  # 分享数
+                    duration = feed["duration"]  # 时长/秒
+                    duration = cls.milliseconds_to_seconds(duration)
+                    main_mv_url = feed["main_mv_url"]  # 视频链接
+                    thumbnail_url = feed["thumbnail_url"]  # 视频封面
+                    user_id = feed["user_id"]  # 用户id非用户主页id
+                    time_data = cls.get_video_data(time_data)
+                    if time_data:
+                        continue
+                    video_percent = '%.2f' % (share_count / like_count)
+                    special = float(0.2)
+                    # if float(video_percent) < special or share_count < 2000 or duration < 30 or duration > 6000:
+                    #     continue
+                    value, age = cls.analyze_photo(photo_id)
+                    # path_url = cls.create_folders()  # 创建目录
+                    # video_path = cls.download_video(main_mv_url, path_url, photo_id)
+                    # if not os.path.isfile(video_path):
+                    #     cls.remove_files(path_url)
+                    #     continue
+                    oss_object_key = Oss.channel_upload_oss(main_mv_url, photo_id)
+                    time.sleep(2)
+                    oss_object = oss_object_key.get("oss_object_key")
+                    pq_id = cls.get_id_by_category(category_name)
+                    if pq_id:
+                        video_uid = cls.insert_piaoquantv(oss_object_key, caption, pq_id)
+                        if video_uid:
+                            print(video_uid)
+                            return 
+                    if oss_object:
+                        pass
+                    if value:
+                        pass
+
+
+                    else:
+                        pass
+
+
+
+# Example usage:
+if __name__ == "__main__":
+    KsFeedVideo.get_data()

common/__init__.py

@@ -1,6 +1,5 @@
-from .common import Common
-from .aliyun_oss_uploading import Oss
-from .material import Material
-from .feishu import Feishu
-from .db import MysqlHelper
-from .pq import PQ
+from .common_log import Common
+from .aliyun_oss import Oss
+from .feishu_form import Material
+from .feishu_utils import Feishu
+from .mysql_db import MysqlHelper

common/aliyun_oss_uploading.py → common/aliyun_oss.py

@@ -1,23 +1,24 @@
 # -*- coding: utf-8 -*-
+# @Time: 2023/12/26
+import time
 from datetime import datetime
 from typing import Dict, Any,  Optional
 
 import oss2
 import requests
 
+# OSS_BUCKET_PATH = "douyin"
 OSS_ACCESS_KEY_ID = "LTAIP6x1l3DXfSxm"
 OSS_ACCESS_KEY_SECRET = "KbTaM9ars4OX3PMS6Xm7rtxGr1FLon"
 OSS_BUCKET_ENDPOINT = "oss-cn-hangzhou-internal.aliyuncs.com"# 内网地址
 # OSS_BUCKET_ENDPOINT = "oss-cn-hangzhou.aliyuncs.com" # 外网地址
 OSS_BUCKET_NAME = "art-crawler"
 class Oss():
-    # 抓取视频上传到art-crawler
+
     @classmethod
-    def video_sync_upload_oss(cls, src_url: str,
-                        video_id: str,
-                        account_id: str,
-                        OSS_BUCKET_PATH: str,
-                        referer: Optional[str] = None) -> Dict[str, Any]:
+    def channel_upload_oss(cls, src_url: str,
+                              video_id: str,
+                              referer: Optional[str] = None) -> Dict[str, Any]:
         headers = {
             'Accept': '*/*',
             'Accept-Language': 'zh-CN,zh;q=0.9',
@@ -32,9 +33,9 @@ class Oss():
         file_content = response.content
         content_type = response.headers.get('Content-Type', 'application/octet-stream')
 
-        oss_object_key = f'{OSS_BUCKET_PATH}/{account_id}/{video_id}'
+        oss_object_key = f'channel/ksczz/{video_id}'
         auth = oss2.Auth(OSS_ACCESS_KEY_ID, OSS_ACCESS_KEY_SECRET)
-        bucket = oss2.Bucket(auth, OSS_BUCKET_ENDPOINT, OSS_BUCKET_NAME)
+        bucket = oss2.Bucket(auth, OSS_BUCKET_ENDPOINT, "art-pubbucket")
         response = bucket.put_object(oss_object_key, file_content, headers={'Content-Type': content_type})
 
         if 'Content-Length' in response.headers:
@@ -43,12 +44,13 @@ class Oss():
                 'oss_object_key': oss_object_key}
         raise AssertionError(f'OSS上传失败，请求ID: \n{response.headers["x-oss-request-id"]}')
 
-
-    #  视频发送到art-pubbucket
+    """
+    视频发送到art-pubbucket
+    """
     @classmethod
     def stitching_sync_upload_oss(cls, src_url: str,
                         video_id: str) -> Dict[str, Any]:
-        oss_object_key = f'agc_oss/agc_video/{video_id}'
+        oss_object_key = f'channel/ks_video/{video_id}'
         auth = oss2.Auth(OSS_ACCESS_KEY_ID, OSS_ACCESS_KEY_SECRET)
         bucket = oss2.Bucket(auth, OSS_BUCKET_ENDPOINT, "art-pubbucket")
         response = bucket.put_object_from_file(oss_object_key, src_url)
@@ -62,35 +64,22 @@ class Oss():
         raise AssertionError(f'OSS上传失败，请求ID: \n{response.headers["x-oss-request-id"]}')
 
 
-    # 获取视频链接 将视频链接有效时间设置为1天
     @classmethod
-    def get_oss_url(cls, videos, video_path):
-        auth = oss2.Auth(OSS_ACCESS_KEY_ID, OSS_ACCESS_KEY_SECRET)
-        bucket = oss2.Bucket(auth, OSS_BUCKET_ENDPOINT, OSS_BUCKET_NAME)
-        list = []
-        for i in videos:
-            try:
-                # 获取指定路径下的对象列表
-                filename = i[2].split("/")[-1]
-                bucket.get_object_to_file(i[2], f'{video_path}{filename}.mp4')
-                list.append([i[0], i[1], i[2], f'{video_path}{filename}.mp4'])
-            except Exception:
-                continue
-        return list
+    def download_video_oss(cls, video_url, video_path_url, v_id):
+        video_path = video_path_url + str(v_id) + '.mp4'
+        oss_object_key = cls.channel_upload_oss(video_url, v_id)
+        time.sleep(2)
+        oss_object = oss_object_key.get("oss_object_key")
+        if oss_object:
+            auth = oss2.Auth(OSS_ACCESS_KEY_ID, OSS_ACCESS_KEY_SECRET)
+            bucket = oss2.Bucket(auth, OSS_BUCKET_ENDPOINT, OSS_BUCKET_NAME)
+            # 获取指定路径下的对象列表
+            bucket.get_object_to_file(oss_object, video_path)
+            time.sleep(5)
+            return video_path
+        else:
+            return video_path
+
+
+
 
-    @classmethod
-    def download_url(cls, videos, video_path, video):
-        for i in range(3):
-            payload = {}
-            headers = {}
-            response = requests.request("GET", videos, headers=headers, data=payload)
-            if response.status_code == 200:
-                video_url = []
-                # 以二进制写入模式打开文件
-                video = video_path+video+'.mp4'
-                with open(f"{video}", "wb") as file:
-                    # 将响应内容写入文件
-                    file.write(response.content)
-                video_url.append(video)
-                return video_url
-        return ''

common/db.py

@@ -1,139 +0,0 @@
-# -*- coding: utf-8 -*-
-# @Time: 2023/12/26
-"""
-数据库连接及操作
-"""
-import redis
-import pymysql
-from common.common import Common
-# from common import Common
-
-class MysqlHelper:
-    @classmethod
-    def connect_mysql(cls, env, machine):
-        if machine == 'aliyun_hk':
-            # 创建一个 Connection 对象，代表了一个数据库连接
-            connection = pymysql.connect(
-                host="rm-j6cz4c6pt96000xi3.mysql.rds.aliyuncs.com",# 数据库IP地址，内网地址
-                # host="rm-j6cz4c6pt96000xi3lo.mysql.rds.aliyuncs.com",# 数据库IP地址，外网地址
-                port=3306,                      # 端口号
-                user="crawler",                 #  mysql用户名
-                passwd="crawler123456@",        # mysql用户登录密码
-                db="piaoquan-crawler" ,         # 数据库名
-                # 如果数据库里面的文本是utf8编码的，charset指定是utf8
-                charset = "utf8")
-        elif env == 'prod':
-            # 创建一个 Connection 对象，代表了一个数据库连接
-            connection = pymysql.connect(
-                host="rm-bp1159bu17li9hi94.mysql.rds.aliyuncs.com",# 数据库IP地址，内网地址
-                # host="rm-bp1159bu17li9hi94ro.mysql.rds.aliyuncs.com",# 数据库IP地址，外网地址
-                port=3306,                      # 端口号
-                user="crawler",                 #  mysql用户名
-                passwd="crawler123456@",        # mysql用户登录密码
-                db="piaoquan-crawler" ,         # 数据库名
-                # 如果数据库里面的文本是utf8编码的，charset指定是utf8
-                charset = "utf8")
-        else:
-            # 创建一个 Connection 对象，代表了一个数据库连接
-            connection = pymysql.connect(
-                host="rm-bp1k5853td1r25g3n690.mysql.rds.aliyuncs.com",# 数据库IP地址，内网地址
-                # host="rm-bp1k5853td1r25g3ndo.mysql.rds.aliyuncs.com",  # 数据库IP地址，外网地址
-                port=3306,  # 端口号
-                user="crawler",  # mysql用户名
-                passwd="crawler123456@",  # mysql用户登录密码
-                db="piaoquan-crawler",  # 数据库名
-                # 如果数据库里面的文本是utf8编码的，charset指定是utf8
-                charset="utf8")
-
-        return connection
-
-    @classmethod
-    def get_values(cls, sql, env):
-        try:
-            machine = ""
-            # 连接数据库
-            connect = cls.connect_mysql(env, machine)
-            # 返回一个 Cursor对象
-            mysql = connect.cursor()
-
-            # 执行 sql 语句
-            mysql.execute(sql)
-
-            # fetchall方法返回的是一个元组，里面每个元素也是元组，代表一行记录
-            data = mysql.fetchall()
-
-            # 关闭数据库连接
-            connect.close()
-
-            # 返回查询结果，元组
-            return data
-        except Exception as e:
-            print(f"get_values异常:{e}\n")
-            # Common.logger(log_type, crawler).error(f"get_values异常:{e}\n")
-
-    @classmethod
-    def update_values(cls, sql, env, machine):
-        # 连接数据库
-        connect = cls.connect_mysql(env, machine)
-        # 返回一个 Cursor对象
-        mysql = connect.cursor()
-
-        try:
-            # 执行 sql 语句
-            res = mysql.execute(sql)
-            # 注意 一定要commit，否则添加数据不生效
-            connect.commit()
-            return res
-        except Exception as e:
-            # print(f"update_values异常，进行回滚操作:{e}\n")
-            # 发生错误时回滚
-            connect.rollback()
-
-        # 关闭数据库连接
-        connect.close()
-
-class RedisHelper:
-    @classmethod
-    def connect_redis(cls, env, machine):
-        if machine == 'aliyun_hk':
-            redis_pool = redis.ConnectionPool(
-                # host='r-bp154bpw97gptefiqk.redis.rds.aliyuncs.com',  # 内网地址
-                host='r-bp154bpw97gptefiqkpd.redis.rds.aliyuncs.com',  # 外网地址
-                port=6379,
-                db=2,
-                password='Qingqu2019'
-            )
-            redis_conn = redis.Redis(connection_pool=redis_pool)
-        elif env == 'prod':
-            redis_pool = redis.ConnectionPool(
-                host='r-bp1mb0v08fqi4hjffu.redis.rds.aliyuncs.com',  # 内网地址
-                # host='r-bp1mb0v08fqi4hjffupd.redis.rds.aliyuncs.com',  # 外网地址
-                port=6379,
-                db=2,
-                password='Qingqu2019'
-            )
-            redis_conn = redis.Redis(connection_pool=redis_pool)
-        else:
-            redis_pool = redis.ConnectionPool(
-                # host='r-bp154bpw97gptefiqk.redis.rds.aliyuncs.com',  # 内网地址
-                host='r-bp154bpw97gptefiqkpd.redis.rds.aliyuncs.com',  # 外网地址
-                port=6379,
-                db=2,
-                password='Qingqu2019'
-            )
-            redis_conn = redis.Redis(connection_pool=redis_pool)
-        return redis_conn
-
-    @classmethod
-    def redis_push(cls, env, machine, data):
-        redis_conn = cls.connect_redis(env, machine)
-        redis_conn.lpush(machine, data)
-
-    @classmethod
-    def redis_pop(cls, env, machine):
-        redis_conn = cls.connect_redis(env, machine)
-        if redis_conn.llen(machine) == 0:
-            return None
-        else:
-            return redis_conn.rpop(machine)
-

common/feishu_form.py

@@ -0,0 +1,132 @@
+# -*- coding: utf-8 -*-
+import os
+import random
+import sys
+import datetime
+
+
+sys.path.append(os.getcwd())
+from common.feishu_utils import Feishu
+
+
+class Material():
+    """
+    获取汇总表所有负责人列表
+    """
+    @classmethod
+    def feishu_list(cls):
+        summary = Feishu.get_values_batch("summary", "bc154d")
+        list = []
+        for row in summary[1:]:
+            mark = row[0]
+            name = row[1]
+            feishu_id = row[3]
+            feishu_sheet = row[4]
+            cookie_sheet = row[5]
+            number = {"mark": mark, "name": name, "feishu_id": feishu_id, "feishu_sheet": feishu_sheet, "cookie_sheet": cookie_sheet}
+            if mark:
+                list.append(number)
+            else:
+                return list
+        return list
+
+    """
+    获取对应负责人任务明细
+    """
+    @classmethod
+    def get_task_data(cls, feishu_id, feishu_sheet):
+        data = Feishu.get_values_batch(feishu_id, feishu_sheet)
+        processed_list = []
+
+        for row in data[1:]:
+            channel_id = row[1]
+            channel_url = row[2]
+            piaoquan_id = row[3]
+            number = row[4]
+            video_share = row[5]
+            video_ending = row[6]
+            crop_tool = row[7]
+            gg_duration = row[8]
+            title = row[9]
+
+            def count_items(item, separator):
+                if item and item not in {'None', ''}:
+                    return len(item.split(separator))
+                return 0
+
+            video_id_total = count_items(str(channel_url), ',')
+            title_total = count_items(str(title), '/')
+            video_ending_total = count_items(str(video_ending), ',')
+
+            values = [channel_id, video_id_total, piaoquan_id, video_share, video_ending_total, crop_tool, gg_duration, title_total]
+            filtered_values = [str(value) for value in values if value is not None and value != "None"]
+            task_mark = "_".join(map(str, filtered_values))
+
+            if piaoquan_id and piaoquan_id not in {'None', ''}:
+                number_dict = {
+                    "task_mark": task_mark,
+                    "channel_id": channel_id,
+                    "channel_url": channel_url,
+                    "piaoquan_id": piaoquan_id,
+                    "number": number,
+                    "title": title,
+                    "video_share": video_share,
+                    "video_ending": video_ending,
+                    "crop_total": crop_tool,
+                    "gg_duration_total": gg_duration,
+                }
+                processed_list.append(number_dict)
+            else:
+                return processed_list
+
+        return processed_list
+
+    """
+    获取对应片尾+srt
+    """
+    @classmethod
+    def get_pwsrt_data(cls, feishu_id, feishu_sheet, video_ending):
+        data = Feishu.get_values_batch(feishu_id, feishu_sheet)
+
+        for row in data[1:]:
+            pw_mark = row[0]
+            pw_id = row[1]
+            pw_srt = row[2]
+            if pw_id != 'None' and pw_id != '' and pw_id != None:
+                if pw_mark == video_ending:
+                    number = {"pw_id": pw_id, "pw_srt": pw_srt}
+                    return number
+        return ''
+
+    """
+    获取对应固定字幕
+    """
+    @classmethod
+    def get_pzsrt_data(cls, feishu_id, feishu_sheet, video_share_name):
+        data = Feishu.get_values_batch(feishu_id, feishu_sheet)
+
+        for row in data[1:]:
+            pz_mark = row[0]
+            pz_zm = row[1]
+            if pz_zm != 'None' and pz_zm != '' and pz_zm != None:
+                if pz_mark == video_share_name:
+                    return pz_zm
+        return ''
+
+
+
+    """
+    获取 cookie 信息
+    """
+    @classmethod
+    def get_cookie_data(cls, feishu_id, cookie_sheet, channel):
+        data = Feishu.get_values_batch(feishu_id, cookie_sheet)
+        for row in data[1:]:
+            channel_mask = row[0]
+            cookie = row[1]
+            if channel_mask == channel:
+                return cookie
+
+
+
+

common/feishu.py → common/feishu_utils.py

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+# @Time: 2023/12/26
 """
 飞书表配置: token 鉴权 / 增删改查 / 机器人报警
 """
@@ -23,7 +24,7 @@ class Feishu:
     @classmethod
     def spreadsheettoken(cls, crawler):
         if crawler == "summary":
-            return "IbVVsKCpbhxhSJtwYOUc8S1jnWb"
+            return "KsoMsyP2ghleM9tzBfmcEEXBnXg"
         else:
             return crawler
 
@@ -106,7 +107,7 @@ class Feishu:
 
     # 工作表，插入行或列
     @classmethod
-    def insert_columns(cls,  crawler, sheetid, majordimension, startindex, endindex):
+    def insert_columns(cls, crawler, sheetid, majordimension, startindex, endindex):
         """
         工作表插入行或列
         :param log_type: 日志路径
@@ -173,7 +174,7 @@ class Feishu:
 
     # 合并单元格
     @classmethod
-    def merge_cells(cls, log_type, crawler, sheetid, ranges):
+    def merge_cells(cls, crawler, sheetid, ranges):
         """
         合并单元格
         :param log_type: 日志路径
@@ -284,7 +285,7 @@ class Feishu:
 
     # 获取用户 ID
     @classmethod
-    def get_userid(cls, log_type, crawler, username):
+    def get_userid(cls, username):
         try:
             url = "https://open.feishu.cn/open-apis/user/v1/batch_get_id?"
             headers = {
@@ -313,36 +314,13 @@ class Feishu:
 
     # 飞书机器人
     @classmethod
-    def bot(cls, log_type, crawler, text, mark, mark_name):
+    def bot(cls, log_type, crawler, text, mark_name):
         try:
-            # url = "https://open.feishu.cn/open-apis/bot/v2/hook/5a6ce4ca-32fa-44fe-bbe4-69ae369bb3cf"
-            # url = "https://open.feishu.cn/open-apis/bot/v2/hook/2b317db6-93ed-43b4-bf01-03c35cfa1d59"
-            url = "https://open.feishu.cn/open-apis/bot/v2/hook/af368a84-545f-4106-9c4c-af64678ad7af"
+            url = "https://open.feishu.cn/open-apis/bot/v2/hook/e7697dc6-5254-4411-8b59-3cd0742bf703"
             headers = {'Content-Type': 'application/json'}
-            if crawler == "抖音":
-                content = "抖音cookie过期"
-                sheet_url = "https://w42nne6hzg.feishu.cn/sheets/IbVVsKCpbhxhSJtwYOUc8S1jnWb?sheet=n9xlLF"
-                users = "<at id=" + str(cls.get_userid(log_type, crawler, mark)) + f">{mark_name}</at>"
-            elif crawler == "管理后台":
-                content = "管理后台cookie过期"
-                sheet_url = "https://w42nne6hzg.feishu.cn/sheets/IbVVsKCpbhxhSJtwYOUc8S1jnWb?sheet=n9xlLF"
-                users = f"<at id=" + str(cls.get_userid(log_type, crawler, mark)) + f">{mark_name}</at>"
-            elif crawler == "快手":
-                content = "快手cookie过期"
-                sheet_url = "https://w42nne6hzg.feishu.cn/sheets/IbVVsKCpbhxhSJtwYOUc8S1jnWb?sheet=n9xlLF"
-                users = f"<at id=" + str(cls.get_userid(log_type, crawler, mark)) + f">{mark_name}</at>"
-            elif crawler == "AGC视频":
-                content = 'AGC视频生成条数详情'
-                sheet_url = "https://w42nne6hzg.feishu.cn/sheets/IbVVsKCpbhxhSJtwYOUc8S1jnWb?sheet=n9xlLF"
-                users = f"<at id=" + str(cls.get_userid(log_type, crawler, mark)) + f">{mark_name}</at> \n"
-            elif crawler == "AGC完成通知":
-                content = "今日所有AGC视频完成啦～～～"
-                sheet_url = "https://w42nne6hzg.feishu.cn/sheets/IbVVsKCpbhxhSJtwYOUc8S1jnWb?sheet=n9xlLF"
-                users = f"<at id=" + str(cls.get_userid(log_type, crawler, mark)) + f">{mark_name}</at>"
-            else:
-                content = "今日所有AGC视频完成啦～～～"
-                sheet_url = "https://w42nne6hzg.feishu.cn/sheets/IbVVsKCpbhxhSJtwYOUc8S1jnWb?sheet=n9xlLF"
-                users = f"<at id=" + str(cls.get_userid(log_type, crawler, mark)) + f">{mark_name}</at>"
+            if crawler == "机器自动改造消息通知":
+                sheet_url = "https://w42nne6hzg.feishu.cn/sheets/S8jusgF83h8gEKtILW4cli2Bngf?sheet=3e1295"
+                users = f"<at id=" + str(cls.get_userid(log_type)) + f">{mark_name}</at>"
 
             data = json.dumps({
                 "msg_type": "interactive",
@@ -361,7 +339,7 @@ class Feishu:
                         "actions": [{
                             "tag": "button",
                             "text": {
-                                "content": content,
+                                "content": "详情,点击～～～～～",
                                 "tag": "lark_md"
                             },
                             "url": sheet_url,
@@ -372,7 +350,7 @@ class Feishu:
                     }],
                     "header": {
                         "title": {
-                            "content": "📣您有新的信息，请注意查收",
+                            "content": "📣消息来喽～～～",
                             "tag": "plain_text"
                         }
                     }

common/ffmpeg.py

@@ -0,0 +1,309 @@
+
+import subprocess
+import time
+
+
+
+class FFmpeg():
+
+    """
+    时间转换
+    """
+    @classmethod
+    def seconds_to_srt_time(cls, seconds):
+        hours = int(seconds // 3600)
+        minutes = int((seconds % 3600) // 60)
+        seconds = seconds % 60
+        milliseconds = int((seconds - int(seconds)) * 1000)
+        return f"{hours:02d}:{minutes:02d}:{int(seconds):02d},{milliseconds:03d}"
+
+    """
+    获取单个视频时长
+    """
+    @classmethod
+    def get_video_duration(cls, video_url):
+        ffprobe_cmd = [
+            "ffprobe",
+            "-i", video_url,
+            "-show_entries", "format=duration",
+            "-v", "quiet",
+            "-of", "csv=p=0"
+        ]
+        output = subprocess.check_output(ffprobe_cmd).decode("utf-8").strip()
+        return float(output)
+
+    """
+    获取视频文件的时长（秒）
+    """
+    @classmethod
+    def get_videos_duration(cls, video_file):
+        result = subprocess.run(
+            ["ffprobe", "-v", "error", "-show_entries", "format=duration",
+             "-of", "default=noprint_wrappers=1:nokey=1", video_file],
+            capture_output=True, text=True)
+        return float(result.stdout)
+
+    """
+    视频裁剪
+    """
+    @classmethod
+    def video_tailor(cls, video_url):
+        output_video_path = ''
+        try:
+            # 获取视频的原始宽高信息
+            width, height = cls.get_w_h_size(video_url)
+            # 计算裁剪后的高度
+            new_height = int(height * 0.8)
+            # 构建 FFmpeg 命令,裁剪视频高度为原始高度的70%，并将宽度缩放为320x480
+            ffmpeg_cmd = [
+                "ffmpeg",
+                "-i", video_url,
+                "-vf", f"crop={width}:{new_height},scale=320:480",
+                "-c:v", "libx264",
+                "-c:a", "aac",
+                "-y",
+                output_video_path
+            ]
+            # 执行 FFmpeg 命令
+            subprocess.run(ffmpeg_cmd, check=True)
+            return output_video_path
+        except Exception as e:
+            return None
+
+    """
+    获取视频宽高
+    """
+    @classmethod
+    def get_w_h_size(cls, new_video_path):
+        try:
+            # 获取视频的原始宽高信息
+            ffprobe_cmd = f"ffprobe -v error -select_streams v:0 -show_entries stream=width,height -of csv=p=0 {new_video_path}"
+            ffprobe_process = subprocess.Popen(ffprobe_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
+            output, _ = ffprobe_process.communicate()
+            output_decoded = output.decode().strip()
+            split_output = [value for value in output_decoded.split(',') if value.strip()]
+            height, width = map(int, split_output)
+            return width, height
+        except ValueError as e:
+            return 1920, 1080
+
+    """
+    视频裁剪
+    """
+    @classmethod
+    def video_crop(cls, new_video_path, video_path_url, pw_random_id):
+        crop_url = video_path_url + str(pw_random_id) + 'crop.mp4'
+        # 获取视频的原始宽高信息
+        ffprobe_cmd = f"ffprobe -v error -select_streams v:0 -show_entries stream=width,height -of csv=p=0 {new_video_path}"
+        ffprobe_process = subprocess.Popen(ffprobe_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
+        output, _ = ffprobe_process.communicate()
+        width, height = map(int, output.decode().strip().split(','))
+
+        # 计算裁剪后的高度
+        new_height = int(height * 0.8)
+
+        # 构建 FFmpeg 命令,裁剪视频高度为原始高度的80%
+        ffmpeg_cmd = [
+            "ffmpeg",
+            "-i", new_video_path,
+            "-vf", f"crop={width}:{new_height}",
+            "-c:v", "libx264",
+            "-c:a", "aac",
+            "-y",
+            crop_url
+        ]
+        subprocess.run(ffmpeg_cmd)
+        return crop_url
+
+    """
+    视频裁剪
+    """
+    @classmethod
+    def video_ggduration(cls, new_video_path, video_path_url, pw_random_id, gg_duration_total):
+        gg_duration_url = video_path_url + str(pw_random_id) + 'gg_duration.mp4'
+        # 获取视频时长
+        total_duration = cls.get_video_duration(new_video_path)
+        duration = int(total_duration) - int(gg_duration_total)
+        if int(total_duration) < int(gg_duration_total):
+            return new_video_path
+        ffmpeg_cmd = [
+            "ffmpeg",
+            "-i", new_video_path,
+            "-c:v", "libx264",
+            "-c:a", "aac",
+            "-t", str(duration),
+            "-y",
+            gg_duration_url
+        ]
+        subprocess.run(ffmpeg_cmd)
+        return gg_duration_url
+
+    """
+     截取原视频最后一帧
+    """
+    @classmethod
+    def video_png(cls, new_video_path, video_path_url, pw_random_id):
+        """
+        jpg_url 生成图片位置
+        :param new_video_path: 视频地址
+        :return:
+        """
+        # 获取视频的原始宽高信息
+        jpg_url = video_path_url + str(pw_random_id) + 'png.jpg'
+        # 获取视频时长
+        total_duration = cls.get_video_duration(new_video_path)
+        time_offset = total_duration - 1  # 提取倒数第一秒的帧
+        # 获取视频最后一秒，生成.jpg
+        subprocess.run(
+            ['ffmpeg', '-ss', str(time_offset), '-i', new_video_path, '-t', str(total_duration), '-vf', 'fps=1', "-y", jpg_url])
+        return jpg_url
+
+    """
+    获取视频音频
+    """
+    @classmethod
+    def get_video_mp3(cls, video_file, video_path_url, pw_random_id):
+        pw_mp3_path = video_path_url + str(pw_random_id) +'pw_video.mp3'
+        command = [
+            'ffmpeg',
+            '-i', video_file,
+            '-q:a', '0',
+            '-map', 'a',
+            # '-codec:a', 'libmp3lame',  # 指定 MP3 编码器
+            pw_mp3_path
+        ]
+        subprocess.run(command)
+        time.sleep(1)
+        return pw_mp3_path
+
+    """
+     生成片尾视频
+    """
+    @classmethod
+    def pw_video(cls, jpg_url, video_path_url, pw_url, pw_srt, pw_random_id, pw_mp3_path):
+        # 添加音频到图片
+        """
+        jpg_url 图片地址
+        pw_video 提供的片尾视频
+        pw_duration  提供的片尾视频时长
+        new_video_path 视频位置
+        subtitle_cmd 字幕
+        pw_url 生成视频地址
+        :return:
+        """
+        pw_srt_path = video_path_url + str(pw_random_id) +'pw_video.srt'
+        # 创建临时字幕文件
+        with open(pw_srt_path, 'w') as f:
+            f.write(pw_srt)
+        # 片尾位置
+        pw_url_path = video_path_url + str(pw_random_id) + 'pw_video.mp4'
+        # 获取视频时长
+        pw_duration = cls.get_video_duration(pw_url)
+        time.sleep(2)
+        # 添加字幕 wqy-zenhei  Hiragino Sans GB
+        height = 1080
+        margin_v = int(height) // 8  # 可根据需要调整字幕和背景之间的距离
+        subtitle_cmd = f"subtitles={pw_srt_path}:force_style='Fontsize=14,Fontname=wqy-zenhei,Outline=0,PrimaryColour=&H000000,SecondaryColour=&H000000,Bold=1,MarginV={margin_v}'"
+        bg_position_offset = (int(height) - margin_v) / 1.75
+        background_cmd = f"drawbox=y=(ih-{int(height)}/2-{bg_position_offset}):color=yellow@1.0:width=iw:height={int(height)}/4:t=fill"
+
+        ffmpeg_cmd = [
+            'ffmpeg',
+            '-loop', '1',
+            '-i', jpg_url,  # 输入的图片文件
+            '-i', pw_mp3_path,  # 输入的音频文件
+            '-c:v', 'libx264',  # 视频编码格式
+            '-t', str(pw_duration),  # 输出视频的持续时间，与音频持续时间相同
+            '-pix_fmt', 'yuv420p',  # 像素格式
+            '-c:a', 'aac',  # 音频编码格式
+            '-strict', 'experimental',  # 使用实验性编码器
+            '-shortest',  # 确保输出视频的长度与音频一致
+            '-vf', f"scale=1080x1920,{background_cmd},{subtitle_cmd}",  # 视频过滤器，设置分辨率和其他过滤器
+            pw_url_path  # 输出的视频文件路径
+        ]
+        subprocess.run(ffmpeg_cmd)
+        return pw_url_path
+
+    """
+    设置统一格式拼接视频
+    """
+    @classmethod
+    def concatenate_videos(cls, video_list, video_path_url):
+        concatenate_videos_url = video_path_url + 'concatenate_videos.mp4'
+        # 获取视频的原始宽高信息
+        width, height = cls.get_w_h_size(video_list[0])
+        # 拼接视频
+        VIDEO_COUNTER = 0
+        FF_INPUT = ""
+        FF_SCALE = ""
+        FF_FILTER = ""
+        ffmpeg_cmd = ["ffmpeg"]
+        for videos in video_list:
+            # 添加输入文件
+            FF_INPUT += f" -i {videos}"
+            # 为每个视频文件统一长宽，并设置SAR(采样宽高比)
+            FF_SCALE += f"[{VIDEO_COUNTER}:v]scale={int(height)}x{int(width)},setsar=1[v{VIDEO_COUNTER}];"
+            # 为每个视频文件创建一个输入流，并添加到-filter_complex参数中
+            FF_FILTER += f"[v{VIDEO_COUNTER}][{VIDEO_COUNTER}:a]"
+            # 增加视频计数器
+            VIDEO_COUNTER += 1
+        # 构建最终的FFmpeg命令
+        ffmpeg_cmd.extend(FF_INPUT.split())
+        ffmpeg_cmd.extend(["-filter_complex", f"{FF_SCALE}{FF_FILTER}concat=n={VIDEO_COUNTER}:v=1:a=1[v][a]",
+                           "-map", "[v]", "-map", "[a]", "-y", concatenate_videos_url])
+        subprocess.run(ffmpeg_cmd)
+        return concatenate_videos_url
+
+
+    """
+    单个视频拼接
+    """
+    @classmethod
+    def single_video(cls, new_video_path, video_share, video_path_url, zm):
+        single_video_url = video_path_url + 'single_video.mp4'
+        single_video_srt = video_path_url + 'single_video.srt'
+        # 获取时长
+        duration = cls.get_video_duration(new_video_path)
+        start_time = cls.seconds_to_srt_time(0)
+        end_time = cls.seconds_to_srt_time(duration)
+        single_video_txt = video_path_url + 'single_video.txt'
+        with open(single_video_txt, 'w') as f:
+            f.write(f"file '{new_video_path}'\n")
+        with open(single_video_srt, 'w') as f:
+            f.write(f"1\n{start_time} --> {end_time}\n\u2764\uFE0F{zm}\n\n")
+        width, height = cls.get_w_h_size(new_video_path)
+        box_height = int(int(height) / 4)  # 框的高度为视频高度的四分之一
+
+        background_cmd = f"drawbox=y=ih-{70 + box_height}-{int(box_height / 20)}:color=yellow@1.0:width=iw:height={box_height}:t=fill"
+
+        if video_share == '有':
+            # 添加字幕 wqy-zenhei  Hiragino Sans GB
+            subtitle_cmd = f"subtitles={single_video_srt}:force_style='Fontsize=14,Fontname=wqy-zenhei,Outline=0,PrimaryColour=&H000000,SecondaryColour=&H000000,Bold=1,MarginV=20'"
+            draw = f"{background_cmd},{subtitle_cmd}"
+        else:
+            subtitle_cmd = f"subtitles={single_video_srt}:force_style='Fontsize=14,Fontname=wqy-zenhei,Outline=2,PrimaryColour=&H00FFFF,SecondaryColour=&H000000,Bold=1,MarginV=20'"
+            draw = f"{subtitle_cmd}"
+        # 多线程数
+        num_threads = 4
+        # 构建 FFmpeg 命令,生成视频
+        ffmpeg_cmd_oss = [
+                "ffmpeg",
+                "-f", "concat",
+                "-safe", "0",
+                "-i",  f"{single_video_txt}",
+                "-c:v", "libx264",
+                "-c:a", "aac",
+                "-threads", str(num_threads),
+                "-vf", f"{draw}",
+                 "-y",
+                 single_video_url
+
+        ]
+        subprocess.run(ffmpeg_cmd_oss)
+        return single_video_url
+
+
+
+
+
+

common/mysql_db.py

@@ -0,0 +1,67 @@
+# -*- coding: utf-8 -*-
+"""
+数据库连接及操作
+"""
+import redis
+import pymysql
+from common.common_log import Common
+# from common import Common
+
+class MysqlHelper:
+    @classmethod
+    def connect_mysql(cls):
+        # 创建一个 Connection 对象，代表了一个数据库连接
+        connection = pymysql.connect(
+            host="rm-bp1159bu17li9hi94.mysql.rds.aliyuncs.com",  # 数据库IP地址，内网地址
+            # host="rm-bp1159bu17li9hi94ro.mysql.rds.aliyuncs.com",# 数据库IP地址，外网地址
+            port=3306,  # 端口号
+            user="crawler",  # mysql用户名
+            passwd="crawler123456@",  # mysql用户登录密码
+            db="piaoquan-crawler",  # 数据库名
+            # 如果数据库里面的文本是utf8编码的，charset指定是utf8
+            charset="utf8")
+        return connection
+
+    @classmethod
+    def get_values(cls, sql, params):
+        try:
+            # 连接数据库
+            connect = cls.connect_mysql()
+            # 返回一个 Cursor对象
+            mysql = connect.cursor()
+
+            # 执行 sql 语句
+            mysql.execute(sql, params)
+
+            # fetchall方法返回的是一个元组，里面每个元素也是元组，代表一行记录
+            data = mysql.fetchall()
+
+            # 关闭数据库连接
+            connect.close()
+
+            # 返回查询结果，元组
+            return data
+        except Exception as e:
+            print(f"get_values异常:{e}\n")
+
+    @classmethod
+    def update_values(cls, sql):
+        # 连接数据库
+        connect = cls.connect_mysql()
+        # 返回一个 Cursor对象
+        mysql = connect.cursor()
+        try:
+            # 执行 sql 语句
+            res = mysql.execute(sql)
+            # 注意 一定要commit，否则添加数据不生效
+            connect.commit()
+            return res
+        except Exception as e:
+            # 发生错误时回滚
+            connect.rollback()
+        # 关闭数据库连接
+        connect.close()
+
+
+
+

common/pq.py

@@ -1,186 +0,0 @@
-
-import os
-import random
-
-import sys
-import time
-import json
-
-import requests
-from urllib.parse import urlencode
-
-sys.path.append(os.getcwd())
-from common import Common,  Feishu
-
-class PQ():
-    """
-    获取封面
-    """
-    @classmethod
-    def get_cover(cls, uid):
-        time.sleep(1)
-        url = "https://admin.piaoquantv.com/manager/video/multiCover/listV2"
-
-        payload = json.dumps({
-            "videoId": uid,
-            "range": "2h"
-        })
-        headers = {
-            'accept': 'application/json',
-            'accept-language': 'zh-CN,zh;q=0.9',
-            'cache-control': 'no-cache',
-            'content-type': 'application/json',
-            'cookie': 'SESSION=YjU3MzgwNTMtM2QyYi00YjljLWI3YWUtZTBjNWYwMGQzYWNl',
-            'origin': 'https://admin.piaoquantv.com',
-            'pragma': 'no-cache',
-            'priority': 'u=1, i',
-            'sec-ch-ua': '"Chromium";v="124", "Google Chrome";v="124", "Not-A.Brand";v="99"',
-            'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36'
-        }
-
-        response = requests.request("POST", url, headers=headers, data=payload)
-        data = response.json()
-        content = data["content"]
-        if len(content) == 1:
-            return content[0]["coverUrl"]
-        max_share_count = 0
-        selected_cover_url = ""
-        for item in content:
-            share_count = item.get("shareWeight")
-            if share_count is not None and share_count > max_share_count:
-                max_share_count = share_count
-                selected_cover_url = item["coverUrl"]
-            elif share_count == max_share_count and item["createUser"] == "用户":
-                selected_cover_url = item["coverUrl"]
-        return selected_cover_url
-
-    """
-    获取标题
-    """
-    @classmethod
-    def get_title(cls, uid):
-        url = "https://admin.piaoquantv.com/manager/video/multiTitleV2/listV2"
-
-        payload = json.dumps({
-            "videoId": uid,
-            "range": "4h"
-        })
-        headers = {
-            'accept': 'application/json',
-            'accept-language': 'zh-CN,zh;q=0.9',
-            'cache-control': 'no-cache',
-            'content-type': 'application/json',
-            'cookie': 'SESSION=YjU3MzgwNTMtM2QyYi00YjljLWI3YWUtZTBjNWYwMGQzYWNl',
-            'origin': 'https://admin.piaoquantv.com',
-            'pragma': 'no-cache',
-            'priority': 'u=1, i',
-            'sec-ch-ua': '"Chromium";v="124", "Google Chrome";v="124", "Not-A.Brand";v="99"',
-            'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36'
-        }
-        response = requests.request("POST", url, headers=headers, data=payload)
-        data = response.json()
-        content = data["content"]
-        if len(content) == 1:
-            return content[0]["title"]
-        max_share_count = 0
-        selected_title = ""
-        for item in content:
-            share_count = item.get("shareWeight")
-            if share_count is not None and share_count > max_share_count:
-                max_share_count = share_count
-                selected_title = item["title"]
-            elif share_count == max_share_count and item["createUser"] == "用户":
-                selected_title = item["title"]
-        return selected_title
-
-
-    """
-    新生成视频上传到对应账号下
-    """
-    @classmethod
-    def insert_piaoquantv(cls, oss_object_key, audio_title, pq_ids_list, cover, uid):
-        if audio_title == '' or None == audio_title:
-            title = cls.get_title(uid)
-        else:
-            if '/' in audio_title:
-                new_titles = audio_title.split('/')
-            else:
-                new_titles = [audio_title]
-            title = random.choice(new_titles)
-
-        cover_url = ''
-        if None == cover or cover == '':
-            cover_url = cls.get_cover(uid)
-        pq_id_list = random.choice(pq_ids_list)
-        url = "https://vlogapi.piaoquantv.com/longvideoapi/crawler/video/send"
-        headers = {
-            'User-Agent': 'PQSpeed/486 CFNetwork/1410.1 Darwin/22.6.0',
-            'cookie': 'JSESSIONID=4DEA2B5173BB9A9E82DB772C0ACDBC9F; JSESSIONID=D02C334150025222A0B824A98B539B78',
-            'referer': 'http://appspeed.piaoquantv.com',
-            'token': '524a8bc871dbb0f4d4717895083172ab37c02d2f',
-            'accept-language': 'zh-CN,zh-Hans;q=0.9',
-            'Content-Type': 'application/x-www-form-urlencoded'
-        }
-        payload = {
-            'coverImgPath': cover_url,
-            'deviceToken': '9ef064f2f7869b3fd67d6141f8a899175dddc91240971172f1f2a662ef891408',
-            'fileExtensions': 'MP4',
-            'loginUid': pq_id_list,
-            'networkType': 'Wi-Fi',
-            'platform': 'iOS',
-            'requestId': 'fb972cbd4f390afcfd3da1869cd7d001',
-            'sessionId': '362290597725ce1fa870d7be4f46dcc2',
-            'subSessionId': '362290597725ce1fa870d7be4f46dcc2',
-            'title': title,
-            'token': '524a8bc871dbb0f4d4717895083172ab37c02d2f',
-            'uid': pq_id_list,
-            'versionCode': '486',
-            'versionName': '3.4.12',
-            'videoFromScene': '1',
-            'videoPath': oss_object_key,
-            'viewStatus': '1'
-        }
-        encoded_payload = urlencode(payload)
-        response = requests.request("POST", url, headers=headers, data=encoded_payload)
-        data = response.json()
-        code = data["code"]
-        if code == 0:
-            new_video_id = data["data"]["id"]
-            return new_video_id, title
-        else:
-            return None, None
-
-    """
-    获取视频链接
-    """
-    @classmethod
-    def get_audio_url(cls, uid):
-        for i in range(3):
-            url = f"https://admin.piaoquantv.com/manager/video/detail/{uid}"
-            payload = {}
-            headers = {
-                'authority': 'admin.piaoquantv.com',
-                'accept': 'application/json, text/plain, */*',
-                'accept-language': 'zh-CN,zh;q=0.9',
-                'cache-control': 'no-cache',
-                'cookie': 'SESSION=YjU3MzgwNTMtM2QyYi00YjljLWI3YWUtZTBjNWYwMGQzYWNl',
-                'pragma': 'no-cache',
-                'referer': f'https://admin.piaoquantv.com/cms/post-detail/{uid}/detail',
-                'sec-ch-ua': '"Not_A Brand";v="8", "Chromium";v="120", "Google Chrome";v="120"',
-                'sec-ch-ua-mobile': '?0',
-                'sec-ch-ua-platform': '"macOS"',
-                'sec-fetch-dest': 'empty',
-                'sec-fetch-mode': 'cors',
-                'sec-fetch-site': 'same-origin',
-                'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
-            }
-
-            response = requests.request("GET", url, headers=headers, data=payload)
-            data = response.json()
-            code = data["code"]
-            if code != 0:
-               continue
-            audio_url = data["content"]["transedVideoPath"]
-            return audio_url
-        return ""
-

common/scheduling_db.py

@@ -4,7 +4,7 @@
 数据库连接及操作
 """
 import pymysql
-from common.common import Common
+from common.common_log import Common
 # from common import Common
 
 

common/sql_help.py

@@ -1,137 +1,57 @@
-import datetime
 import os
-import random
 import sys
-from datetime import timedelta
+from datetime import datetime, timedelta
+
+
 
 sys.path.append(os.getcwd())
 from datetime import datetime
 from common import MysqlHelper
-from common import Common
-
-
-class sqlHelp():
-
-    @classmethod
-    def get_count_list(cls, name_list):
-        count_list = []
-        current_time = datetime.now()
-        formatted_time = current_time.strftime("%Y-%m-%d")
-        for name in name_list:
-
-            count = f"""SELECT COUNT(*) AS agc_video_deposit FROM ( SELECT audio, account_id FROM agc_video_deposit WHERE time = '{formatted_time}' AND mark = '{name["mark"]}' GROUP BY audio, account_id) AS subquery;"""
-            count = MysqlHelper.get_values(count, "prod")
-            if count == None:
-                count = 0
-            count = str(count).replace('(', '').replace(')', '').replace(',', '')
-            count_list.append(f"{name['mark_name']}生成条数为:{count}条 \n")
-        return count_list
-
-    """
-    获取未使用的视频链接
-    """
-    @classmethod
-    def get_url_list(cls, user_list, mark, limit_count):
-        for i in range(8):
-            user = str(random.choice(user_list))
-            user = user.replace('(', '').replace(')', '').replace(',', '')
-            current_time = datetime.now()
-            three_days_ago = current_time - timedelta(days=1)
-            formatted_current_time = current_time.strftime("%Y-%m-%d")
-            formatted_three_days_ago = three_days_ago.strftime("%Y-%m-%d")
-            url_list = f"""SELECT a.video_id, a.account_id, a.oss_object_key 
-                                                 FROM agc_video_url a 
-                                                 LEFT JOIN agc_video_deposit b 
-                                                 ON a.oss_object_key = b.oss_object_key 
-                                                 AND b.time >= '{formatted_three_days_ago}' 
-                                                 AND b.time <= '{formatted_current_time}' 
-                                                 WHERE b.video_id IS NULL 
-                                                 AND a.account_id = {user} 
-                                                 AND a.status = 1 
-                                                 AND a.mark = '{mark}' 
-                                                 LIMIT {limit_count};"""
-            url_list = MysqlHelper.get_values(url_list, "prod")
-            if url_list:
-                if len(url_list) >= 35:
-                    return url_list, user
-        return None, None
 
-    """
-    获取已入库的用户id
-    """
-    @classmethod
-    def get_user_id(cls, channel_type, mark):
-        account_id = f"""select account_id from agc_video_url where mark = '{mark}' and  oss_object_key LIKE '%{channel_type}%' group by account_id ;"""
-        account_id = MysqlHelper.get_values(account_id, "prod")
-        return account_id
-
-    """
-    获取已入库的用户id
-    """
-    @classmethod
-    def get_link_count(cls, mark, platform):
-        current_time = datetime.now()
-        formatted_time = current_time.strftime("%Y-%m-%d")
-        count = f"""SELECT COUNT(*) AS total_count FROM ( SELECT audio, account_id FROM agc_video_deposit WHERE time = '{formatted_time}' AND platform = '{platform}' and mark = '{mark}' GROUP BY audio, account_id) AS subquery;"""
-        count = MysqlHelper.get_values(count, "prod")
-        if count == None:
-            count = 0
-        count = str(count).replace('(', '').replace(')', '').replace(',', '')
-        return int(count)
-
-    """
-    获取跟随脚本已入库数量
-    """
-    @classmethod
-    def get_link_gs_count(cls, mark):
-        current_time = datetime.now()
-        formatted_time = current_time.strftime("%Y-%m-%d")
-        count = f"""SELECT COUNT(*) AS total_count FROM ( SELECT audio, account_id FROM agc_video_deposit WHERE time = '{formatted_time}' and mark LIKE '%{mark}%' GROUP BY audio, account_id) AS subquery;"""
-        count = MysqlHelper.get_values(count, "prod")
-        if count == None:
-            count = 0
-        count = str(count).replace('(', '').replace(')', '').replace(',', '')
-        return int(count)
-
-    """
-    获取跟随脚本站外已入库数量
-    """
-    @classmethod
-    def get_link_zw_count(cls, mark, platform):
-        current_time = datetime.now()
-        formatted_time = current_time.strftime("%Y-%m-%d")
-        count = f"""SELECT COUNT(*) AS total_count FROM ( SELECT audio, account_id FROM agc_video_deposit WHERE time = '{formatted_time}' and mark = '{mark}' GROUP BY audio, account_id) AS subquery;"""
-        count = MysqlHelper.get_values(count, "prod")
-        if count == None:
-            count = 0
-        count = str(count).replace('(', '').replace(')', '').replace(',', '')
-        return int(count)
-
-    """
-    获取跟随脚本站内已入库数量
-    """
-    @classmethod
-    def get_link_zn_count(cls, mark, platform):
-        current_time = datetime.now()
-        formatted_time = current_time.strftime("%Y-%m-%d")
-        count = f"""SELECT COUNT(*) AS total_count FROM ( SELECT audio, account_id FROM agc_video_deposit WHERE time = '{formatted_time}' AND platform = '{platform}' and mark = '{mark}' GROUP BY audio, account_id) AS subquery;"""
-        count = MysqlHelper.get_values(count, "prod")
-        if count == None:
-            count = 0
-        count = str(count).replace('(', '').replace(')', '').replace(',', '')
-        return int(count)
 
+class sqlCollect():
     """
-    已使用视频链接存表
+    视频信息写入库中
     """
     @classmethod
-    def insert_videoAudio(cls, video_files, uid, platform, mark):
+    def insert_task(cls, task_mark, video_id, mark, channel):
         current_time = datetime.now()
-        formatted_time = current_time.strftime("%Y-%m-%d")
-        for j in video_files:
-            insert_sql = f"""INSERT INTO agc_video_deposit (audio, video_id, account_id, oss_object_key, time, platform, mark) values ('{uid}', '{j[0]}', '{j[1]}', '{j[2]}', '{formatted_time}', '{platform}', '{mark}')"""
-            MysqlHelper.update_values(
-                sql=insert_sql,
-                env="prod",
-                machine="",
-            )
+        formatted_time = current_time.strftime("%Y-%m-%d %H:%M:%S")
+        insert_sql = f"""INSERT INTO pj_video_data (task_name, used_video_id, mark_name, data_time, channel) values ('{task_mark}' ,'{video_id}','{mark}', '{formatted_time}', '{channel}')"""
+        MysqlHelper.update_values(
+            sql=insert_sql
+        )
+
+    """
+    判断该任务id是否用过
+    """
+    @classmethod
+    def is_used(cls, photo_id):
+        sql = """
+            SELECT photo_id
+            FROM ks_category_video
+            WHERE photo_id = %s 
+        """
+        data = MysqlHelper.get_values(sql, (int(photo_id)))
+        if len(data) == 0 or data == ():
+            return True
+        return False
+
+    @classmethod
+    def get_history_id(cls, channel, url):
+        """
+        从数据库表中读取 id
+        """
+        sql = f"""select name_id from accounts where name = %s and platform = %s and useful = 1 limit 1"""
+        data = MysqlHelper.get_values(sql, (url,  channel))
+        if data:
+            return data[0][0]
+        else:
+            return False
+
+    @classmethod
+    def insert_ks_data(cls, account_name, target, channel):
+        insert_sql = f"""INSERT INTO accounts (name, name_id, platform, useful) values ("{account_name}", "{target}", "{channel}", 1 )"""
+        MysqlHelper.update_values(
+            sql=insert_sql
+        )

common/srt.py

@@ -1,64 +0,0 @@
-import re
-
-import requests
-import json
-
-class SRT():
-
-
-    @classmethod
-    # 定义函数去掉文字后的标点符号并添加换行符
-    def process_srt(cls, srt):
-        lines = srt.strip().split('\n')
-        processed_lines = []
-
-        for line in lines:
-            # 匹配字幕编号行
-            if re.match(r'^\d+$', line):
-                processed_lines.append(line)
-            # 匹配时间戳行
-            elif re.match(r'^\d{2}:\d{2}:\d{2}\.\d{1,3}-->\d{2}:\d{2}:\d{2}\.\d{1,3}$', line):
-                processed_lines.append(line.replace('-->', ' --> '))
-            # 处理字幕文本行
-            else:
-                # 去掉末尾的标点符号
-                line = re.sub(r'[，。！？；、]$', '', line)
-                # 添加换行符
-                processed_lines.append(line + '\n')
-
-        return '\n'.join(processed_lines)
-
-    @classmethod
-    def getSrt(cls, mp3_id):
-        url = "http://api-internal.piaoquantv.com/produce-center/srt/get/content"
-
-        payload = json.dumps({
-            "params": {
-                "resourceChannel": "inner",
-                "videoId": mp3_id
-            }
-        })
-        headers = {
-            'User-Agent': 'Apifox/1.0.0 (https://apifox.com)',
-            'Content-Type': 'application/json',
-            'Accept': '*/*',
-            'Host': 'api-internal.piaoquantv.com',
-            'Connection': 'keep-alive'
-        }
-
-        response = requests.request("POST", url, headers=headers, data=payload)
-        data_list = response.json()
-        code = data_list["code"]
-        if code == 0:
-            srt = data_list["data"]
-            if srt:
-                srt = srt.replace("/n", "\n")
-                # srt = re.sub(r'(\w+)([,。!?])', r'\n\n', srt)
-                new_srt = cls.process_srt(srt)
-                return new_srt
-            else:
-                return None
-        else:
-            return None
-
-

data_channel/piaoquan.py

@@ -0,0 +1,115 @@
+import random
+import time
+import json
+import requests
+from urllib.parse import urlencode
+
+from common import Common
+from common.sql_help import sqlCollect
+
+
+class PQ:
+
+    """
+    获取视频链接
+    """
+    @classmethod
+    def get_pw_url(cls, user_id):
+        url = f"https://admin.piaoquantv.com/manager/video/detail/{user_id}"
+        payload = {}
+        headers = {
+            'authority': 'admin.piaoquantv.com',
+            'accept': 'application/json, text/plain, */*',
+            'accept-language': 'zh-CN,zh;q=0.9',
+            'cache-control': 'no-cache',
+            'cookie': 'SESSION=YjU3MzgwNTMtM2QyYi00YjljLWI3YWUtZTBjNWYwMGQzYWNl',
+            'pragma': 'no-cache',
+            'referer': f'https://admin.piaoquantv.com/cms/post-detail/{user_id}/detail',
+            'sec-ch-ua': '"Not_A Brand";v="8", "Chromium";v="120", "Google Chrome";v="120"',
+            'sec-ch-ua-mobile': '?0',
+            'sec-ch-ua-platform': '"macOS"',
+            'sec-fetch-dest': 'empty',
+            'sec-fetch-mode': 'cors',
+            'sec-fetch-site': 'same-origin',
+            'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
+        }
+
+        response = requests.request("GET", url, headers=headers, data=payload)
+        data = response.json()
+        try:
+            video_url = data["content"]["transedVideoPath"]
+            return video_url
+        except Exception as e:
+            Common.logger("video").warning(f"获取视频链接失败:{e}\n")
+            return ""
+
+
+
+
+
+
+
+    """
+    新生成视频上传到对应账号下
+    """
+    @classmethod
+    def insert_piaoquantv(cls, new_video_path, new_title, cover, n_id):
+
+
+        url = "https://vlogapi.piaoquantv.com/longvideoapi/crawler/video/send"
+        headers = {
+            'User-Agent': 'PQSpeed/486 CFNetwork/1410.1 Darwin/22.6.0',
+            'cookie': 'JSESSIONID=4DEA2B5173BB9A9E82DB772C0ACDBC9F; JSESSIONID=D02C334150025222A0B824A98B539B78',
+            'referer': 'http://appspeed.piaoquantv.com',
+            'token': '524a8bc871dbb0f4d4717895083172ab37c02d2f',
+            'accept-language': 'zh-CN,zh-Hans;q=0.9',
+            'Content-Type': 'application/x-www-form-urlencoded'
+        }
+        payload = {
+            'coverImgPath': cover,
+            'deviceToken': '9ef064f2f7869b3fd67d6141f8a899175dddc91240971172f1f2a662ef891408',
+            'fileExtensions': 'MP4',
+            'loginUid': n_id,
+            'networkType': 'Wi-Fi',
+            'platform': 'iOS',
+            'requestId': 'fb972cbd4f390afcfd3da1869cd7d001',
+            'sessionId': '362290597725ce1fa870d7be4f46dcc2',
+            'subSessionId': '362290597725ce1fa870d7be4f46dcc2',
+            'title': new_title,
+            'token': '524a8bc871dbb0f4d4717895083172ab37c02d2f',
+            'uid': n_id,
+            'versionCode': '486',
+            'versionName': '3.4.12',
+            'videoFromScene': '1',
+            'videoPath': new_video_path,
+            'viewStatus': '1'
+        }
+        encoded_payload = urlencode(payload)
+        response = requests.request("POST", url, headers=headers, data=encoded_payload)
+        data = response.json()
+        code = data["code"]
+        if code == 0:
+            new_video_id = data["data"]["id"]
+            return new_video_id
+        else:
+            return ''
+
+
+    """
+    票圈站内视频下载
+    """
+    @classmethod
+    def download_video(cls, video_url, video_path_url, video_id):
+        for i in range(3):
+            payload = {}
+            headers = {}
+            response = requests.request("GET", video_url, headers=headers, data=payload)
+            if response.status_code == 200:
+                # 以二进制写入模式打开文件
+                video = video_path_url + str(video_id) + '.mp4'
+                with open(f"{video}", "wb") as file:
+                    # 将响应内容写入文件
+                    file.write(response.content)
+                time.sleep(5)
+                return video
+        return ''