调试

channel_ks/ks_feed_list.py

@@ -6,7 +6,7 @@ import time
 from urllib.parse import urlencode
 from datetime import datetime, timedelta
 
-from common import Oss
+from common import Oss, Feishu
 from common.sql_help import sqlCollect
 
 headers = {
@@ -124,9 +124,9 @@ class KsFeedVideo:
         value = age_range[5]['value']
         value = int(value.strip('%'))
         if value >= 50:
-            return True, value
-        else:
             return False, value
+        else:
+            return True, value
 
     """
     视频时长转换成秒
@@ -149,50 +149,6 @@ class KsFeedVideo:
             return False
         else:
             return True
-
-    """
-    生成目录
-    """
-    @classmethod
-    def create_folders(cls):
-        video_path_url = "/Users/tzld/Desktop/ks_automation/path_video/"
-        if not os.path.exists(video_path_url):
-            os.makedirs(video_path_url)
-        return video_path_url
-
-    """
-    删除文件
-    """
-    @classmethod
-    def remove_files(cls, video_path_url):
-        if os.path.exists(video_path_url) and os.path.isdir(video_path_url):
-            for root, dirs, files in os.walk(video_path_url):
-                for file in files:
-                    file_path = os.path.join(root, file)
-                    os.remove(file_path)
-                for dir in dirs:
-                    dir_path = os.path.join(root, dir)
-                    os.rmdir(dir_path)
-
-    """
-    视频下载
-    """
-    @classmethod
-    def download_video(cls, video_url, path_url, video_id):
-        for i in range(3):
-            payload = {}
-            headers = {}
-            response = requests.request("GET", video_url, headers=headers, data=payload)
-            if response.status_code == 200:
-                # 以二进制写入模式打开文件
-                video = path_url + str(video_id) + '.mp4'
-                with open(f"{video}", "wb") as file:
-                    # 将响应内容写入文件
-                    file.write(response.content)
-                time.sleep(5)
-                return video
-        return ''
-
     """
     获取票圈ID
     """
@@ -236,7 +192,7 @@ class KsFeedVideo:
     新生成视频上传到对应账号下
     """
     @classmethod
-    def insert_piaoquantv(cls, new_video_path, new_title, n_id):
+    def insert_piaoquantv(cls, new_video_path, new_title, n_id, cover):
 
         url = "https://vlogapi.piaoquantv.com/longvideoapi/crawler/video/send"
         headers = {
@@ -248,6 +204,7 @@ class KsFeedVideo:
             'Content-Type': 'application/x-www-form-urlencoded'
         }
         payload = {
+            'coverImgPath': cover,
             'deviceToken': '9ef064f2f7869b3fd67d6141f8a899175dddc91240971172f1f2a662ef891408',
             'fileExtensions': 'MP4',
             'loginUid': n_id,
@@ -282,51 +239,57 @@ class KsFeedVideo:
             feed_data = json.loads(feed_data)
             feeds = feed_data['feeds']
             for feed in feeds:
-                photo_id = feed["photo_id"]  # 视频ID
+                photo_id = 5241345676200718454
                 status = sqlCollect.is_used(photo_id)
                 if status:
-                    user_name = feed["user_name"]  # 用户名
-                    user_sex = feed["user_sex"]  # 性别 F为女，U为男
-                    time_data = feed["time"]  # 发布时间
-                    caption = feed["caption"]  # 标题
-                    view_count = feed["view_count"]  # 浏览数
-                    like_count = feed["like_count"]  # 点赞数
-                    share_count = feed["share_count"]  # 分享数
-                    duration = feed["duration"]  # 时长/秒
-                    duration = cls.milliseconds_to_seconds(duration)
-                    main_mv_url = feed["main_mv_url"]  # 视频链接
-                    thumbnail_url = feed["thumbnail_url"]  # 视频封面
-                    user_id = feed["user_id"]  # 用户id非用户主页id
-                    time_data = cls.get_video_data(time_data)
-                    if time_data:
-                        continue
-                    video_percent = '%.2f' % (share_count / like_count)
-                    special = float(0.2)
-                    # if float(video_percent) < special or share_count < 2000 or duration < 30 or duration > 6000:
-                    #     continue
-                    value, age = cls.analyze_photo(photo_id)
-                    # path_url = cls.create_folders()  # 创建目录
-                    # video_path = cls.download_video(main_mv_url, path_url, photo_id)
-                    # if not os.path.isfile(video_path):
-                    #     cls.remove_files(path_url)
-                    #     continue
-                    oss_object_key = Oss.channel_upload_oss(main_mv_url, photo_id)
-                    time.sleep(2)
-                    oss_object = oss_object_key.get("oss_object_key")
-                    pq_id = cls.get_id_by_category(category_name)
-                    if pq_id:
-                        video_uid = cls.insert_piaoquantv(oss_object_key, caption, pq_id)
-                        if video_uid:
-                            print(video_uid)
-                            return video_uid
-                    if oss_object:
-                        pass
-                    if value:
-                        pass
+                    continue
+                user_name = feed["user_name"]  # 用户名
+                user_sex = feed["user_sex"]  # 性别 F为女，U为男
+                time_data = feed["time"]  # 发布时间
+                caption = feed["caption"]  # 标题
+                view_count = feed["view_count"]  # 浏览数
+                like_count = feed["like_count"]  # 点赞数
+                share_count = feed["share_count"]  # 分享数
+                duration = feed["duration"]  # 时长/秒
+                duration = cls.milliseconds_to_seconds(duration)
+                main_mv_url = feed["main_mv_url"]  # 视频链接
+                thumbnail_url = feed["thumbnail_url"]  # 视频封面
+                user_id = feed["user_id"]  # 用户id非用户主页id
+                time_status = cls.get_video_data(time_data)
+                if time_status:
+                    sqlCollect.insert_ks_data(user_name, user_sex, time_data, caption, view_count, like_count, share_count, duration, main_mv_url, thumbnail_url, user_id, '1', photo_id, category_name, age=None, oss_object=None, video_uid=None)
+                    continue
+                video_percent = '%.2f' % (share_count / like_count)
+                special = float(0.2)
+                if float(video_percent) < special or share_count < 2000 or duration < 30 or duration > 6000:
+                    sqlCollect.insert_ks_data(user_name, user_sex, time_data, caption, view_count, like_count, share_count, duration, main_mv_url, thumbnail_url, user_id, '1', photo_id, category_name, age=None, oss_object=None, video_uid=None)
+                    continue
+                value, age = cls.analyze_photo(photo_id)
+                if value:
+                    sqlCollect.insert_ks_data(user_name, user_sex, time_data, caption, view_count, like_count,
+                                              share_count, duration, main_mv_url, thumbnail_url, user_id, '1',
+                                              photo_id, category_name, age=None, oss_object=None, video_uid=None)
 
+                    continue
+                oss_object_key = Oss.channel_upload_oss(main_mv_url, photo_id)
+                time.sleep(2)
+                oss_object = oss_object_key.get("oss_object_key")
+                pq_id = cls.get_id_by_category(category_name)
+                if pq_id:
+                    video_uid = cls.insert_piaoquantv(oss_object, caption, pq_id, thumbnail_url)
+                    sqlCollect.insert_ks_data(user_name, user_sex, time_data, caption, view_count,
+                                              like_count, share_count, duration, main_mv_url, thumbnail_url,
+                                              user_id, '0', photo_id, category_name, age, oss_object, video_uid)
+                    current_time = datetime.now()
+                    formatted_time = current_time.strftime("%Y-%m-%d %H:%M:%S")
+                    values = [
+                        [category_name, user_name, user_sex, caption, view_count, like_count, share_count, duration,
+                         main_mv_url, thumbnail_url, user_id, age, pq_id, video_uid, time_data, formatted_time]]
+                    Feishu.insert_columns("PlcisKhObhzmBothRutc65sJnph", "823f74", "ROWS", 1, 2)
+                    time.sleep(0.5)
+                    Feishu.update_values("PlcisKhObhzmBothRutc65sJnph", "823f74", "A2:Z2", values)
+            time.sleep(120)
 
-                    else:
-                        pass
 
 
 

common/aliyun_oss.py

@@ -10,8 +10,8 @@ import requests
 # OSS_BUCKET_PATH = "douyin"
 OSS_ACCESS_KEY_ID = "LTAIP6x1l3DXfSxm"
 OSS_ACCESS_KEY_SECRET = "KbTaM9ars4OX3PMS6Xm7rtxGr1FLon"
-OSS_BUCKET_ENDPOINT = "oss-cn-hangzhou-internal.aliyuncs.com"# 内网地址
-# OSS_BUCKET_ENDPOINT = "oss-cn-hangzhou.aliyuncs.com" # 外网地址
+# OSS_BUCKET_ENDPOINT = "oss-cn-hangzhou-internal.aliyuncs.com"# 内网地址
+OSS_BUCKET_ENDPOINT = "oss-cn-hangzhou.aliyuncs.com" # 外网地址
 OSS_BUCKET_NAME = "art-crawler"
 class Oss():
 
@@ -33,7 +33,7 @@ class Oss():
         file_content = response.content
         content_type = response.headers.get('Content-Type', 'application/octet-stream')
 
-        oss_object_key = f'channel/ksczz/{video_id}'
+        oss_object_key = f'channel/ks_czz/video/{video_id}'
         auth = oss2.Auth(OSS_ACCESS_KEY_ID, OSS_ACCESS_KEY_SECRET)
         bucket = oss2.Bucket(auth, OSS_BUCKET_ENDPOINT, "art-pubbucket")
         response = bucket.put_object(oss_object_key, file_content, headers={'Content-Type': content_type})
@@ -50,7 +50,7 @@ class Oss():
     @classmethod
     def stitching_sync_upload_oss(cls, src_url: str,
                         video_id: str) -> Dict[str, Any]:
-        oss_object_key = f'channel/ks_video/{video_id}'
+        oss_object_key = f'channel/ks_czz/video/{video_id}'
         auth = oss2.Auth(OSS_ACCESS_KEY_ID, OSS_ACCESS_KEY_SECRET)
         bucket = oss2.Bucket(auth, OSS_BUCKET_ENDPOINT, "art-pubbucket")
         response = bucket.put_object_from_file(oss_object_key, src_url)

common/sql_help.py

@@ -1,6 +1,7 @@
 import os
 import sys
 from datetime import datetime, timedelta
+from typing import Optional
 
 
 
@@ -10,18 +11,6 @@ from common import MysqlHelper
 
 
 class sqlCollect():
-    """
-    视频信息写入库中
-    """
-    @classmethod
-    def insert_task(cls, task_mark, video_id, mark, channel):
-        current_time = datetime.now()
-        formatted_time = current_time.strftime("%Y-%m-%d %H:%M:%S")
-        insert_sql = f"""INSERT INTO pj_video_data (task_name, used_video_id, mark_name, data_time, channel) values ('{task_mark}' ,'{video_id}','{mark}', '{formatted_time}', '{channel}')"""
-        MysqlHelper.update_values(
-            sql=insert_sql
-        )
-
     """
     判断该任务id是否用过
     """
@@ -30,28 +19,19 @@ class sqlCollect():
         sql = """
             SELECT photo_id
             FROM ks_category_video
-            WHERE photo_id = %s 
+            WHERE photo_id = %s  
         """
-        data = MysqlHelper.get_values(sql, (int(photo_id)))
+        data = MysqlHelper.get_values(sql, (str(photo_id)))
         if len(data) == 0 or data == ():
-            return True
-        return False
-
-    @classmethod
-    def get_history_id(cls, channel, url):
-        """
-        从数据库表中读取 id
-        """
-        sql = f"""select name_id from accounts where name = %s and platform = %s and useful = 1 limit 1"""
-        data = MysqlHelper.get_values(sql, (url,  channel))
-        if data:
-            return data[0][0]
-        else:
             return False
+        return True
 
     @classmethod
-    def insert_ks_data(cls, account_name, target, channel):
-        insert_sql = f"""INSERT INTO accounts (name, name_id, platform, useful) values ("{account_name}", "{target}", "{channel}", 1 )"""
-        MysqlHelper.update_values(
+    def insert_ks_data(cls, user_name: str, user_sex: str, time_data, caption: str, view_count: str, like_count: str, share_count: str, duration: str, main_mv_url: str, thumbnail_url: str, user_id: str, status: str, photo_id: str,  category_name: str, age: Optional[str] = None, oss_object: Optional[str] = None, video_uid: Optional[str] = None):
+        current_time = datetime.now()
+        formatted_time = current_time.strftime("%Y-%m-%d %H:%M:%S")
+        insert_sql = f"""INSERT INTO ks_category_video (user_name, user_sex, time_data, caption, view_count, like_count, share_count, duration, main_mv_url, thumbnail_url, user_id, status, age_proportion, video_oss_path, pq_video_id, update_time, photo_id, category_name) values ("{user_name}", "{user_sex}", "{time_data}", "{caption}", "{view_count}", "{like_count}", "{share_count}", "{duration}", "{main_mv_url}", "{thumbnail_url}", "{user_id}", "{status}", "{age}", "{oss_object}", "{video_uid}", "{formatted_time}", "{photo_id}", "{category_name}")"""
+        res = MysqlHelper.update_values(
             sql=insert_sql
         )
+        print(res)

main.py

@@ -1,3 +1,21 @@
+import time
+from datetime import datetime
+
+import schedule
+
 from channel_ks.ks_feed_list import KsFeedVideo
 
+def video_start():
+    current_time = datetime.now()
+    formatted_time = current_time.strftime("%Y-%m-%d %H:%M:%S")
+    print(f"开始执行：{formatted_time}")
+    KsFeedVideo.get_data()
+    print(f"执行结束：{formatted_time}")
+
+
+schedule.every(1).hours.do(video_start)
 KsFeedVideo.get_data()
+
+while True:
+    schedule.run_pending()
+    time.sleep(1)