修改去重规则，抖音历史数据更换app

common/feishu_form.py

@@ -107,12 +107,15 @@ class Material():
                             "gg_duration_total": gg_duration,
                         }
                         processed_list.append(json.dumps(number_dict, ensure_ascii=False))
-                        if channel_id == "抖音" or channel_id == "快手":
+                        if channel_id == "抖音" or channel_id == "快手" or channel_id == "视频号":
                             if ls_number and ls_number not in {'None', ''}:
                                 if channel_id == "抖音":
                                     new_channel_id = "抖音历史"
                                 if channel_id == "快手":
                                     new_channel_id = "快手历史"
+                                if channel_id == "视频号":
+                                    new_channel_id = "视频号历史"
+
                                 values1 = [new_channel_id, video_id_total, piaoquan_id, video_share, video_ending_total, crop_tool,
                                           gg_duration, title_total]
                                 filtered_values1 = [str(value) for value in values1 if value is not None and value != "None"]

common/sql_help.py

@@ -26,15 +26,15 @@ class sqlCollect():
     判断该任务id是否用过
     """
     @classmethod
-    def is_used(cls, video_id, mark_name, channel):
+    def is_used(cls, task_mark, video_id, mark_name, channel):
         sql = """
             SELECT used_video_id
             FROM pj_video_data
-            WHERE used_video_id = %s  AND mark_name = %s AND channel = %s 
+            WHERE used_video_id = %s AND task_name = %s AND mark_name = %s AND channel = %s 
             ORDER BY data_time DESC
             LIMIT 1
         """
-        data = MysqlHelper.get_values(sql, (str(video_id), mark_name, channel))
+        data = MysqlHelper.get_values(sql, (str(video_id), task_mark, mark_name, channel))
         if len(data) == 0 or data == ():
             return True
         return False

data_channel/douyin.py

@@ -62,7 +62,7 @@ class DY:
                 if entity_type == 4:
                     # is_top = data[i].get('is_top')  # 是否置顶
                     video_id = data[i].get('aweme_id')  # 文章id
-                    status = sqlCollect.is_used(video_id, mark, channel_id)
+                    status = sqlCollect.is_used(task_mark, video_id, mark, channel_id)
                     if status:
                         video_uri = data[i].get('video', {}).get('play_addr', {}).get('uri')
                         ratio = f'{data[i].get("video", {}).get("height")}p'

data_channel/dy_ls.py

@@ -9,6 +9,79 @@ from common.sql_help import sqlCollect
 
 
 class DYLS:
+    @classmethod
+    def get_dy_zr_list(cls, task_mark, url_id, number, mark):
+        url = "http://8.217.190.241:8888/crawler/dou_yin/blogger"
+        list = []
+        next_cursor = ''
+        for i in range(20):
+            try:
+                payload = json.dumps({
+                    "account_id": url_id,
+                    "source": "app",
+                    "sort": "最热",
+                    "cursor": next_cursor
+                })
+                headers = {
+                    'Content-Type': 'application/json'
+                }
+
+                response = requests.request("POST", url, headers=headers, data=payload)
+                time.sleep(random.randint(1, 5))
+                response = response.json()
+                code = response['code']
+                if code != 0:
+                    Common.logger("dy-ls").info(f"抖音历史数据获取失败，接口为/dou_yin/blogge\n")
+                    return list
+                data_list = response['data']
+                next_cursor = str(data_list['next_cursor'])
+                data = data_list['data']
+                for i in range(len(data)):
+                    video_id = data[i].get('aweme_id')  # 文章id
+                    # status = sqlCollect.is_used(task_mark, video_id, mark, "抖音")
+                    # if status:
+                    status = sqlCollect.is_used(task_mark, video_id, mark, "抖音历史")
+                    if status == False:
+                        continue
+                    video_uri = data[i].get('video', {}).get('play_addr', {}).get('uri')
+                    ratio = f'{data[i].get("video", {}).get("height")}p'
+                    video_url = f'https://www.iesdouyin.com/aweme/v1/play/?video_id={video_uri}&ratio={ratio}&line=0'  # 视频链接
+                    digg_count = int(data[i].get('statistics').get('digg_count'))  # 点赞
+                    share_count = int(data[i].get('statistics').get('share_count'))  # 转发
+                    duration = data[i].get('duration')
+                    old_title = data[i].get('desc', "").strip().replace("\n", "") \
+                        .replace("/", "").replace("\\", "").replace("\r", "") \
+                        .replace(":", "").replace("*", "").replace("？", "") \
+                        .replace("?", "").replace('"', "").replace("<", "") \
+                        .replace(">", "").replace("|", "").replace(" ", "") \
+                        .replace("&NBSP", "").replace(".", "。").replace(" ", "") \
+                        .replace("'", "").replace("#", "").replace("Merge", "")
+                    Common.logger("dy-ls").info(
+                        f"扫描：{task_mark},用户主页id:{url_id},视频id{video_id} ,分享:{share_count},点赞{digg_count}")
+                    video_percent = '%.2f' % (int(share_count) / int(digg_count))
+                    special = float(0.25)
+                    duration = duration / 1000
+                    if int(share_count) < 500 or float(video_percent) < special or int(duration) < 30 or int(
+                            duration) > 720:
+                        Common.logger("dy-ls").info(
+                            f"不符合规则：{task_mark},用户主页id:{url_id},视频id{video_id} ,分享:{share_count},点赞{digg_count} ,时长:{int(duration)} ")
+                        continue
+                    cover_url = data[i].get('video').get('cover').get('url_list')[0]  # 视频封面
+                    all_data = {"video_id": video_id, "cover": cover_url, "video_url": video_url, "rule": video_percent,
+                                "old_title": old_title}
+                    list.append(all_data)
+                    if len(list) == int(number):
+                        Common.logger("dy-ls").info(f"获取抖音历史视频总数:{len(list)}\n")
+                        return list
+
+                if next_cursor == False:
+                    return list
+            except Exception as exc:
+                Common.logger("dy-ls").info(f"抖音历史数据获取失败:{exc}\n")
+                return list
+            return list
+        return list
+
 
     @classmethod
     def get_dyls_list(cls, task_mark, url_id, number, mark):
@@ -100,5 +173,5 @@ class DYLS:
 
 
 if __name__ == '__main__':
-    DYLS.get_video("7314923922602954022")
+    DYLS.get_dy_zr_list(1,2,1,3)
     # DYLS.get_dyls_list("1","MS4wLjABAAAA2QEvnEb7cQDAg6vZXq3j8_LlbO_DiturnV7VeybFKY4",1,"1")

data_channel/ks_ls.py

@@ -40,10 +40,10 @@ class KSLS:
                 data_list = data_all_list["data"]
                 for data in data_list:
                     photo_id = data["photo_id"]
-                    status = sqlCollect.is_used( photo_id, mark, "快手")
-                    if status == False:
-                        continue
-                    status = sqlCollect.is_used(photo_id, mark, "快手历史")
+                    # status = sqlCollect.is_used( photo_id, mark, "快手")
+                    # if status == False:
+                    #     continue
+                    status = sqlCollect.is_used(task_mark, photo_id, mark, "快手历史")
                     if status == False:
                         continue
                     view_count = data["view_count"]

data_channel/kuaishou.py

@@ -94,7 +94,7 @@ class KS:
                         video_id = feeds[i].get("photo", {}).get("videoResource").get("h264", {}).get("videoId", "")
                     except KeyError:
                         video_id = feeds[i].get("photo", {}).get("videoResource").get("hevc", {}).get("videoId", "")
-                    status = sqlCollect.is_used(video_id, mark, channel_id)
+                    status = sqlCollect.is_used(task_mark, video_id, mark, channel_id)
                     if status:
                         video_id = feeds[i].get("photo", {}).get("id", "")
                         status = sqlCollect.is_used(video_id, mark, channel_id)

data_channel/piaoquan.py

@@ -73,7 +73,7 @@ class PQ:
             list = []
             video_id = data["content"]["id"]
             if mark:
-                status = sqlCollect.is_used(task_mark, video_id, mark)
+                status = sqlCollect.is_used(task_mark, video_id, mark, "票圈")
             else:
                 status = True
             if status:
@@ -116,7 +116,7 @@ class PQ:
             list = []
             for url in content:
                 video_id = url["id"]
-                status = sqlCollect.is_used(video_id, mark, "票圈")
+                status = sqlCollect.is_used(task_mark, video_id, mark, "票圈")
                 if status:
                     cover = url["coverImgPath"]
                     video_url = url["transedVideoPath"]

data_channel/shipinhao.py

@@ -86,7 +86,7 @@ class SPH:
                     last_buffer = res_json.get('last_buffer')
                     for obj in res_json["UpMasterHomePage"]:
                         objectId = obj['objectId']
-                        status = sqlCollect.is_used(objectId, mark, "视频号")
+                        status = sqlCollect.is_used(task_mark ,objectId, mark, "视频号")
                         if status:
                             objectNonceId = obj['objectNonceId']
                             url = "http://61.48.133.26:30001/GetFinderDownloadAddress"

data_channel/sph_ls.py

@@ -6,12 +6,12 @@ class SPHLS:
 
     @classmethod
     def get_sphls_data(cls, task_mark, url, number, mark):
-        data_list = sqlCollect.sph_data_info_list("郑蓝旗")
+        data_list = sqlCollect.sph_data_info_list(url)
         list = []
         if data_list:
             for data in data_list:
                 video_id = data[0]
-                status = sqlCollect.is_used(video_id, mark, "视频号")
+                status = sqlCollect.is_used(task_mark,video_id, mark, "视频号历史")
                 if status:
                     old_title = data[1]
                     share_cnt = int(data[2])

video_rewriting/video_processor.py

@@ -278,7 +278,7 @@ class VideoProcessor:
         elif channel_id == "单点视频":
             return SPHDD.get_sphdd_data(url)
         elif channel_id == "抖音历史":
-            return DYLS.get_dyls_list(task_mark, url, number, mark)
+            return DYLS.get_dy_zr_list(task_mark, url, number, mark)
         elif channel_id == "快手历史":
             return KSLS.get_ksls_list(task_mark, url, number, mark)
         elif channel_id == "视频号历史":