Merge remote-tracking branch 'origin/master'

douyin/douyin_author/douyin_author_scheduling_new.py

@@ -63,20 +63,17 @@ class DouyinauthorScheduling:
             if flag == "V1":
                 special = 0.01
                 rule_dict = {
-                    "like_cnt": {"min": 10000, "max": 0},
-                    'period': {"min": 90, "max": 90},
+                    'period': {"min": 15, "max": 0},
                 }
             elif flag == "V2":
                 special = 0.01
                 rule_dict = {
-                    "like_cnt": {"min": 2000, "max": 0},
-                    'period': {"min": 90, "max": 90},
+                    'period': {"min": 15, "max": 0},
                 }
             elif flag == "V3":
                 special = 0.01
                 rule_dict = {
-                    "like_cnt": {"min": 100, "max": 0},
-                    'period': {"min": 90, "max": 90},
+                    'period': {"min": 15, "max": 0},
                 }
             cookie = cls.get_cookie(log_type, crawler, env)["cookie"]
             if user_dict['link'][0] == "V":
@@ -176,20 +173,20 @@ class DouyinauthorScheduling:
                         comment_count = int(data[i].get('statistics').get('comment_count'))  # 评论
                         # collect_count = data[i].get('statistics').get('collect_count')  # 收藏
                         share_count = int(data[i].get('statistics').get('share_count'))  # 转发
-                        if special != 0:
-                            if share_count != 0:
-                                video_percent = '%.2f' % (share_count / digg_count)
-                                special = float(special)
-                                if float(video_percent) < special:
-                                    Common.logger(log_type, crawler).info(f"不符合条件：分享/点赞-{video_percent}\n")
-                                    AliyunLogger.logging(
-                                        code="2004",
-                                        platform=crawler,
-                                        mode=log_type,
-                                        env=env,
-                                        message=f"不符合条件：分享/点赞-{video_percent},点赞量-{digg_count}\n"
-                                    )
-                                    continue
+                        # if special != 0:
+                        #     if share_count != 0:
+                        #         video_percent = '%.2f' % (share_count / digg_count)
+                        #         special = float(special)
+                        #         if float(video_percent) < special:
+                        #             Common.logger(log_type, crawler).info(f"不符合条件：分享/点赞-{video_percent}\n")
+                        #             AliyunLogger.logging(
+                        #                 code="2004",
+                        #                 platform=crawler,
+                        #                 mode=log_type,
+                        #                 env=env,
+                        #                 message=f"不符合条件：分享/点赞-{video_percent},点赞量-{digg_count}\n"
+                        #             )
+                        #             continue
                         video_dict = {'video_title': video_title,
                                       'video_id': video_id,
                                       'play_cnt': 0,

kuaishou/kuaishou_author/kuaishou_author_scheduling_new.py

@@ -99,20 +99,17 @@ class KuaishouauthorScheduling:
                 flag = user_dict["link"].split("_")[0]
                 if flag == "V1":
                     rule_dict = {
-                        "play_cnt": {"min": 10000, "max": 0},
-                        'period': {"min": 90, "max": 90},
+                        'period': {"min": 15, "max": 0},
                     }
                     special = 0.01
                 elif flag == "V2":
                     rule_dict = {
-                        "play_cnt": {"min": 2000, "max": 0},
-                        'period': {"min": 90, "max": 90},
+                        'period': {"min": 15, "max": 0},
                     }
                     special = 0.01
                 elif flag == "V3":
                     rule_dict = {
-                        "play_cnt": {"min": 100, "max": 0},
-                        'period': {"min": 90, "max": 90},
+                        'period': {"min": 15, "max": 0},
                     }
                     special = 0.01
             else:
@@ -197,7 +194,6 @@ class KuaishouauthorScheduling:
                     env=env,
                     message=f"response:{response.json()}\n"
                 )
-                Feishu.bot(log_type, 'kuaishou', f'快手cookie失效，请及时更换～')
                 return
             elif len(response.json()["data"]["visionProfilePhotoList"]["feeds"]) == 0:
                 Common.logger(log_type, crawler).warning(f"没有更多视频啦 ~\n")
@@ -248,19 +244,19 @@ class KuaishouauthorScheduling:
 
                     viewCount = int(feeds[i].get('photo', {}).get('viewCount', 0))
                     realLikeCount = int(feeds[i].get('photo', {}).get('realLikeCount', 0))
-                    video_percent = '%.2f' % (realLikeCount / viewCount)
-                    if special != 0:
-                        special = float(special)
-                        if float(video_percent) < special:
-                            Common.logger(log_type, crawler).info(f"不符合条件：点赞/播放-{video_percent}\n")
-                            AliyunLogger.logging(
-                                code="2004",
-                                platform=crawler,
-                                mode=log_type,
-                                env=env,
-                                message=f"点赞量：{realLikeCount}\n"
-                            )
-                            continue
+                    # video_percent = '%.2f' % (realLikeCount / viewCount)
+                    # if special != 0:
+                    #     special = float(special)
+                    #     if float(video_percent) < special:
+                    #         Common.logger(log_type, crawler).info(f"不符合条件：点赞/播放-{video_percent}\n")
+                    #         AliyunLogger.logging(
+                    #             code="2004",
+                    #             platform=crawler,
+                    #             mode=log_type,
+                    #             env=env,
+                    #             message=f"点赞量：{realLikeCount}\n"
+                    #         )
+                    #         continue
                     video_dict = {'video_title': video_title,
                                   'video_id': video_id,
                                   'play_cnt': int(feeds[i].get('photo', {}).get('viewCount', 0)),

kuaishou/kuaishou_main/run_ks_author_dev.py

@@ -10,14 +10,14 @@ from kuaishou.kuaishou_author.kuaishou_author_scheduling_new import Kuaishouauth
 
 def kuaishou_recommend_main(log_type, crawler, env):
     Common.logger(log_type, crawler).info("开始抓取:快手账号\n")
-    Common.logging(log_type, crawler, env, "开始抓取:快手账号\n")
+    # Common.logging(log_type, crawler, env, "开始抓取:快手账号\n")
     KuaishouauthorScheduling.get_author_videos(log_type=log_type,
                                                crawler=crawler,
                                                rule_dict={"videos_cnt": {"min": 5, "max": 0},
                                                           "like_cnt": {"min": 100, "max": 0},
                                                           "duration": {"min": 30, "max": 0},
                                                           "period": {"min": 365, "max": 365}},
-                                               user_list=[{"uid": 6267140, "source": "kuaishou", "link": "V2_https://www.kuaishou.com/profile/3xx43zn5t7m64qa",
+                                               user_list=[{"uid": 6267140, "source": "kuaishou", "link": "V3_https://www.kuaishou.com/profile/3x5is4uxbwyvc7m",
                                                            "nick_name": "买两个橘子",
                                                            "avatar_url": "http://rescdn.yishihui.com/user/default/avatar/live/1616555578819_u=1922778943,2660693611&fm=26&gp=0.jpg",
                                                            "mode": "author"},
@@ -28,8 +28,8 @@ def kuaishou_recommend_main(log_type, crawler, env):
                                                env=env)
     Common.del_logs(log_type, crawler)
     Common.logger(log_type, crawler).info("抓取一轮结束\n")
-    Common.logging(log_type, crawler, env, "抓取一轮结束\n")
+    # Common.logging(log_type, crawler, env, "抓取一轮结束\n")
 
 
 if __name__ == "__main__":
-    kuaishou_recommend_main("author", "kuaishou", "dev")
+    kuaishou_recommend_main("author", "kuaishou", "prod")

shipinhao/shipinhao_author/shipinhao_author.py

@@ -237,11 +237,12 @@ class ShiPinHaoAuthor(object):
         }
         fav_count = int(obj['fav_count'])
         if fav_count == 0:
+            fav_count = 1
             divisor_cnt = 0
         else:
-            divisor_cnt = int(share_cnt / (fav_count+1))
+            divisor_cnt = int(share_cnt / fav_count)
         # 视频时长小于30秒 返回
-        if duration < 20:
+        if duration < 15:
             values = [[
                 obj['nickname'],
                 publish_time_str,
@@ -261,48 +262,48 @@ class ShiPinHaoAuthor(object):
             time.sleep(0.5)
             Feishu.update_values(self.platform, 'shipinhao', "Vq7NeH", "A2:Z2", values)
             return True
-        # 分享小于1000 返回
-        if share_cnt < 500:
-            values = [[
-                obj['nickname'],
-                publish_time_str,
-                formatted_time,
-                int(obj['fav_count']),
-                int(obj['comment_count']),
-                int(obj['like_count']),
-                int(obj['forward_count']),
-                divisor_cnt,
-                video_obj.get('title').split("\n")[0].split("#")[0],
-                duration,
-                '否',
-                '分享小于500',
-                video_obj.get('DownloadAddress')
-            ]]
-            Feishu.insert_columns(self.platform, 'shipinhao', "Vq7NeH", "ROWS", 1, 2)
-            time.sleep(0.5)
-            Feishu.update_values(self.platform, 'shipinhao', "Vq7NeH", "A2:Z2", values)
-            return True
-        # 分享小于等于99999
-        if share_cnt <= 99999 and divisor_cnt < 2:
-            values = [[
-                obj['nickname'],
-                publish_time_str,
-                formatted_time,
-                int(obj['fav_count']),
-                int(obj['comment_count']),
-                int(obj['like_count']),
-                int(obj['forward_count']),
-                divisor_cnt,
-                video_obj.get('title').split("\n")[0].split("#")[0],
-                duration,
-                '否',
-                f'分享小于100000,分享/大拇指：{divisor_cnt}',
-                video_obj.get('DownloadAddress')
-            ]]
-            Feishu.insert_columns(self.platform, 'shipinhao', "Vq7NeH", "ROWS", 1, 2)
-            time.sleep(0.5)
-            Feishu.update_values(self.platform, 'shipinhao', "Vq7NeH", "A2:Z2", values)
-            return True
+        # # 分享小于1000 返回
+        # if share_cnt < 500:
+        #     values = [[
+        #         obj['nickname'],
+        #         publish_time_str,
+        #         formatted_time,
+        #         int(obj['fav_count']),
+        #         int(obj['comment_count']),
+        #         int(obj['like_count']),
+        #         int(obj['forward_count']),
+        #         divisor_cnt,
+        #         video_obj.get('title').split("\n")[0].split("#")[0],
+        #         duration,
+        #         '否',
+        #         '分享小于500',
+        #         video_obj.get('DownloadAddress')
+        #     ]]
+        #     Feishu.insert_columns(self.platform, 'shipinhao', "Vq7NeH", "ROWS", 1, 2)
+        #     time.sleep(0.5)
+        #     Feishu.update_values(self.platform, 'shipinhao', "Vq7NeH", "A2:Z2", values)
+        #     return True
+        # # 分享小于等于99999
+        # if share_cnt <= 99999 and divisor_cnt < 1:
+        #     values = [[
+        #         obj['nickname'],
+        #         publish_time_str,
+        #         formatted_time,
+        #         int(obj['fav_count']),
+        #         int(obj['comment_count']),
+        #         int(obj['like_count']),
+        #         int(obj['forward_count']),
+        #         divisor_cnt,
+        #         video_obj.get('title').split("\n")[0].split("#")[0],
+        #         duration,
+        #         '否',
+        #         f'分享小于100000,分享/大拇指：{divisor_cnt}',
+        #         video_obj.get('DownloadAddress')
+        #     ]]
+        #     Feishu.insert_columns(self.platform, 'shipinhao', "Vq7NeH", "ROWS", 1, 2)
+        #     time.sleep(0.5)
+        #     Feishu.update_values(self.platform, 'shipinhao', "Vq7NeH", "A2:Z2", values)
+        #     return True
         pipeline = PiaoQuanPipeline(
             platform=self.platform,
             mode=self.mode,
@@ -385,7 +386,7 @@ if __name__ == "__main__":
 
         platform="shipinhao",
         mode="author",
-        user_dict={"uid": "123456", "link": "心血管王医生", "user_id": "1234565"},
+        user_dict={"uid": "123456", "link": "老赵讲育儿", "user_id": "1234565"},
         rule_dict={},
         env="prod",
     )