修复 limit方法

common/limit.py

@@ -1,8 +1,12 @@
+import os
+import sys
 import json
-import time
 import requests
 from datetime import datetime, timedelta
-from .aliyun_log import AliyunLogger
+
+sys.path.append(os.getcwd())
+
+from common.aliyun_log import AliyunLogger
 from common.scheduling_db import MysqlHelper
 from common.db import RedisClient
 
@@ -110,7 +114,7 @@ class AuthorLimit(object):
                         if tag_count <= 300:
                             tag_count += 1
                             expire_seconds = generate_expire_time()
-                            RedisClient().insert(
+                            R.insert(
                                 key=tag, value=tag_count, expire_time=expire_seconds
                             )
                             return True
@@ -122,7 +126,7 @@ class AuthorLimit(object):
                     else:
                         tag_count = 1
                         expire_seconds = generate_expire_time()
-                        RedisClient().insert(
+                        R.insert(
                             key=tag, value=tag_count, expire_time=expire_seconds
                         )
                         return True

xiaoniangao/xiaoniangao_author/xiaoniangao_author_v2.py

@@ -13,7 +13,7 @@ sys.path.append(os.getcwd())
 from common.common import Common
 from common import AliyunLogger, PiaoQuanPipeline
 from common.public import get_config_from_mysql, clean_title
-# from common.limit import AuthorLimit
+from common.limit import AuthorLimit
 
 
 def tunnel_proxies():
@@ -44,7 +44,7 @@ class XiaoNianGaoAuthor:
         self.download_count = 0
         self.test_account = [58528285, 58527674, 58528085, 58527582, 58527601, 58527612, 58528281, 58528095, 58527323,
                              58528071, 58527278]
-        # self.limiter = AuthorLimit(platform=self.platform, mode=self.mode)
+        self.limiter = AuthorLimit(platform=self.platform, mode=self.mode)
 
     def get_author_list(self):
         # 每轮只抓取定量的数据，到达数量后自己退出
@@ -297,33 +297,33 @@ class XiaoNianGaoAuthor:
             video_dict["user_id"] = user_dict["uid"]
             video_dict["publish_time"] = video_dict["publish_time_str"]
             # print(video_dict)
-            # limit_flag = self.limiter.author_limitation(user_id=video_dict['user_id'])
-            # if limit_flag:
-            self.mq.send_msg(video_dict)
-            self.download_count += 1
-            AliyunLogger.logging(
-                code="1002",
-                platform=self.platform,
-                mode=self.mode,
-                env=self.env,
-                data=video_dict,
-                trace_id=trace_id,
-                message="成功发送 MQ 至 ETL",
-            )
-            Common.logging(
-                log_type=self.mode,
-                crawler=self.platform,
-                env=self.env,
-                message="成功发送 MQ 至 ETL",
-            )
-        # else:
-        #     AliyunLogger.logging(
-        #         code="8808",
-        #         platform=self.platform,
-        #         mode=self.mode,
-        #         env=self.env,
-        #         trace_id=trace_id,
-        #         account=video_dict['user_id'],
-        #         message="监测到个人账号数量超过 300，停止抓取该账号"
-        #     )
+            limit_flag = self.limiter.author_limitation(user_id=video_dict['user_id'])
+            if limit_flag:
+                self.mq.send_msg(video_dict)
+                self.download_count += 1
+                AliyunLogger.logging(
+                    code="1002",
+                    platform=self.platform,
+                    mode=self.mode,
+                    env=self.env,
+                    data=video_dict,
+                    trace_id=trace_id,
+                    message="成功发送 MQ 至 ETL",
+                )
+                Common.logging(
+                    log_type=self.mode,
+                    crawler=self.platform,
+                    env=self.env,
+                    message="成功发送 MQ 至 ETL",
+                )
+            else:
+                AliyunLogger.logging(
+                    code="8808",
+                    platform=self.platform,
+                    mode=self.mode,
+                    env=self.env,
+                    trace_id=trace_id,
+                    account=video_dict['user_id'],
+                    message="监测到个人账号数量超过 300，停止抓取该账号"
+                )
         return True