|
|
@@ -24,7 +24,7 @@ class ZhuWanWuFuSuRecommend(object):
|
|
|
self.env = env
|
|
|
self.download_cnt = 0
|
|
|
self.mq = MQ(topic_name="topic_crawler_etl_" + self.env)
|
|
|
- self.expire_flag = False
|
|
|
+ self.limit_flag = False
|
|
|
self.cryptor = AES()
|
|
|
|
|
|
def get_recommend_list(self):
|
|
|
@@ -141,7 +141,6 @@ class ZhuWanWuFuSuRecommend(object):
|
|
|
proxies=tunnel_proxies())
|
|
|
detail_video = json.loads(self.cryptor.aes_decrypt(detail_response.text))
|
|
|
if detail_video['success']:
|
|
|
- # print(json.dumps(detail_video, ensure_ascii=False, indent=4))
|
|
|
try:
|
|
|
AliyunLogger.logging(
|
|
|
code="1001",
|
|
|
@@ -196,8 +195,7 @@ class ZhuWanWuFuSuRecommend(object):
|
|
|
)
|
|
|
if pipeline.process_item():
|
|
|
self.download_cnt += 1
|
|
|
- print(mq_obj)
|
|
|
- # self.mq.send_msg(mq_obj)
|
|
|
+ self.mq.send_msg(mq_obj)
|
|
|
AliyunLogger.logging(
|
|
|
code="1002",
|
|
|
platform=self.platform,
|
|
|
@@ -206,6 +204,8 @@ class ZhuWanWuFuSuRecommend(object):
|
|
|
message="成功发送至 ETL",
|
|
|
data=mq_obj
|
|
|
)
|
|
|
+ if self.download_cnt >= int(self.rule_dict.get("videos_cnt", {}).get("min", 200)):
|
|
|
+ self.limit_flag = True
|
|
|
|
|
|
def manage_auth_id(self, out_user_id, out_user_name):
|
|
|
"""
|
|
|
@@ -273,6 +273,15 @@ class ZhuWanWuFuSuRecommend(object):
|
|
|
if user_list:
|
|
|
for index, user_id in enumerate(user_list):
|
|
|
try:
|
|
|
+ if self.limit_flag:
|
|
|
+ AliyunLogger.logging(
|
|
|
+ code="2000",
|
|
|
+ platform=self.platform,
|
|
|
+ mode=self.mode,
|
|
|
+ env=self.env,
|
|
|
+ message="本轮已经抓取足够数量的视频"
|
|
|
+ )
|
|
|
+ return
|
|
|
self.get_user_videos(user_id=user_id)
|
|
|
except Exception as e:
|
|
|
AliyunLogger.logging(
|
