|
@@ -12,22 +12,22 @@ from xiaoniangao.xiaoniangao_follow.xiaoniangao_follow import XiaoniangaoFollow
|
|
|
|
|
|
def main(log_type, crawler, env):
|
|
def main(log_type, crawler, env):
|
|
while True:
|
|
while True:
|
|
- try:
|
|
|
|
- if env == "dev":
|
|
|
|
- oss_endpoint = "out"
|
|
|
|
- else:
|
|
|
|
- oss_endpoint = "inner"
|
|
|
|
- Common.logger(log_type, crawler).info('开始抓取 小年糕 定向榜\n')
|
|
|
|
- XiaoniangaoFollow.get_follow_videos(log_type=log_type,
|
|
|
|
- crawler=crawler,
|
|
|
|
- strategy="定向爬虫策略",
|
|
|
|
- oss_endpoint=oss_endpoint,
|
|
|
|
- env=env)
|
|
|
|
- Common.del_logs(log_type, crawler)
|
|
|
|
- Common.logger(log_type, crawler).info('抓取完一轮,休眠 1 分钟\n')
|
|
|
|
- time.sleep(60)
|
|
|
|
- except Exception as e:
|
|
|
|
- Common.logger(log_type, crawler).info(f"小年糕定向抓取异常:{e}\n")
|
|
|
|
|
|
+ # try:
|
|
|
|
+ if env == "dev":
|
|
|
|
+ oss_endpoint = "out"
|
|
|
|
+ else:
|
|
|
|
+ oss_endpoint = "inner"
|
|
|
|
+ Common.logger(log_type, crawler).info('开始抓取 小年糕 定向榜\n')
|
|
|
|
+ XiaoniangaoFollow.get_follow_videos(log_type=log_type,
|
|
|
|
+ crawler=crawler,
|
|
|
|
+ strategy="定向爬虫策略",
|
|
|
|
+ oss_endpoint=oss_endpoint,
|
|
|
|
+ env=env)
|
|
|
|
+ Common.del_logs(log_type, crawler)
|
|
|
|
+ Common.logger(log_type, crawler).info('抓取完一轮,休眠 1 分钟\n')
|
|
|
|
+ time.sleep(60)
|
|
|
|
+ # except Exception as e:
|
|
|
|
+ # Common.logger(log_type, crawler).info(f"小年糕定向抓取异常:{e}\n")
|
|
|
|
|
|
|
|
|
|
if __name__ == "__main__":
|
|
if __name__ == "__main__":
|