|
@@ -13,15 +13,15 @@ from gongzhonghao.gongzhonghao_author.gongzhonghao1_author import GongzhonghaoAu
|
|
|
|
|
|
|
|
|
|
def main(log_type, crawler, task, env):
|
|
def main(log_type, crawler, task, env):
|
|
- while True:
|
|
|
|
- cmd = 'ps -ef | grep "run_gongzhonghao1_author_scheduling.py" | grep -v "grep"'
|
|
|
|
- result = os.popen(cmd).read()
|
|
|
|
- Common.logger(log_type, crawler).info(f"len_result:{len(result)}")
|
|
|
|
- if len(result) > 573:
|
|
|
|
- Common.logger(log_type, crawler).info("公众号_1抓取未完成,无需启动新进程")
|
|
|
|
- time.sleep(1)
|
|
|
|
- else:
|
|
|
|
- break
|
|
|
|
|
|
+ # while True:
|
|
|
|
+ # cmd = 'ps -ef | grep "run_gongzhonghao1_author_scheduling.py" | grep -v "grep"'
|
|
|
|
+ # result = os.popen(cmd).read()
|
|
|
|
+ # Common.logger(log_type, crawler).info(f"len_result:{len(result)}")
|
|
|
|
+ # if len(result) > 573:
|
|
|
|
+ # Common.logger(log_type, crawler).info("公众号_1抓取未完成,无需启动新进程")
|
|
|
|
+ # time.sleep(1)
|
|
|
|
+ # else:
|
|
|
|
+ # break
|
|
task_dict = task_fun(task)['task_dict']
|
|
task_dict = task_fun(task)['task_dict']
|
|
rule_dict = task_fun(task)['rule_dict']
|
|
rule_dict = task_fun(task)['rule_dict']
|
|
Common.logger(log_type, crawler).info(f"调度任务:{task_dict}")
|
|
Common.logger(log_type, crawler).info(f"调度任务:{task_dict}")
|