update

gongzhonghao/gongzhonghao_author/demo.py

@@ -11,6 +11,7 @@ class Demo:
         # cmd = "ps aux | grep Appium.app"
         cmd = 'ps -ef | grep "run_gongzhonghao1_author_scheduling.py" | grep -v "grep"'
         result = os.popen(cmd).read()
+        print(len(result))
         if len(result) == 0:
             print("yes")
 

gongzhonghao/gongzhonghao_main/run_gongzhonghao1_author_scheduling.py

@@ -13,15 +13,15 @@ from gongzhonghao.gongzhonghao_author.gongzhonghao1_author import GongzhonghaoAu
 
 
 def main(log_type, crawler, task, env):
-    while True:
-        cmd = 'ps -ef | grep "run_gongzhonghao1_author_scheduling.py" | grep -v "grep"'
-        result = os.popen(cmd).read()
-        Common.logger(log_type, crawler).info(f"len_result:{len(result)}")
-        if len(result) > 573:
-            Common.logger(log_type, crawler).info("公众号_1抓取未完成，无需启动新进程")
-            time.sleep(1)
-        else:
-            break
+    # while True:
+    #     cmd = 'ps -ef | grep "run_gongzhonghao1_author_scheduling.py" | grep -v "grep"'
+    #     result = os.popen(cmd).read()
+    #     Common.logger(log_type, crawler).info(f"len_result:{len(result)}")
+    #     if len(result) > 573:
+    #         Common.logger(log_type, crawler).info("公众号_1抓取未完成，无需启动新进程")
+    #         time.sleep(1)
+    #     else:
+    #         break
     task_dict = task_fun(task)['task_dict']
     rule_dict = task_fun(task)['rule_dict']
     Common.logger(log_type, crawler).info(f"调度任务:{task_dict}")

gongzhonghao/gongzhonghao_main/run_gongzhonghao2_author_scheduling.py

@@ -12,15 +12,15 @@ from gongzhonghao.gongzhonghao_author.gongzhonghao2_author import GongzhonghaoAu
 
 
 def main(log_type, crawler, task, env):
-    while True:
-        cmd = 'ps -ef | grep "run_gongzhonghao2_author_scheduling.py" | grep -v "grep"'
-        result = os.popen(cmd).read()
-        Common.logger(log_type, crawler).info(f"len_result:{len(result)}")
-        if len(result) > 573:
-            Common.logger(log_type, crawler).info("公众号_2抓取未完成，无需启动新进程")
-            time.sleep(1)
-        else:
-            break
+    # while True:
+    #     cmd = 'ps -ef | grep "run_gongzhonghao2_author_scheduling.py" | grep -v "grep"'
+    #     result = os.popen(cmd).read()
+    #     Common.logger(log_type, crawler).info(f"len_result:{len(result)}")
+    #     if len(result) > 573:
+    #         Common.logger(log_type, crawler).info("公众号_2抓取未完成，无需启动新进程")
+    #         time.sleep(1)
+    #     else:
+    #         break
     task_dict = task_fun(task)['task_dict']
     rule_dict = task_fun(task)['rule_dict']
     Common.logger(log_type, crawler).info(f"调度任务:{task_dict}")

gongzhonghao/gongzhonghao_main/run_gongzhonghao3_author_scheduling.py

@@ -12,14 +12,14 @@ from gongzhonghao.gongzhonghao_author.gongzhonghao3_author import GongzhonghaoAu
 
 
 def main(log_type, crawler, task, env):
-    while True:
-        cmd = 'ps -ef | grep "run_gongzhonghao3_author_scheduling.py" | grep -v "grep"'
-        result = os.popen(cmd).read()
-        if len(result) > 573:
-            Common.logger(log_type, crawler).info("公众号_3抓取未完成，无需启动新进程")
-            time.sleep(1)
-        else:
-            break
+    # while True:
+    #     cmd = 'ps -ef | grep "run_gongzhonghao3_author_scheduling.py" | grep -v "grep"'
+    #     result = os.popen(cmd).read()
+    #     if len(result) > 573:
+    #         Common.logger(log_type, crawler).info("公众号_3抓取未完成，无需启动新进程")
+    #         time.sleep(1)
+    #     else:
+    #         break
     task_dict = task_fun(task)['task_dict']
     rule_dict = task_fun(task)['rule_dict']
     Common.logger(log_type, crawler).info(f"调度任务:{task_dict}")

gongzhonghao/gongzhonghao_main/run_gongzhonghao4_author_scheduling.py

@@ -12,14 +12,14 @@ from gongzhonghao.gongzhonghao_author.gongzhonghao4_author import GongzhonghaoAu
 
 
 def main(log_type, crawler, task, env):
-    while True:
-        cmd = 'ps -ef | grep "run_gongzhonghao2_author_scheduling.py" | grep -v "grep"'
-        result = os.popen(cmd).read()
-        if len(result) > 573:
-            Common.logger(log_type, crawler).info("公众号_4抓取未完成，无需启动新进程")
-            time.sleep(1)
-        else:
-            break
+    # while True:
+    #     cmd = 'ps -ef | grep "run_gongzhonghao2_author_scheduling.py" | grep -v "grep"'
+    #     result = os.popen(cmd).read()
+    #     if len(result) > 573:
+    #         Common.logger(log_type, crawler).info("公众号_4抓取未完成，无需启动新进程")
+    #         time.sleep(1)
+    #     else:
+    #         break
     task_dict = task_fun(task)['task_dict']
     rule_dict = task_fun(task)['rule_dict']
     Common.logger(log_type, crawler).info(f"调度任务:{task_dict}")

gongzhonghao/gongzhonghao_main/run_gongzhonghao5_author_scheduling.py

@@ -12,14 +12,14 @@ from gongzhonghao.gongzhonghao_author.gongzhonghao5_author import GongzhonghaoAu
 
 
 def main(log_type, crawler, task, env):
-    while True:
-        cmd = 'ps -ef | grep "run_gongzhonghao5_author_scheduling.py" | grep -v "grep"'
-        result = os.popen(cmd).read()
-        if len(result) > 573:
-            Common.logger(log_type, crawler).info("公众号_5抓取未完成，无需启动新进程")
-            time.sleep(1)
-        else:
-            break
+    # while True:
+    #     cmd = 'ps -ef | grep "run_gongzhonghao5_author_scheduling.py" | grep -v "grep"'
+    #     result = os.popen(cmd).read()
+    #     if len(result) > 573:
+    #         Common.logger(log_type, crawler).info("公众号_5抓取未完成，无需启动新进程")
+    #         time.sleep(1)
+    #     else:
+    #         break
     task_dict = task_fun(task)['task_dict']
     rule_dict = task_fun(task)['rule_dict']
     Common.logger(log_type, crawler).info(f"调度任务:{task_dict}")

scheduling/scheduling_v3/crawler_scheduling_v3.py

@@ -99,6 +99,32 @@ class SchedulingV3:
         else:
             mode = mode
 
+        if spider_name == "run_gongzhonghao1_author_scheduling":
+            cmd = 'ps -ef | grep "run_gongzhonghao1_author_scheduling" | grep -v "grep"'
+            result1 = os.popen(cmd).read()
+            if len(result1) != 0:
+                return
+        if spider_name == "run_gongzhonghao2_author_scheduling":
+            cmd = 'ps -ef | grep "run_gongzhonghao2_author_scheduling" | grep -v "grep"'
+            result2 = os.popen(cmd).read()
+            if len(result2) != 0:
+                return
+        if spider_name == "run_gongzhonghao3_author_scheduling":
+            cmd = 'ps -ef | grep "run_gongzhonghao3_author_scheduling" | grep -v "grep"'
+            result3 = os.popen(cmd).read()
+            if len(result3) != 0:
+                return
+        if spider_name == "run_gongzhonghao4_author_scheduling":
+            cmd = 'ps -ef | grep "run_gongzhonghao4_author_scheduling" | grep -v "grep"'
+            result4 = os.popen(cmd).read()
+            if len(result4) != 0:
+                return
+        if spider_name == "run_gongzhonghao5_author_scheduling":
+            cmd = 'ps -ef | grep "run_gongzhonghao5_author_scheduling" | grep -v "grep"'
+            result5 = os.popen(cmd).read()
+            if len(result5) != 0:
+                return
+
         # 正式环境，调度任务
         Common.logger(log_type, crawler).info(f"开始调度任务")
         task_str = [