update

gongzhonghao/gongzhonghao_author/demo.py

@@ -0,0 +1,23 @@
+# -*- coding: utf-8 -*-
+# @Author: wangkun
+# @Time: 2023/5/23
+import os
+import psutil
+
+
+class Demo:
+    @classmethod
+    def demo1(cls):
+        # cmd = "ps aux | grep Appium.app"
+        cmd = 'ps -ef | grep "run_gongzhonghao1_author_scheduling.py" | grep -v "grep"'
+        result = os.popen(cmd).read()
+        if len(result) == 0:
+            print("yes")
+
+        # for process in psutil.process_iter():
+        #     # if "shipinhao" in process.name():
+        #     print(f"{process.pid}, {process.name()}")
+
+
+if __name__ == "__main__":
+    Demo.demo1()

gongzhonghao/gongzhonghao_main/run_gongzhonghao1_author_scheduling.py

@@ -4,6 +4,8 @@
 import argparse
 import os
 import sys
+import time
+
 sys.path.append(os.getcwd())
 from common.public import task_fun
 from common.common import Common
@@ -11,6 +13,14 @@ from gongzhonghao.gongzhonghao_author.gongzhonghao1_author import GongzhonghaoAu
 
 
 def main(log_type, crawler, task, env):
+    while True:
+        cmd = 'ps -ef | grep "run_gongzhonghao1_author_scheduling.py" | grep -v "grep"'
+        result = os.popen(cmd).read()
+        if len(result) != 0:
+            Common.logger(log_type, crawler).info("公众号_1抓取未完成，无需启动新进程")
+            time.sleep(1)
+        else:
+            break
     task_dict = task_fun(task)['task_dict']
     rule_dict = task_fun(task)['rule_dict']
     Common.logger(log_type, crawler).info(f"调度任务:{task_dict}")

gongzhonghao/gongzhonghao_main/run_gongzhonghao2_author_scheduling.py

@@ -4,6 +4,7 @@
 import argparse
 import os
 import sys
+import time
 sys.path.append(os.getcwd())
 from common.public import task_fun
 from common.common import Common
@@ -11,6 +12,14 @@ from gongzhonghao.gongzhonghao_author.gongzhonghao2_author import GongzhonghaoAu
 
 
 def main(log_type, crawler, task, env):
+    while True:
+        cmd = 'ps -ef | grep "run_gongzhonghao2_author_scheduling.py" | grep -v "grep"'
+        result = os.popen(cmd).read()
+        if len(result) != 0:
+            Common.logger(log_type, crawler).info("公众号_2抓取未完成，无需启动新进程")
+            time.sleep(1)
+        else:
+            break
     task_dict = task_fun(task)['task_dict']
     rule_dict = task_fun(task)['rule_dict']
     Common.logger(log_type, crawler).info(f"调度任务:{task_dict}")

gongzhonghao/gongzhonghao_main/run_gongzhonghao3_author_scheduling.py

@@ -4,6 +4,7 @@
 import argparse
 import os
 import sys
+import time
 sys.path.append(os.getcwd())
 from common.public import task_fun
 from common.common import Common
@@ -11,6 +12,14 @@ from gongzhonghao.gongzhonghao_author.gongzhonghao3_author import GongzhonghaoAu
 
 
 def main(log_type, crawler, task, env):
+    while True:
+        cmd = 'ps -ef | grep "run_gongzhonghao3_author_scheduling.py" | grep -v "grep"'
+        result = os.popen(cmd).read()
+        if len(result) != 0:
+            Common.logger(log_type, crawler).info("公众号_3抓取未完成，无需启动新进程")
+            time.sleep(1)
+        else:
+            break
     task_dict = task_fun(task)['task_dict']
     rule_dict = task_fun(task)['rule_dict']
     Common.logger(log_type, crawler).info(f"调度任务:{task_dict}")

gongzhonghao/gongzhonghao_main/run_gongzhonghao4_author_scheduling.py

@@ -4,6 +4,7 @@
 import argparse
 import os
 import sys
+import time
 sys.path.append(os.getcwd())
 from common.public import task_fun
 from common.common import Common
@@ -12,6 +13,14 @@ from gongzhonghao.gongzhonghao_author.gongzhonghao4_author import GongzhonghaoAu
 
 
 def main(log_type, crawler, task, oss_endpoint, env):
+    while True:
+        cmd = 'ps -ef | grep "run_gongzhonghao2_author_scheduling.py" | grep -v "grep"'
+        result = os.popen(cmd).read()
+        if len(result) != 0:
+            Common.logger(log_type, crawler).info("公众号_4抓取未完成，无需启动新进程")
+            time.sleep(1)
+        else:
+            break
     task_dict = task_fun(task)['task_dict']
     rule_dict = task_fun(task)['rule_dict']
     task_id = task_dict['task_id']

gongzhonghao/gongzhonghao_main/run_gongzhonghao5_author_scheduling.py

@@ -4,6 +4,7 @@
 import argparse
 import os
 import sys
+import time
 sys.path.append(os.getcwd())
 from common.public import task_fun
 from common.common import Common
@@ -12,6 +13,14 @@ from gongzhonghao.gongzhonghao_author.gongzhonghao5_author import GongzhonghaoAu
 
 
 def main(log_type, crawler, task, oss_endpoint, env):
+    while True:
+        cmd = 'ps -ef | grep "run_gongzhonghao5_author_scheduling.py" | grep -v "grep"'
+        result = os.popen(cmd).read()
+        if len(result) != 0:
+            Common.logger(log_type, crawler).info("公众号_5抓取未完成，无需启动新进程")
+            time.sleep(1)
+        else:
+            break
     task_dict = task_fun(task)['task_dict']
     rule_dict = task_fun(task)['rule_dict']
     task_id = task_dict['task_id']