update

gongzhonghao/gongzhonghao_main/run_gongzhonghao4_author_scheduling.py

@@ -8,11 +8,10 @@ import time
 sys.path.append(os.getcwd())
 from common.public import task_fun
 from common.common import Common
-from common.scheduling_db import MysqlHelper
 from gongzhonghao.gongzhonghao_author.gongzhonghao4_author import GongzhonghaoAuthor4
 
 
-def main(log_type, crawler, task, oss_endpoint, env):
+def main(log_type, crawler, task, env):
     while True:
         cmd = 'ps -ef | grep "run_gongzhonghao2_author_scheduling.py" | grep -v "grep"'
         result = os.popen(cmd).read()
@@ -23,19 +22,13 @@ def main(log_type, crawler, task, oss_endpoint, env):
             break
     task_dict = task_fun(task)['task_dict']
     rule_dict = task_fun(task)['rule_dict']
-    task_id = task_dict['task_id']
-    select_user_sql = f"""select * from crawler_user_v3 where task_id={task_id}"""
-    user_list = MysqlHelper.get_values(log_type, crawler, select_user_sql, env, action="")
-    Common.logger(log_type, crawler).info(f"调度任务:\n{task_dict}")
-    Common.logger(log_type, crawler).info(f"抓取规则:\n{rule_dict}")
-    Common.logger(log_type, crawler).info(f"用户列表:\n{user_list}")
+    Common.logger(log_type, crawler).info(f"调度任务:{task_dict}")
+    Common.logger(log_type, crawler).info(f"抓取规则:{rule_dict}\n")
     Common.logger(log_type, crawler).info(f'开始抓取 {task_dict["task_name"]} 定向榜\n')
     GongzhonghaoAuthor4.get_all_videos(log_type=log_type,
-                                        crawler=crawler,
-                                        user_list=user_list,
-                                        rule_dict=rule_dict,
-                                        oss_endpoint=oss_endpoint,
-                                        env=env)
+                                       crawler=crawler,
+                                       rule_dict=rule_dict,
+                                       env=env)
     Common.del_logs(log_type, crawler)
     Common.logger(log_type, crawler).info('抓取完一轮\n')
 
@@ -45,11 +38,9 @@ if __name__ == "__main__":
     parser.add_argument('--log_type', type=str)  ## 添加参数,注明参数类型
     parser.add_argument('--crawler')  ## 添加参数
     parser.add_argument('--task')  ## 添加参数
-    parser.add_argument('--oss_endpoint')  ## 添加参数
     parser.add_argument('--env')  ## 添加参数
     args = parser.parse_args()  ### 参数赋值，也可以通过终端赋值
     main(log_type=args.log_type,
          crawler=args.crawler,
          task=args.task,
-         oss_endpoint=args.oss_endpoint,
          env=args.env)

gongzhonghao/gongzhonghao_main/run_gongzhonghao5_author_scheduling.py

@@ -8,11 +8,10 @@ import time
 sys.path.append(os.getcwd())
 from common.public import task_fun
 from common.common import Common
-from common.scheduling_db import MysqlHelper
 from gongzhonghao.gongzhonghao_author.gongzhonghao5_author import GongzhonghaoAuthor5
 
 
-def main(log_type, crawler, task, oss_endpoint, env):
+def main(log_type, crawler, task, env):
     while True:
         cmd = 'ps -ef | grep "run_gongzhonghao5_author_scheduling.py" | grep -v "grep"'
         result = os.popen(cmd).read()
@@ -23,19 +22,13 @@ def main(log_type, crawler, task, oss_endpoint, env):
             break
     task_dict = task_fun(task)['task_dict']
     rule_dict = task_fun(task)['rule_dict']
-    task_id = task_dict['task_id']
-    select_user_sql = f"""select * from crawler_user_v3 where task_id={task_id}"""
-    user_list = MysqlHelper.get_values(log_type, crawler, select_user_sql, env, action="")
-    Common.logger(log_type, crawler).info(f"调度任务:\n{task_dict}")
-    Common.logger(log_type, crawler).info(f"抓取规则:\n{rule_dict}")
-    Common.logger(log_type, crawler).info(f"用户列表:\n{user_list}")
+    Common.logger(log_type, crawler).info(f"调度任务:{task_dict}")
+    Common.logger(log_type, crawler).info(f"抓取规则:{rule_dict}\n")
     Common.logger(log_type, crawler).info(f'开始抓取 {task_dict["task_name"]} 定向榜\n')
     GongzhonghaoAuthor5.get_all_videos(log_type=log_type,
-                                        crawler=crawler,
-                                        user_list=user_list,
-                                        rule_dict=rule_dict,
-                                        oss_endpoint=oss_endpoint,
-                                        env=env)
+                                       crawler=crawler,
+                                       rule_dict=rule_dict,
+                                       env=env)
     Common.del_logs(log_type, crawler)
     Common.logger(log_type, crawler).info('抓取完一轮\n')
 
@@ -45,11 +38,9 @@ if __name__ == "__main__":
     parser.add_argument('--log_type', type=str)  ## 添加参数,注明参数类型
     parser.add_argument('--crawler')  ## 添加参数
     parser.add_argument('--task')  ## 添加参数
-    parser.add_argument('--oss_endpoint')  ## 添加参数
     parser.add_argument('--env')  ## 添加参数
     args = parser.parse_args()  ### 参数赋值，也可以通过终端赋值
     main(log_type=args.log_type,
          crawler=args.crawler,
          task=args.task,
-         oss_endpoint=args.oss_endpoint,
          env=args.env)