update

gongzhonghao/gongzhonghao_main/run_gzh_author.py

@@ -10,10 +10,10 @@ sys.path.append(os.getcwd())
 from common.public import task_fun_mq, get_consumer, ack_message
 from common.common import Common
 from common.scheduling_db import MysqlHelper
-from gongzhonghao.gongzhonghao_author.gongzhonghao_author_lock import GongzhonghaoAuthor
+from gongzhonghao.gongzhonghao_author.gongzhonghao_author import GongzhonghaoAuthor
 
 
-def get_author_videos(log_type, crawler, task_dict, rule_dict, user_list, env):
+def get_author_videos(log_type, crawler, token_index, task_dict, rule_dict, user_list, env):
     Common.logger(log_type, crawler).info(f'开始抓取:{task_dict["taskName"]}\n')
     Common.logging(log_type, crawler, env, f'开始抓取:{task_dict["taskName"]}\n')
     Common.logger(log_type, crawler).info(f"user_list:{user_list}")
@@ -21,10 +21,11 @@ def get_author_videos(log_type, crawler, task_dict, rule_dict, user_list, env):
     GongzhonghaoAuthor.get_all_videos(log_type=log_type,
                                       crawler=crawler,
                                       task_dict=task_dict,
+                                      token_index=token_index,
                                       rule_dict=rule_dict,
                                       user_list=user_list,
                                       env=env)
-    # Common.del_logs(log_type, crawler)
+    Common.del_logs(log_type, crawler)
     Common.logger(log_type, crawler).info('抓取一轮结束\n')
     Common.logging(log_type, crawler, env, '抓取一轮结束\n')
 
@@ -102,7 +103,7 @@ def main(log_type, crawler, topic_name, group_id, env):
                     start = i * chunk_size
                     end = min((i + 1) * chunk_size, user_num + 1)
                     process = Process(target=get_author_videos, args=(
-                    f"{log_type}{i + 1}", crawler, task_dict, rule_dict, user_list[start:end], env))
+                    f"{log_type}{i + 1}", crawler, i + 1, task_dict, rule_dict, user_list[start:end], env))
                     process.start()
                     processes.append(process)
 
@@ -134,4 +135,4 @@ if __name__ == "__main__":
          crawler=args.crawler,
          topic_name=args.topic_name,
          group_id=args.group_id,
-         env=args.env)
+         env=args.env)

gongzhonghao/gongzhonghao_main/run_gzh_author_old.py → gongzhonghao/gongzhonghao_main/run_gzh_author_lock.py

@@ -10,10 +10,10 @@ sys.path.append(os.getcwd())
 from common.public import task_fun_mq, get_consumer, ack_message
 from common.common import Common
 from common.scheduling_db import MysqlHelper
-from gongzhonghao.gongzhonghao_author.gongzhonghao_author import GongzhonghaoAuthor
+from gongzhonghao.gongzhonghao_author.gongzhonghao_author_lock import GongzhonghaoAuthor
 
 
-def get_author_videos(log_type, crawler, token_index, task_dict, rule_dict, user_list, env):
+def get_author_videos(log_type, crawler, task_dict, rule_dict, user_list, env):
     Common.logger(log_type, crawler).info(f'开始抓取:{task_dict["taskName"]}\n')
     Common.logging(log_type, crawler, env, f'开始抓取:{task_dict["taskName"]}\n')
     Common.logger(log_type, crawler).info(f"user_list:{user_list}")
@@ -21,11 +21,10 @@ def get_author_videos(log_type, crawler, token_index, task_dict, rule_dict, user
     GongzhonghaoAuthor.get_all_videos(log_type=log_type,
                                       crawler=crawler,
                                       task_dict=task_dict,
-                                      token_index=token_index,
                                       rule_dict=rule_dict,
                                       user_list=user_list,
                                       env=env)
-    Common.del_logs(log_type, crawler)
+    # Common.del_logs(log_type, crawler)
     Common.logger(log_type, crawler).info('抓取一轮结束\n')
     Common.logging(log_type, crawler, env, '抓取一轮结束\n')
 
@@ -103,7 +102,7 @@ def main(log_type, crawler, topic_name, group_id, env):
                     start = i * chunk_size
                     end = min((i + 1) * chunk_size, user_num + 1)
                     process = Process(target=get_author_videos, args=(
-                    f"{log_type}{i + 1}", crawler, i + 1, task_dict, rule_dict, user_list[start:end], env))
+                    f"{log_type}{i + 1}", crawler, task_dict, rule_dict, user_list[start:end], env))
                     process.start()
                     processes.append(process)
 
@@ -135,4 +134,4 @@ if __name__ == "__main__":
          crawler=args.crawler,
          topic_name=args.topic_name,
          group_id=args.group_id,
-         env=args.env)
+         env=args.env)