福气旺修改抓取频率

fuqiwang/fuqiwang_recommend/fqw_recommend.py

@@ -32,7 +32,7 @@ class FqwRecommend:
         mq = MQ(topic_name="topic_crawler_etl_" + env)
         # uuid1 = str(uuid.uuid1())
         page = 1
-        while True:
+        while page <= 20:
             try:
                 Common.logger(log_type, crawler).info(f"正在抓取第{page}页")
                 Common.logging(log_type, crawler, env, f"正在抓取第{page}页")

gongzhonghao/gongzhonghao_accounts_manage.py

@@ -82,7 +82,6 @@ def run():
 
 
 if __name__ == '__main__':
-    # run()
     schedule.every().day.at("00:05").do(run)
     while True:
         schedule.run_pending()

test.py

@@ -0,0 +1,16 @@
+import os
+
+f = open("crontab.txt", "a+")
+
+for parent, dirs, files in os.walk("./"):
+    if files and "main" in parent:
+        for file in files:
+            if file.startswith("run") and "dev" not in file:
+                file_p = os.path.join(parent, file)
+                proj = file_p.split("/")[1]
+                s = file_p.split("run_")[1].split("_")[0]
+                method = file_p.split("_")[-1][:-3]
+                if method in ["recommend", "hour", "author", "play"]:
+                    line = '* * * * * /usr/bin/sh /root/piaoquan_crawler/main/process_mq.sh "{}" "{}" "{}" "prod"'.format(s, proj, method)
+                    print(line)
+                    f.write(line + "\n")