吉祥幸福

common/common.py

@@ -159,7 +159,6 @@ class Common:
         while True:
             # charles 抓包文件保存目录
             charles_file_dir = f"./{crawler}/chlsfiles/"
-
             if int(len(os.listdir(charles_file_dir))) == 1:
                 Common.logger(log_type, crawler).info("未找到chlsfile文件，等待60s")
                 cls.logging(log_type, crawler, env, "未找到chlsfile文件，等待60s")

kanyikan/kanyikan_main/run_kykzzcl_recommend.py

@@ -73,6 +73,7 @@ def main(log_type, crawler, topic_name, group_id, env):
                 Common.logger(log_type, crawler).info(f"用户列表:{user_list}\n")
                 Common.logger(log_type, crawler).info(f'开始抓取:{task_dict["taskName"]}\n')
                 Common.logging(log_type, crawler, env, f'开始抓取:{task_dict["taskName"]}\n')
+
                 KanyikanRecommend.get_videoList(log_type=log_type,
                                                 crawler=crawler,
                                                 rule_dict=rule_dict,

kanyikan/kanyikan_recommend/kanyikan_recommend_zzcl.py

@@ -30,8 +30,9 @@ class KanyikanRecommend:
     def get_videoList(cls, log_type, crawler, our_uid, rule_dict, env):
         mq = MQ(topic_name="topic_crawler_etl_" + env)
         try:
-            Common.logger(log_type, crawler).info(f"正在抓取列表页")
+            Common.logger(log_type, crawler).info(f"正在抓取列表页{crawler}")
             Common.logging(log_type, crawler, env, f"正在抓取列表页")
+            Common.logger(log_type, crawler).info(f"Test{crawler}")
             session = Common.get_session(log_type, crawler, env)
             if session is None:
                 time.sleep(1)
@@ -62,6 +63,7 @@ class KanyikanRecommend:
             }
             urllib3.disable_warnings()
             response = requests.get(url=url, headers=header, params=params, proxies=proxies, verify=False)
+            # print(response)
             if "data" not in response.text:
                 Common.logger(log_type, crawler).info("获取视频list时，session过期，随机睡眠 31-50 秒")
                 Common.logging(log_type, crawler, env, "获取视频list时，session过期，随机睡眠 31-50 秒")
@@ -164,9 +166,16 @@ class KanyikanRecommend:
 
 
 if __name__ == "__main__":
-    print(get_config_from_mysql(log_type="recommend",
-                                source="kanyikan",
-                                env="dev",
-                                text="filter",
-                                action=""))
-    pass
+    KanyikanRecommend.get_videoList(
+        log_type="recommend",
+        crawler="kanyikan",
+        env="prod",
+        rule_dict={'share_cnt': {'min': 300, 'max': 0}},
+        our_uid=64080779
+    )
+    # print(get_config_from_mysql(log_type="recommend",
+    #                             source="kanyikan",
+    #                             env="dev",
+    #                             text="filter",
+    #                             action=""))
+    # pass

main/process_mq.sh

@@ -6,12 +6,12 @@ log_type=$3 # 爬虫策略
 env=$4      # 环境
 
 if [ ${env} = "dev" ];then
-  piaoquan_crawler_dir=/Users/tzld/Desktop/piaoquan_crawler/
+  piaoquan_crawler_dir=/Users/crawler/Desktop/piaoquan_crawler/
   profile_path=/etc/profile
   python=python3
   log_path=${piaoquan_crawler_dir}main/main_logs/process-mq-$(date +%Y-%m-%d).log
 elif [ ${crawler} = "zfqz" ];then
-  piaoquan_crawler_dir=/Users/piaoquan/Desktop/piaoquan_crawler/
+  piaoquan_crawler_dir=/Users/crawler/Desktop/piaoquan_crawler/
   profile_path=./base_profile
   python=python3
   log_path=${piaoquan_crawler_dir}main/main_logs/process-mq-$(date +%Y-%m-%d).log
@@ -25,6 +25,11 @@ elif [ ${crawler} = "kykjk" ];then
   profile_path=/.base_profile
   python=python3
   log_path=${piaoquan_crawler_dir}main/main_logs/process-mq-$(date +%Y-%m-%d).log
+elif [ ${crawler} = "kykzzcl" ];then
+  piaoquan_crawler_dir=/Users/crawler/Desktop/piaoquan_crawler/
+  profile_path=/.base_profile
+  python=python3
+  log_path=${piaoquan_crawler_dir}main/main_logs/process-mq-$(date +%Y-%m-%d).log
 elif [ ${crawler} = "sph" ] && [ ${log_type} = "search" ];then
   piaoquan_crawler_dir=/Users/piaoquan/Desktop/piaoquan_crawler/
   profile_path=/etc/profile

main/process_offline.sh

@@ -70,7 +70,7 @@ else
 fi
 
 # 吉祥幸福
-if [[ "$time" > "02:00:00"  &&  "$time" < "02:59:59" || "$time" > "14:00:00"  &&  "$time" < "14:59:59" ]];then
+if [[ "$time" > "02:00:00"  &&  "$time" < "02:59:59" || "$time" > "14:00:00"  &&  "$time" < "14:59:59" || "$time" > "19:00:00"  &&  "$time" < "20:59:59" ]];then
   echo "$(date "+%Y-%m-%d %H:%M:%S") 开始启动 吉祥幸福 爬虫脚本任务" >> ${log_path}
   ps aux | grep run_htzf | grep -v grep | awk '{print $2}' | xargs kill -9
   ps aux | grep run_zhongmiaoyinxin | grep -v grep | awk '{print $2}' | xargs kill -9

zhufuquanzi/zhufuquanzi_main/run_zfqz_recommend.py

@@ -3,6 +3,8 @@
 # @Time: 2023/9/7
 import argparse
 import random
+import multiprocessing
+import time
 
 from mq_http_sdk.mq_client import *
 from mq_http_sdk.mq_consumer import *
@@ -15,6 +17,14 @@ from common.scheduling_db import MysqlHelper
 from zhufuquanzi.zhufuquanzi_recommend.zhufuquanzi_recommend2 import ZFQZRecommend
 
 
+def run(args1, args2, args3, args4, args5):
+    ZFQZRecommend.start_wechat(log_type=args1,
+                               crawler=args2,
+                               rule_dict=args3,
+                               our_uid=args4,
+                               env=args5)
+
+
 class ZFQZMain:
     @classmethod
     def zhufuquanzi_main(cls, log_type, crawler, topic_name, group_id, env):
@@ -79,12 +89,33 @@ class ZFQZMain:
                     Common.logger(log_type, crawler).info(f'开始抓取:{task_dict["taskName"]}\n')
                     Common.logging(log_type, crawler, env, f'开始抓取:{task_dict["taskName"]}\n')
 
-                    # 抓取符合规则的视频列表
-                    ZFQZRecommend.start_wechat(log_type=log_type,
-                                               crawler=crawler,
-                                               rule_dict=rule_dict,
-                                               our_uid=our_uid,
-                                               env=env)
+                    process = multiprocessing.Process(
+                        target=run,
+                        args=(log_type, crawler, rule_dict, our_uid, env)
+                    )
+                    process.start()
+                    print("进程开始")
+
+                    while True:
+                        if not process.is_alive():
+
+                            print("进程异常，准备重启")
+                            process.terminate()
+                            os.system("adb forward --remove-all")
+                            time.sleep(60)
+                            process = multiprocessing.Process(
+                                target=run,
+                                args=(log_type, crawler, rule_dict, our_uid, env)
+                            )
+                            process.start()
+                        time.sleep(60)
+
+                    # # 抓取符合规则的视频列表
+                    # ZFQZRecommend.start_wechat(log_type=log_type,
+                    #                            crawler=crawler,
+                    #                            rule_dict=rule_dict,
+                    #                            our_uid=our_uid,
+                    #                            env=env)
                     Common.logger(log_type, crawler).info('抓取一轮结束\n')
                     Common.logging(log_type, crawler, env, '抓取一轮结束\n')
 

zhufuquanzi/zhufuquanzi_recommend/zhufuquanzi_recommend2.py

@@ -32,14 +32,14 @@ class ZFQZRecommend:
         if env == "dev":
             chromedriverExecutable = "/Users/wangkun/Downloads/chromedriver/chromedriver_v111/chromedriver"
         else:
-            chromedriverExecutable = "/Users/piaoquan/Downloads/chromedriver"
+            chromedriverExecutable = "/Users/crawler/Downloads/chromedriver_v111/chromedriver"
 
         Common.logger(log_type, crawler).info("启动微信")
         Common.logging(log_type, crawler, env, '启动微信')
         caps = {
             "platformName": "Android",
             "devicesName": "Android",
-            "platformVersion": "11",
+            # "platformVersion": "11",
             # "udid": "emulator-5554",
             "appPackage": "com.tencent.mm",
             "appActivity": ".ui.LauncherUI",