小年糕线下 代码优化

jixiangxingfu/jixiangxingfu_recommend/jixiangxingfu_recommend.py

@@ -55,41 +55,33 @@ class JixiangxingfuRecommend:
     def start_wechat(cls, log_type, crawler, env):
         try:
             if env == "dev":
-                chromedriverExecutable = '/Users/wangkun/Downloads/chromedriver/chromedriver_v107/chromedriver'
-                # chromedriverExecutable = '/Users/tzld/Downloads/chromedriver_v111/chromedriver'  # 本地
-
-                # chromedriverExecutable = 'C:\\chromedriver\\chromedriver.exe'  # 阿里云 Windows
+                chromedriverExecutable = "/Users/piaoquan/Downloads/chromedriver"
             else:
-                chromedriverExecutable = '/Users/piaoquan/Downloads/chromedriver'  # Mac 爬虫机器
-                # chromedriverExecutable = '/Users/tzld/Downloads/chromedriver_v111/chromedriver'  # 本地
-
-                # chromedriverExecutable = 'C:\\chromedriver\\chromedriver.exe'  # 阿里云 Windows
+                chromedriverExecutable = "/Users/piaoquan/Downloads/chromedriver"
 
-            Common.logger(log_type, crawler).info('启动微信')
+            Common.logger(log_type, crawler).info("启动微信")
+            # Common.logging(self.log_type, self.crawler, self.env, '启动微信')
+            # 微信的配置文件
             caps = {
-                "platformName": "Android",  # 手机操作系统 Android / iOS
-                "deviceName": "a0a65126",  # 连接的设备名（模拟器或真机），安卓可以随便写
-                # "udid": "emulator-5554",  # 指定 adb devices 中的哪一台设备
-                "platforVersion": "11",  # 手机对应的系统版本
-                "appPackage": "com.tencent.mm",  # 被测APP的包名，乐活圈 Android
-                "appActivity": ".ui.LauncherUI",  # 启动的Activity名
-                "autoGrantPermissions": "true",  # 让 appium 自动授权 base 权限，
-                # 如果 noReset 为 True，则该条不生效（该参数为 Android 独有），对应的值为 True 或 False
-                "unicodekeyboard": True,  # 使用自带输入法，输入中文时填True
-                "resetkeyboard": True,  # 执行完程序恢复原来输入法
-                "noReset": True,  # 不重置APP
-                "printPageSourceOnFailure": True,  # 找不到元素时，appium log 会完整记录当前页面的 pagesource
-                "newCommandTimeout": 6000,  # 初始等待时间
-                "automationName": "UiAutomator2",  # 使用引擎，默认为 Appium，
-                # 其中 Appium、UiAutomator2、Selendroid、Espresso 用于 Android，XCUITest 用于 iOS
+                "platformName": "Android",
+                "devicesName": "Android",
+                # "platformVersion": "11",
+                # "udid": "emulator-5554",
+                "appPackage": "com.tencent.mm",
+                "appActivity": ".ui.LauncherUI",
+                "autoGrantPermissions": "true",
+                "noReset": True,
+                "resetkeyboard": True,
+                "unicodekeyboard": True,
                 "showChromedriverLog": True,
-                'enableWebviewDetailsCollection': True,
-                'setWebContentsDebuggingEnabled': True,
-                'recreateChromeDriverSessions': True,
-                'chromedriverExecutable': chromedriverExecutable,
+                "printPageSourceOnFailure": True,
+                "recreateChromeDriverSessions": True,
+                "enableWebviewDetailsCollection": True,
+                "setWebContentsDebuggingEnabled": True,
+                "newCommandTimeout": 6000,
+                "automationName": "UiAutomator2",
+                "chromedriverExecutable": chromedriverExecutable,
                 "chromeOptions": {"androidProcess": "com.tencent.mm:appbrand0"},
-                # "chromeOptions": {"androidProcess": "com.tencent.mm:tools"},
-                'browserName': ''
             }
             try:
                 driver = webdriver.Remote("http://localhost:4723/wd/hub", caps)
@@ -128,7 +120,7 @@ class JixiangxingfuRecommend:
                         )
                         size = driver.get_window_size()
                         driver.swipe(int(size['width'] * 0.5), int(size['height'] * 0.8),
-                                          int(size['width'] * 0.5), int(size['height'] * 0.2), 200)
+                                          int(size['width'] * 0.5), int(size['height'] * 0.2), 100)
                         # driver.find_element(By.ID, 'com.android.systemui:id/dismiss_view').click()
                     else:
                         pass
@@ -144,11 +136,11 @@ class JixiangxingfuRecommend:
             )
             size = driver.get_window_size()
             driver.swipe(int(size['width'] * 0.5), int(size['height'] * 0.2), int(size['width'] * 0.5),
-                         int(size['height'] * 0.8), 200)
+                         int(size['height'] * 0.8), 100)
             # 打开小程序"祝福大家好才是真好"
             time.sleep(5)
             Common.logger(log_type, crawler).info('打开小程序"祝福每天好运来相伴"')
-            driver.find_elements(By.XPATH, '//*[@text="祝福每天好运来相伴"]')[-1].click()
+            driver.find_elements(By.XPATH, '//*[@text="祝福大家发财幸福噢"]')[-1].click()
             AliyunLogger.logging(
                 code="1000",
                 platform=JixiangxingfuRecommend.platform,

main/process_offline.sh

@@ -56,7 +56,7 @@ else
 fi
 
 # 祝福圈子
-if [[ "$time" > "00:05:00"  &&  "$time" < "08:59:59" || "$time" > "17:00:00"  &&  "$time" < "19:59:59" ]];then
+if [[ "$time" > "00:05:00"  &&  "$time" < "08:59:59" || "$time" > "17:00:00"  &&  "$time" < "18:59:59" ]];then
   echo "$(date "+%Y-%m-%d %H:%M:%S") 开始启动 祝福圈子 爬虫脚本任务" >> ${log_path}
   ps aux | grep run_xngplus | grep -v grep | awk '{print $2}' | xargs kill -9
   ps aux | grep run_xngrule | grep -v grep | awk '{print $2}' | xargs kill -9
@@ -77,6 +77,27 @@ else
   echo "$(date "+%Y-%m-%d %H:%M:%S") 祝福圈子 爬虫脚本任务结束" >> ${log_path}
 fi
 
+# 小年糕-rule
+if [[ "$time" > "19:00:00"  &&  "$time" < "19:59:59" ]];then
+  echo "$(date "+%Y-%m-%d %H:%M:%S") 开始启动 小年糕-rule 爬虫脚本任务" >> ${log_path}
+  ps aux | grep run_zfqz | grep -v grep | awk '{print $2}' | xargs kill -9
+  ps aux | grep run_xngplus | grep -v grep | awk '{print $2}' | xargs kill -9
+  ps -ef | grep "run_xngrule_recommend.py" | grep -v "grep"
+  if [ "$?" -eq 1 ];then
+    echo "$(date "+%Y-%m-%d %H:%M:%S") 小年糕-rule 小程序爬虫, 异常停止, 正在重启!" >> ${log_path}
+    adb forward --remove-all
+    cd ${piaoquan_crawler_dir}
+    nohup python3 -u xiaoniangaoplus/xiaoniangaoplus_main/run_xngrule_recommend.py --log_type="recommend" --crawler="xiaoniangaoplus" --env=${env} >>xiaoniangaoplus/logs/nohup-recommend.log 2>&1 &
+    echo "$(date "+%Y-%m-%d %H:%M:%S") 重启完成!" >> ${log_path}
+  else
+    echo "$(date "+%Y-%m-%d %H:%M:%S") 小年糕-rule 程序爬虫, 进程状态正常" >> ${log_path}
+  fi
+
+else
+  echo "$(date "+%Y-%m-%d %H:%M:%S") 小年糕-rule 爬虫脚本任务结束" >> ${log_path}
+fi
+
+
 # 删除日志
 echo "$(date "+%Y-%m-%d %H:%M:%S") 开始清理 10 天前的日志文件" >> ${log_path}
 find ${piaoquan_crawler_dir}main/main_logs/ -mtime +10 -name "*.log" -exec rm -rf {} \;

xiaoniangaoplus/xiaoniangaoplus/xiaoniangao_plus_get_userid.py

@@ -5,6 +5,8 @@ import os
 import random
 import sys
 import time
+import subprocess
+
 from datetime import date, timedelta
 
 import requests
@@ -98,6 +100,9 @@ class XiaoNianGaoPlusRecommend:
         self.driver.swipe(int(size['width'] * 0.5), int(size['height'] * 0.2),
                           int(size['width'] * 0.5), int(size['height'] * 0.8), 200)
         time.sleep(1)
+        command = 'adb shell service call statusbar 2'
+        process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE)
+        process.communicate()
         Common.logger(self.log_type, self.crawler).info('打开小程序"小年糕+"')
         self.driver.find_elements(By.XPATH, '//*[@text="小年糕+"]')[-1].click()
         time.sleep(5)

zhufuquanzi/zhufuquanzi_main/run_zfqz_recommend.py

@@ -15,7 +15,7 @@ sys.path.append(os.getcwd())
 from common.common import Common
 from common.public import get_consumer, ack_message, task_fun_mq
 from common.scheduling_db import MysqlHelper
-from zhufuquanzi.zhufuquanzi_recommend.zhufuquanzi_recommend_new2 import ZFQZRecommendNew
+from zhufuquanzi.zhufuquanzi_recommend.zhufuquanzi_recommend_new import ZFQZRecommendNew
 from common import AliyunLogger
 
 

zhufuquanzi/zhufuquanzi_recommend/zhufuquanzi_recommend_new.py

@@ -46,7 +46,7 @@ class ZFQZRecommendNew:
         if self.env == "dev":
             chromedriverExecutable = "/Users/tzld/Downloads/chromedriver_mac64/chromedriver"
         else:
-            chromedriverExecutable = "/Users/piaoquan/Downloads/chromedriver"
+            chromedriverExecutable = "/Users/piaoquan/Downloads/chromedriver-mac-x64/chromedriver"
 
         Common.logger(self.log_type, self.crawler).info("启动微信")
         # Common.logging(self.log_type, self.crawler, self.env, '启动微信')
@@ -129,7 +129,7 @@ class ZFQZRecommendNew:
         time.sleep(1)
         time.sleep(1)
         Common.logger(log_type, crawler).info('打开小程序"祝福年糕圈"')
-        self.driver.find_elements(By.XPATH, '//*[@text="祝福年糕圈"]')[-1].click()
+        self.driver.find_elements(By.XPATH, '//*[@text=开心年糕圈"]')[-1].click()
         AliyunLogger.logging(
             code="1000",
             platform=self.platform,

zhufuquanzi/zhufuquanzi_recommend/zhufuquanzi_recommend_new2.py

@@ -6,6 +6,7 @@ import random
 import sys
 import time
 import uuid
+import subprocess
 from datetime import datetime, timedelta
 from hashlib import md5
 
@@ -44,7 +45,7 @@ class ZFQZRecommendNew:
         if self.env == "dev":
             chromedriverExecutable = "/Users/piaoquan/Downloads/chromedriver"
         else:
-            chromedriverExecutable = "/Users/piaoquan/Downloads/chromedriver"
+            chromedriverExecutable = "/Users/piaoquan/Downloads/chromedriver-mac-x64/chromedriver"
 
         Common.logger(self.log_type, self.crawler).info("启动微信")
         # Common.logging(self.log_type, self.crawler, self.env, '启动微信')
@@ -107,7 +108,7 @@ class ZFQZRecommendNew:
                     )
                     size = self.driver.get_window_size()
                     self.driver.swipe(int(size['width'] * 0.5), int(size['height'] * 0.8),
-                                      int(size['width'] * 0.5), int(size['height'] * 0.2), 200)
+                                      int(size['width'] * 0.5), int(size['height'] * 0.2), 100)
                 else:
                     pass
             except Exception as e:
@@ -124,7 +125,10 @@ class ZFQZRecommendNew:
         size = self.driver.get_window_size()
         self.driver.swipe(int(size['width'] * 0.5), int(size['height'] * 0.2),
                           int(size['width'] * 0.5), int(size['height'] * 0.8), 200)
-        time.sleep(1)
+        command = 'adb shell service call statusbar 2'
+        process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE)
+        process.communicate()
+
         time.sleep(1)
         Common.logger(log_type, crawler).info('打开小程序"祝福圈子"')
         self.driver.find_elements(By.XPATH, '//*[@text="祝福圈子"]')[-1].click()
@@ -186,7 +190,7 @@ class ZFQZRecommendNew:
 
         self.check_to_applet(xpath='//*[@class="tags--tag tags--tag-0 tags--checked"]')
         time.sleep(1)
-        name = ["推荐", "春节"]
+        name = ["推荐"]
         selected_text = random.choice(name)
         try:
             self.driver.find_element(By.XPATH, f"//wx-button[contains(., '{selected_text}')]").click()
@@ -257,11 +261,11 @@ class ZFQZRecommendNew:
             Common.logger(self.log_type, self.crawler).info("点击标题")
             video_title_element[0].click()
             time.sleep(5)
-            self.check_to_applet(xpath=r'//*[@class="index--video-item index--video"]')
+            self.check_to_applet(xpath=r'//*[@class="dynamic-index--video-item dynamic-index--video"]')
             Common.logger(self.log_type, self.crawler).info("点击标题完成")
             time.sleep(10)
             video_url_elements = self.search_elements(
-                '//*[@class="index--video-item index--video"]')
+                '//*[@class="dynamic-index--video-item dynamic-index--video"]')
             Common.logger(self.log_type, self.crawler).info(f"{video_url_elements[0].get_attribute('src')}")
             return video_url_elements[0].get_attribute('src')