update

ganggangdouchuan/ganggangdouchuan_main/run_ganggangdouchuan_recommend.py

@@ -11,12 +11,12 @@ from ganggangdouchuan.ganggangdouchuan_recommend.ganggangdouchuan_recommend impo
 
 
 def main(log_type, crawler, env):
-    oss_endpoint = "out"
+    oss_endpoint = "inner"
     Common.logger(log_type, crawler).info('开始抓取:刚刚都传小程序\n')
     Common.logging(log_type, crawler, env, '开始抓取:刚刚都传小程序\n')
     ganggangdouchuan_start_time = int(time.time())
     GanggangdouchuanRecommend.start_wechat(log_type, crawler, oss_endpoint, env)
-    # Common.del_logs(log_type, crawler)
+    Common.del_logs(log_type, crawler)
     Common.logger(log_type, crawler).info('抓取完一轮\n')
     Common.logging(log_type, crawler, env, '抓取完一轮\n')
     ganggangdouchuan_end_time = int(time.time())

ganggangdouchuan/ganggangdouchuan_recommend/ganggangdouchuan_recommend.py

@@ -29,14 +29,16 @@ class GanggangdouchuanRecommend:
             if env == "dev":
                 chromedriverExecutable = '/Users/wangkun/Downloads/chromedriver/chromedriver_v107/chromedriver'
             else:
-                chromedriverExecutable = '/Users/piaoquan/Downloads/chromedriver'
+                # chromedriverExecutable = '/Users/piaoquan/Downloads/chromedriver'  # Mac 爬虫机器
+                chromedriverExecutable = 'C:\\chromedriver\\chromedriver.exe'  # 阿里云 Windows
 
             Common.logger(log_type, crawler).info('启动微信')
             Common.logging(log_type, crawler, env, '启动微信')
             caps = {
                 "platformName": "Android",  # 手机操作系统 Android / iOS
                 "deviceName": "a0a65126",  # 连接的设备名（模拟器或真机），安卓可以随便写
-                "platforVersion": "11",  # 手机对应的系统版本（Android 11）
+                "udid": "emulator-5554",  # 指定 adb devices 中的哪一台设备
+                "platforVersion": "12",  # 手机对应的系统版本
                 "appPackage": "com.tencent.mm",  # 被测APP的包名，乐活圈 Android
                 "appActivity": ".ui.LauncherUI",  # 启动的Activity名
                 "autoGrantPermissions": "true",  # 让 appium 自动授权 base 权限，
@@ -56,8 +58,8 @@ class GanggangdouchuanRecommend:
                 "chromeOptions": {"androidProcess": "com.tencent.mm:appbrand0"},
                 'browserName': ''
             }
-            driver = webdriver.Remote("http://localhost:4723/wd/hub", caps)
-            driver.implicitly_wait(20)
+            driver = webdriver.Remote("http://localhost:4723/wd/hub", caps)  # 指定 Appium 端口号：4723
+            driver.implicitly_wait(30)
             # 向下滑动页面，展示出小程序选择面板
             for i in range(120):
                 try:
@@ -79,11 +81,11 @@ class GanggangdouchuanRecommend:
             driver.swipe(int(size['width'] * 0.5), int(size['height'] * 0.2),
                          int(size['width'] * 0.5), int(size['height'] * 0.8), 200)
             # 打开小程序"刚刚都传"
-            time.sleep(3)
+            time.sleep(5)
             Common.logger(log_type, crawler).info('打开小程序"刚刚都传"')
             Common.logging(log_type, crawler, env, '打开小程序"刚刚都传"')
             driver.find_elements(By.XPATH, '//*[@text="刚刚都传"]')[-1].click()
-            time.sleep(10)
+            time.sleep(15)
             cls.get_videoList(log_type, crawler, oss_endpoint, env, driver)
             driver.quit()
             Common.logger(log_type, crawler).info('退出微信成功\n')
@@ -91,8 +93,6 @@ class GanggangdouchuanRecommend:
         except Exception as e:
             Common.logger(log_type, crawler).error(f'start_wechat异常:{e}\n')
             Common.logging(log_type, crawler, env, f'start_wechat异常:{e}\n')
-            cmd = "adb kill-server && adb start-server"
-            os.system(cmd)
 
     @classmethod
     def get_video_url(cls, log_type, crawler, env, driver: WebDriver, video_element):
@@ -177,11 +177,12 @@ class GanggangdouchuanRecommend:
 
     @classmethod
     def get_videoList(cls, log_type, crawler, oss_endpoint, env, driver: WebDriver):
+        driver.implicitly_wait(20)
         # 切换到小程序
         cls.check_to_applet(log_type, crawler, env, driver)
-        time.sleep(1)
-        index = 0
 
+        time.sleep(5)
+        index = 0
         while True:
             try:
                 if cls.search_elements(log_type, crawler, env, driver, '//wx-view[@class="lists"]') is None:
@@ -365,6 +366,5 @@ class GanggangdouchuanRecommend:
 
 
 if __name__ == '__main__':
-    GanggangdouchuanRecommend.start_wechat('recommend', 'ganggangdouchuan', 'out', 'dev')
-
+    GanggangdouchuanRecommend.start_wechat('recommend', 'ganggangdouchuan', 'inner', 'dev')
     pass

gongzhonghao/gongzhonghao_author/gongzhonghao_author.py

@@ -255,7 +255,9 @@ class GongzhonghaoAuthor:
                 for article in app_msg_list:
                     try:
                         create_time = article.get('create_time', 0)
+                        update_time = article.get('update_time', 0)
                         publish_time_stamp = int(create_time)
+                        update_time_stamp = int(update_time)
                         publish_time_str = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(publish_time_stamp))
                         article_url = article.get('link', '')
                         video_dict = {
@@ -279,7 +281,8 @@ class GongzhonghaoAuthor:
                             Common.logger(log_type, crawler).info(f"{k}:{v}")
                         Common.logging(log_type, crawler, env, f'video_dict:{video_dict}')
 
-                        if int(time.time()) - publish_time_stamp > 3600 * 24 * int(rule_dict.get('period', {}).get('max', 1000)):
+                        if (int(time.time()) - publish_time_stamp > 3600 * 24 * int(rule_dict.get('period', {}).get('max', 1000)))\
+                                and (int(time.time()) - update_time_stamp > 3600 * 24 * int(rule_dict.get('period', {}).get('max', 1000))):
                             Common.logger(log_type, crawler).info(f"发布时间超过{int(rule_dict.get('period', {}).get('max', 1000))}天\n")
                             Common.logging(log_type, crawler, env, f"发布时间超过{int(rule_dict.get('period', {}).get('max', 1000))}天\n")
                             return

jixiangxingfu/jixiangxingfu_recommend/jixiangxingfu_recommend.py

@@ -8,7 +8,7 @@ import sys
 import time
 from hashlib import md5
 from appium import webdriver
-from appium.webdriver.common.touch_action import TouchAction
+# from appium.webdriver.common.touch_action import TouchAction
 from appium.webdriver.extensions.android.nativekey import AndroidKey
 from appium.webdriver.webdriver import WebDriver
 from selenium.common import NoSuchElementException
@@ -53,12 +53,15 @@ class JixiangxingfuRecommend:
             if env == "dev":
                 chromedriverExecutable = '/Users/wangkun/Downloads/chromedriver/chromedriver_v107/chromedriver'
             else:
-                chromedriverExecutable = '/Users/piaoquan/Downloads/chromedriver'
+                # chromedriverExecutable = '/Users/piaoquan/Downloads/chromedriver'  # Mac 爬虫机器
+                chromedriverExecutable = 'C:\\chromedriver\\chromedriver.exe'  # 阿里云 Windows
+
             Common.logger(log_type, crawler).info('启动微信')
             caps = {
                 "platformName": "Android",  # 手机操作系统 Android / iOS
                 "deviceName": "a0a65126",  # 连接的设备名（模拟器或真机），安卓可以随便写
-                "platforVersion": "11",  # 手机对应的系统版本（Android 11）
+                "udid": "emulator-5554",  # 指定 adb devices 中的哪一台设备
+                "platforVersion": "12",  # 手机对应的系统版本
                 "appPackage": "com.tencent.mm",  # 被测APP的包名，乐活圈 Android
                 "appActivity": ".ui.LauncherUI",  # 启动的Activity名
                 "autoGrantPermissions": "true",  # 让 appium 自动授权 base 权限，
@@ -80,7 +83,7 @@ class JixiangxingfuRecommend:
                 'browserName': ''
             }
             driver = webdriver.Remote("http://localhost:4723/wd/hub", caps)
-            driver.implicitly_wait(20)
+            driver.implicitly_wait(30)
             # 向下滑动页面，展示出小程序选择面板
             for i in range(120):
                 try:
@@ -105,16 +108,14 @@ class JixiangxingfuRecommend:
             driver.find_elements(By.XPATH, '//*[@text="祝福每天好运来相伴"]')[-1].click()
 
             # 获取视频信息
-            time.sleep(5)
+            time.sleep(15)
             cls.get_videoList(log_type, crawler, driver, env)
 
             # 退出微信
             cls.quit(log_type, crawler, driver)
 
         except Exception as e:
-            Common.logger(log_type, crawler).error('start_wechat异常:{}，重启 ADB\n', e)
-            cmd = "adb kill-server && adb start-server"
-            os.system(cmd)
+            Common.logger(log_type, crawler).error('start_wechat异常:{}\n', e)
 
     # 退出 APP
     @classmethod
@@ -212,6 +213,7 @@ class JixiangxingfuRecommend:
 
     @classmethod
     def get_videoList(cls, log_type, crawler, driver, env):
+        driver.implicitly_wait(20)
         # # 鼠标左键点击， 1为x坐标， 2为y坐标
         # Common.logger(log_type, crawler).info('关闭广告')
         # size = driver.get_window_size()
@@ -334,7 +336,7 @@ class JixiangxingfuRecommend:
                                                   strategy="推荐榜爬虫策略",
                                                   our_uid="recommend",
                                                   env=env,
-                                                  oss_endpoint="out")
+                                                  oss_endpoint="inner")
         if env == 'dev':
             our_video_link = f"https://testadmin.piaoquantv.com/cms/post-detail/{our_video_id}/info"
         else:
@@ -401,6 +403,6 @@ class JixiangxingfuRecommend:
 
 
 if __name__ == "__main__":
-    # JixiangxingfuRecommend.start_wechat('recommend', 'jixiangxingfu', 'dev')
-    print(JixiangxingfuRecommend.jixiangxingfu_config("recommend", "jixiangxingfu", "filter", "dev"))
+    JixiangxingfuRecommend.start_wechat('recommend', 'jixiangxingfu', 'dev')
+    # print(JixiangxingfuRecommend.jixiangxingfu_config("recommend", "jixiangxingfu", "filter", "dev"))
     pass

zhongmiaoyinxin/zhongmiaoyinxin_recommend/zhongmiaoyinxin_recommend.py

@@ -53,12 +53,15 @@ class ZhongmiaoyinxinRecommend:
             if env == "dev":
                 chromedriverExecutable = "/Users/wangkun/Downloads/chromedriver/chromedriver_v107/chromedriver"
             else:
-                chromedriverExecutable = '/Users/piaoquan/Downloads/chromedriver'
+                # chromedriverExecutable = '/Users/piaoquan/Downloads/chromedriver'  # Mac 爬虫机器
+                chromedriverExecutable = 'C:\\chromedriver\\chromedriver.exe'  # 阿里云 Windows
+
             Common.logger(log_type, crawler).info('启动微信')
             caps = {
                 "platformName": "Android",  # 手机操作系统 Android / iOS
                 "deviceName": "a0a65126",  # 连接的设备名（模拟器或真机），安卓可以随便写
-                "platforVersion": "11",  # 手机对应的系统版本（Android 11）
+                "udid": "emulator-5554",  # 指定 adb devices 中的哪一台设备
+                "platforVersion": "12",  # 手机对应的系统版本
                 "appPackage": "com.tencent.mm",  # 被测APP的包名，乐活圈 Android
                 "appActivity": ".ui.LauncherUI",  # 启动的Activity名
                 "autoGrantPermissions": "true",  # 让 appium 自动授权 base 权限，
@@ -80,7 +83,7 @@ class ZhongmiaoyinxinRecommend:
                 'browserName': ''
             }
             driver = webdriver.Remote("http://localhost:4723/wd/hub", caps)
-            driver.implicitly_wait(20)
+            driver.implicitly_wait(30)
             # 向下滑动页面，展示出小程序选择面板
             for i in range(120):
                 try:
@@ -104,14 +107,12 @@ class ZhongmiaoyinxinRecommend:
             Common.logger(log_type, crawler).info('打开小程序"西瓜悦"')
             driver.find_elements(By.XPATH, '//*[@text="西瓜悦"]')[-1].click()
 
-            time.sleep(5)
+            time.sleep(15)
             cls.get_videoList(log_type, crawler, driver, env)
 
             cls.quit(log_type, crawler, driver)
         except Exception as e:
             Common.logger(log_type, crawler).error('start_wechat异常:{}\n', e)
-            cmd = "adb kill-server && adb start-server"
-            os.system(cmd)
 
     @classmethod
     def quit(cls, log_type, crawler, driver: WebDriver):
@@ -209,6 +210,7 @@ class ZhongmiaoyinxinRecommend:
 
     @classmethod
     def get_videoList(cls, log_type, crawler, driver: WebDriver, env):
+        driver.implicitly_wait(20)
         # 鼠标左键点击， 1为x坐标， 2为y坐标
         Common.logger(log_type, crawler).info('关闭广告')
         size = driver.get_window_size()
@@ -333,7 +335,7 @@ class ZhongmiaoyinxinRecommend:
                                                   strategy="推荐榜爬虫策略",
                                                   our_uid="recommend",
                                                   env=env,
-                                                  oss_endpoint="out")
+                                                  oss_endpoint="inner")
         if env == 'dev':
             our_video_link = f"https://testadmin.piaoquantv.com/cms/post-detail/{our_video_id}/info"
         else: