Merge branch 'master' into lxr_etl_20230620

README.MD

@@ -164,6 +164,7 @@ ps aux | grep kuaishou | grep -v grep | awk '{print $2}' | xargs kill -9
 ps aux | grep douyin | grep -v grep | awk '{print $2}' | xargs kill -9
 ps aux | grep xiaoniangao | grep -v grep | awk '{print $2}' | xargs kill -9
 ps aux | grep kanyikan | grep -v grep | awk '{print $2}' | xargs kill -9
+ps aux | grep Appium | grep -v grep | awk '{print $2}' | xargs kill -9
 ```
 
 #### 生成 requirements.txt

kanyikan/kanyikan_recommend/kanyikan_recommend0627.py

@@ -21,7 +21,7 @@ proxies = {"http": None, "https": None}
 
 class KanyikanRecommend:
     platform = "看一看"
-    strategy = "推荐榜爬虫策略"
+    strategy = "推荐抓取策略"
 
     @classmethod
     def repeat_video(cls, log_type, crawler, video_id, env):
@@ -223,7 +223,7 @@ class KanyikanRecommend:
                                                 {our_uid},
                                                 "{video_dict['user_id']}",
                                                 "{cls.platform}",
-                                                "推荐抓取策略",
+                                                "{cls.strategy}",
                                                 "{video_dict['video_id']}",
                                                 "{video_dict['video_title']}",
                                                 "{video_dict['cover_url']}",

xigua/xigua_search/xigua_search_scheduling.py

@@ -569,7 +569,7 @@ class XiguasearchScheduling:
         Common.logger(log_type, crawler).info(f"打开搜索页:{user_dict['link']}")
         Common.logging(log_type, crawler, env, f"打开搜索页:{user_dict['link']}")
         driver.get(f"https://www.ixigua.com/search/{user_dict['link']}/")
-        time.sleep(3)
+        time.sleep(2)
         Common.logger(log_type, crawler).info("关闭登录弹框")
         Common.logging(log_type, crawler, env, "关闭登录弹框")
         if driver.find_elements(By.XPATH, '//*[@class="xg-notification-close"]') != 0:
@@ -581,12 +581,11 @@ class XiguasearchScheduling:
         Common.logger(log_type, crawler).info("点击最新排序")
         Common.logging(log_type, crawler, env, "点击最新排序")
         driver.find_element(By.XPATH, '//*[@class="searchPageV2__header-categories-wrapper"]/*[1]/*[2]/*[1]').click()
-        time.sleep(3)
+        time.sleep(1)
         # driver.get_screenshot_as_file(f"./{crawler}/photos/{user_dict['link']}-最新排序.png")
         Common.logger(log_type, crawler).info("收起筛选按钮\n")
         Common.logging(log_type, crawler, env, "收起筛选按钮\n")
         driver.find_element(By.XPATH, '//*[@class="searchPageV2__header-icons-categories"]').click()
-        time.sleep(1)
 
         index = 0
         num = 0
@@ -616,7 +615,7 @@ class XiguasearchScheduling:
                     Common.logger(log_type, crawler).info(f'拖动"视频"列表第{num}个至屏幕中间')
                     Common.logging(log_type, crawler, env, f'拖动"视频"列表第{num}个至屏幕中间')
                     driver.execute_script("arguments[0].scrollIntoView({block:'center',inline:'center'})", video_element)
-                    time.sleep(3)
+                    time.sleep(1)
                     # driver.get_screenshot_as_file(f"./{crawler}/photos/{user_dict['link']}-{num}.png")
                     title = video_element.find_elements(By.XPATH, '//*[@class="HorizontalFeedCard__coverWrapper disableZoomAnimation"]')[index+i-1].get_attribute('title')
                     publish_day = video_element.find_elements(By.XPATH, '//*[@class="HorizontalFeedCard-accessories-bottomInfo__statistics"]')[index+i-1].text.split('· ')[-1]

zhongmiaoyinxin/zhongmiaoyinxin_recommend/zhongmiaoyinxin_recommend.py

@@ -49,67 +49,67 @@ class ZhongmiaoyinxinRecommend:
 
     @classmethod
     def start_wechat(cls, log_type, crawler, env):
-        try:
-            if env == "dev":
-                chromedriverExecutable = "/Users/wangkun/Downloads/chromedriver/chromedriver_v107/chromedriver"
-            else:
-                chromedriverExecutable = '/Users/piaoquan/Downloads/chromedriver'
-            Common.logger(log_type, crawler).info('启动微信')
-            caps = {
-                "platformName": "Android",  # 手机操作系统 Android / iOS
-                "deviceName": "a0a65126",  # 连接的设备名（模拟器或真机），安卓可以随便写
-                "platforVersion": "11",  # 手机对应的系统版本（Android 11）
-                "appPackage": "com.tencent.mm",  # 被测APP的包名，乐活圈 Android
-                "appActivity": ".ui.LauncherUI",  # 启动的Activity名
-                "autoGrantPermissions": "true",  # 让 appium 自动授权 base 权限，
-                # 如果 noReset 为 True，则该条不生效（该参数为 Android 独有），对应的值为 True 或 False
-                "unicodekeyboard": True,  # 使用自带输入法，输入中文时填True
-                "resetkeyboard": True,  # 执行完程序恢复原来输入法
-                "noReset": True,  # 不重置APP
-                "printPageSourceOnFailure": True,  # 找不到元素时，appium log 会完整记录当前页面的 pagesource
-                "newCommandTimeout": 6000,  # 初始等待时间
-                "automationName": "UiAutomator2",  # 使用引擎，默认为 Appium，
-                # 其中 Appium、UiAutomator2、Selendroid、Espresso 用于 Android，XCUITest 用于 iOS
-                "showChromedriverLog": True,
-                'enableWebviewDetailsCollection': True,
-                'setWebContentsDebuggingEnabled': True,
-                'recreateChromeDriverSessions': True,
-                'chromedriverExecutable': chromedriverExecutable,
-                "chromeOptions": {"androidProcess": "com.tencent.mm:appbrand0"},
-                # "chromeOptions": {"androidProcess": "com.tencent.mm:tools"},
-                'browserName': ''
-            }
-            driver = webdriver.Remote("http://localhost:4723/wd/hub", caps)
-            driver.implicitly_wait(20)
-            # 向下滑动页面，展示出小程序选择面板
-            for i in range(120):
-                try:
-                    # 发现微信消息 TAB，代表微信已启动成功
-                    if driver.find_elements(By.ID, 'com.tencent.mm:id/f2s'):
-                        break
-                    # 发现并关闭系统菜单栏
-                    elif driver.find_element(By.ID, 'com.android.systemui:id/dismiss_view'):
-                        Common.logger(log_type, crawler).info('发现并关闭系统下拉菜单栏')
-                        driver.find_element(By.ID, 'com.android.systemui:id/dismiss_view').click()
-                    else:
-                        pass
-                except NoSuchElementException:
-                    time.sleep(1)
-            Common.logger(log_type, crawler).info('下滑，展示小程序选择面板')
-            size = driver.get_window_size()
-            driver.swipe(int(size['width'] * 0.5), int(size['height'] * 0.2),
-                         int(size['width'] * 0.5), int(size['height'] * 0.8), 200)
-            # 打开小程序"众妙之上"
-            time.sleep(5)
-            Common.logger(log_type, crawler).info('打开小程序"西瓜说事"')
-            driver.find_elements(By.XPATH, '//*[@text="西瓜说事"]')[-1].click()
+        # try:
+        if env == "dev":
+            chromedriverExecutable = "/Users/wangkun/Downloads/chromedriver/chromedriver_v107/chromedriver"
+        else:
+            chromedriverExecutable = '/Users/piaoquan/Downloads/chromedriver'
+        Common.logger(log_type, crawler).info('启动微信')
+        caps = {
+            "platformName": "Android",  # 手机操作系统 Android / iOS
+            "deviceName": "a0a65126",  # 连接的设备名（模拟器或真机），安卓可以随便写
+            "platforVersion": "11",  # 手机对应的系统版本（Android 11）
+            "appPackage": "com.tencent.mm",  # 被测APP的包名，乐活圈 Android
+            "appActivity": ".ui.LauncherUI",  # 启动的Activity名
+            "autoGrantPermissions": "true",  # 让 appium 自动授权 base 权限，
+            # 如果 noReset 为 True，则该条不生效（该参数为 Android 独有），对应的值为 True 或 False
+            "unicodekeyboard": True,  # 使用自带输入法，输入中文时填True
+            "resetkeyboard": True,  # 执行完程序恢复原来输入法
+            "noReset": True,  # 不重置APP
+            "printPageSourceOnFailure": True,  # 找不到元素时，appium log 会完整记录当前页面的 pagesource
+            "newCommandTimeout": 6000,  # 初始等待时间
+            "automationName": "UiAutomator2",  # 使用引擎，默认为 Appium，
+            # 其中 Appium、UiAutomator2、Selendroid、Espresso 用于 Android，XCUITest 用于 iOS
+            "showChromedriverLog": True,
+            'enableWebviewDetailsCollection': True,
+            'setWebContentsDebuggingEnabled': True,
+            'recreateChromeDriverSessions': True,
+            'chromedriverExecutable': chromedriverExecutable,
+            "chromeOptions": {"androidProcess": "com.tencent.mm:appbrand0"},
+            # "chromeOptions": {"androidProcess": "com.tencent.mm:tools"},
+            'browserName': ''
+        }
+        driver = webdriver.Remote("http://localhost:4723/wd/hub", caps)
+        driver.implicitly_wait(20)
+        # 向下滑动页面，展示出小程序选择面板
+        for i in range(120):
+            try:
+                # 发现微信消息 TAB，代表微信已启动成功
+                if driver.find_elements(By.ID, 'com.tencent.mm:id/f2s'):
+                    break
+                # 发现并关闭系统菜单栏
+                elif driver.find_element(By.ID, 'com.android.systemui:id/dismiss_view'):
+                    Common.logger(log_type, crawler).info('发现并关闭系统下拉菜单栏')
+                    driver.find_element(By.ID, 'com.android.systemui:id/dismiss_view').click()
+                else:
+                    pass
+            except NoSuchElementException:
+                time.sleep(1)
+        Common.logger(log_type, crawler).info('下滑，展示小程序选择面板')
+        size = driver.get_window_size()
+        driver.swipe(int(size['width'] * 0.5), int(size['height'] * 0.2),
+                     int(size['width'] * 0.5), int(size['height'] * 0.8), 200)
+        # 打开小程序"众妙之上"
+        time.sleep(5)
+        Common.logger(log_type, crawler).info('打开小程序"西瓜悦"')
+        driver.find_elements(By.XPATH, '//*[@text="西瓜悦"]')[-1].click()
 
-            time.sleep(5)
-            cls.get_videoList(log_type, crawler, driver, env)
+        time.sleep(5)
+        cls.get_videoList(log_type, crawler, driver, env)
 
-            cls.quit(log_type, crawler, driver)
-        except Exception as e:
-            Common.logger(log_type, crawler).error('start_wechat异常:{}\n', e)
+        cls.quit(log_type, crawler, driver)
+        # except Exception as e:
+        #     Common.logger(log_type, crawler).error('start_wechat异常:{}\n', e)
 
     @classmethod
     def quit(cls, log_type, crawler, driver: WebDriver):
@@ -120,6 +120,7 @@ class ZhongmiaoyinxinRecommend:
     def check_to_applet(cls, log_type, crawler, driver: WebDriver):
         while True:
             webview = driver.contexts
+            Common.logger(log_type, crawler).info(f"webview:{webview}")
             driver.switch_to.context(webview[1])
             windowHandles = driver.window_handles
             for handle in windowHandles: