update

zhiqingtiantiankan/zhiqingtiantiankan_main/run_zhiqingtiantiankan_recommend.py

@@ -12,7 +12,7 @@ from zhiqingtiantiankan.zhiqingtiantiankan_recommend.zhiqingtiantiankan_recommen
 def main(log_type, crawler, env):
     Common.logger(log_type, crawler).info('开始抓取:知青天天看小程序\n')
     ZhiqingtiantiankanRecommend.start_wechat(log_type, crawler, env)
-    # Common.del_logs(log_type, crawler)
+    Common.del_logs(log_type, crawler)
     Common.logger(log_type, crawler).info('抓取完一轮\n')
 
 

zhiqingtiantiankan/zhiqingtiantiankan_recommend/zhiqingtiantiankan_recommend.py

@@ -49,73 +49,71 @@ class ZhiqingtiantiankanRecommend:
 
     @classmethod
     def start_wechat(cls, log_type, crawler, env):
-        # try:
-        Common.logger(log_type, crawler).info('启动微信')
-        if env == "dev":
-            chromedriverExecutable = '/Users/wangkun/Downloads/chromedriver/chromedriver_v107/chromedriver'
-        else:
-            # chromedriverExecutable = '/Users/piaoquan/Downloads/chromedriver'  # Mac 爬虫机器
-            chromedriverExecutable = 'C:\\chromedriver\\chromedriver.exe'  # 阿里云 Windows
-        caps = {
-            "platformName": "Android",  # 手机操作系统 Android / iOS
-            "deviceName": "Android",  # 连接的设备名（模拟器或真机），安卓可以随便写
-            "udid": "emulator-5554",  # 指定 adb devices 中的哪一台设备
-            "platforVersion": "12",  # 手机对应的系统版本
-            "appPackage": "com.tencent.mm",  # 被测APP的包名
-            "appActivity": ".ui.LauncherUI",  # 启动的Activity名
-            "autoGrantPermissions": "true",  # 让 appium 自动授权 base 权限，
-            # 如果 noReset 为 True，则该条不生效（该参数为 Android 独有），对应的值为 True 或 False
-            "unicodekeyboard": True,  # 使用自带输入法，输入中文时填True
-            "resetkeyboard": True,  # 执行完程序恢复原来输入法
-            "noReset": True,  # 不重置APP
-            "printPageSourceOnFailure": True,  # 找不到元素时，appium log 会完整记录当前页面的 pagesource
-            "newCommandTimeout": 6000,  # 初始等待时间
-            "automationName": "UiAutomator2",  # 使用引擎，默认为 Appium，
-            # 其中 Appium、UiAutomator2、Selendroid、Espresso 用于 Android，XCUITest 用于 iOS
-            "showChromedriverLog": True,
-            'enableWebviewDetailsCollection': True,
-            'setWebContentsDebuggingEnabled': True,
-            'recreateChromeDriverSessions': True,
-            'chromedriverExecutable': chromedriverExecutable,
-            "chromeOptions": {"androidProcess": "com.tencent.mm:appbrand0"},
-            # "chromeOptions": {"androidProcess": "com.tencent.mm:tools"},
-            'browserName': ''
-        }
-        driver = webdriver.Remote("http://localhost:4723/wd/hub", caps)  # 指定 Appium 端口号：4723
-        driver.implicitly_wait(30)
-        # 向下滑动页面，展示出小程序选择面板
-        for i in range(120):
-            try:
-                # 发现微信消息 TAB，代表微信已启动成功
-                if driver.find_elements(By.ID, 'com.tencent.mm:id/f2s'):
-                    break
-                # 发现并关闭系统菜单栏
-                elif driver.find_element(By.ID, 'com.android.systemui:id/dismiss_view'):
-                    Common.logger(log_type, crawler).info('发现并关闭系统下拉菜单栏')
-                    driver.find_element(By.ID, 'com.android.systemui:id/dismiss_view').click()
-                else:
-                    pass
-            except NoSuchElementException:
-                time.sleep(1)
-        Common.logger(log_type, crawler).info('下滑，展示小程序选择面板')
-        size = driver.get_window_size()
-        driver.swipe(int(size['width'] * 0.5), int(size['height'] * 0.2), int(size['width'] * 0.5),
-                     int(size['height'] * 0.8), 200)
-        # 打开小程序"知青天天看"
-        time.sleep(5)
-        Common.logger(log_type, crawler).info('打开小程序"知青天天看"')
-        driver.find_elements(By.XPATH, '//*[@text="知青天天看"]')[-1].click()
-
-        # 获取视频信息
-        time.sleep(15)
-        cls.get_videoList(log_type, crawler, driver, env)
-
-        # 退出微信
-        cls.quit(log_type, crawler, driver)
-        # except Exception as e:
-        #     Common.logger(log_type, crawler).error('start_wechat异常:{}\n', e)
-        #     cmd = "adb kill-server && adb start-server"
-        #     os.system(cmd)
+        try:
+            Common.logger(log_type, crawler).info('启动微信')
+            if env == "dev":
+                chromedriverExecutable = '/Users/wangkun/Downloads/chromedriver/chromedriver_v107/chromedriver'
+            else:
+                # chromedriverExecutable = '/Users/piaoquan/Downloads/chromedriver'  # Mac 爬虫机器
+                chromedriverExecutable = 'C:\\chromedriver\\chromedriver.exe'  # 阿里云 Windows
+            caps = {
+                "platformName": "Android",  # 手机操作系统 Android / iOS
+                "deviceName": "Android",  # 连接的设备名（模拟器或真机），安卓可以随便写
+                "udid": "emulator-5554",  # 指定 adb devices 中的哪一台设备
+                "platforVersion": "12",  # 手机对应的系统版本
+                "appPackage": "com.tencent.mm",  # 被测APP的包名
+                "appActivity": ".ui.LauncherUI",  # 启动的Activity名
+                "autoGrantPermissions": "true",  # 让 appium 自动授权 base 权限，
+                # 如果 noReset 为 True，则该条不生效（该参数为 Android 独有），对应的值为 True 或 False
+                "unicodekeyboard": True,  # 使用自带输入法，输入中文时填True
+                "resetkeyboard": True,  # 执行完程序恢复原来输入法
+                "noReset": True,  # 不重置APP
+                "printPageSourceOnFailure": True,  # 找不到元素时，appium log 会完整记录当前页面的 pagesource
+                "newCommandTimeout": 6000,  # 初始等待时间
+                "automationName": "UiAutomator2",  # 使用引擎，默认为 Appium，
+                # 其中 Appium、UiAutomator2、Selendroid、Espresso 用于 Android，XCUITest 用于 iOS
+                "showChromedriverLog": True,
+                'enableWebviewDetailsCollection': True,
+                'setWebContentsDebuggingEnabled': True,
+                'recreateChromeDriverSessions': True,
+                'chromedriverExecutable': chromedriverExecutable,
+                "chromeOptions": {"androidProcess": "com.tencent.mm:appbrand0"},
+                # "chromeOptions": {"androidProcess": "com.tencent.mm:tools"},
+                'browserName': ''
+            }
+            driver = webdriver.Remote("http://localhost:4723/wd/hub", caps)  # 指定 Appium 端口号：4723
+            driver.implicitly_wait(30)
+            # 向下滑动页面，展示出小程序选择面板
+            for i in range(120):
+                try:
+                    # 发现微信消息 TAB，代表微信已启动成功
+                    if driver.find_elements(By.ID, 'com.tencent.mm:id/f2s'):
+                        break
+                    # 发现并关闭系统菜单栏
+                    elif driver.find_element(By.ID, 'com.android.systemui:id/dismiss_view'):
+                        Common.logger(log_type, crawler).info('发现并关闭系统下拉菜单栏')
+                        driver.find_element(By.ID, 'com.android.systemui:id/dismiss_view').click()
+                    else:
+                        pass
+                except NoSuchElementException:
+                    time.sleep(1)
+            Common.logger(log_type, crawler).info('下滑，展示小程序选择面板')
+            size = driver.get_window_size()
+            driver.swipe(int(size['width'] * 0.5), int(size['height'] * 0.2), int(size['width'] * 0.5),
+                         int(size['height'] * 0.8), 200)
+            # 打开小程序"知青天天看"
+            time.sleep(5)
+            Common.logger(log_type, crawler).info('打开小程序"知青天天看"')
+            driver.find_elements(By.XPATH, '//*[@text="知青天天看"]')[-1].click()
+
+            # 获取视频信息
+            time.sleep(15)
+            cls.get_videoList(log_type, crawler, driver, env)
+
+            # 退出微信
+            cls.quit(log_type, crawler, driver)
+        except Exception as e:
+            Common.logger(log_type, crawler).error('start_wechat异常:{}\n', e)
 
     # 退出 APP
     @classmethod
@@ -232,89 +230,89 @@ class ZhiqingtiantiankanRecommend:
         time.sleep(5)
         index = 0
         while True:
-            # try:
-            if cls.search_elements(driver, '//wx-view[@class="listbox"]') is None:
-                Common.logger(log_type, crawler).info('窗口已销毁\n')
-                return
-
-            Common.logger(log_type, crawler).info('获取视频列表\n')
-            video_elements = cls.search_elements(driver, '//wx-view[@class="videolist-box videolist--videolist-box"]')
-            if video_elements is None:
-                Common.logger(log_type, crawler).warning(f'video_elements:{video_elements}')
-                return
+            try:
+                if cls.search_elements(driver, '//wx-view[@class="listbox"]') is None:
+                    Common.logger(log_type, crawler).info('窗口已销毁\n')
+                    return
 
-            video_element_temp = video_elements[index:]
-            if len(video_element_temp) == 0:
-                Common.logger(log_type, crawler).info('到底啦~~~~~~~~~~~~~\n')
-                return
+                Common.logger(log_type, crawler).info('获取视频列表\n')
+                video_elements = cls.search_elements(driver, '//wx-view[@class="videolist-box videolist--videolist-box"]')
+                if video_elements is None:
+                    Common.logger(log_type, crawler).warning(f'video_elements:{video_elements}')
+                    return
 
-            for i, video_element in enumerate(video_element_temp):
-                if video_element is None:
-                    Common.logger(log_type, crawler).info('到底啦~\n')
+                video_element_temp = video_elements[index:]
+                if len(video_element_temp) == 0:
+                    Common.logger(log_type, crawler).info('到底啦~~~~~~~~~~~~~\n')
                     return
-                cls.i += 1
-                cls.search_elements(driver, '//wx-view[@class="videolist-box videolist--videolist-box"]')
-
-                Common.logger(log_type, crawler).info(f'拖动"视频"列表第{cls.i}个至屏幕中间')
-                time.sleep(3)
-                driver.execute_script("arguments[0].scrollIntoView({block:'center',inline:'center'})", video_element)
-
-                # video_title = video_element.find_elements(By.XPATH, '//wx-view[@class="video_title videolist--video_title"]')[cls.i-1].text
-                # cover_url = video_element.find_elements(By.XPATH, '//wx-image[@class="itemimage videolist--itemimage"]')[cls.i-1].get_attribute('src')
-                # play_cnt = video_element.find_elements(By.XPATH, '//wx-view[@class="clickbox videolist--clickbox"]')[cls.i-1].text
-
-                video_title = video_element.find_elements(By.XPATH, '//wx-view[@class="video_title videolist--video_title"]')[index+i].text
-                cover_url = video_element.find_elements(By.XPATH, '//wx-image[@class="itemimage videolist--itemimage"]')[index+i].get_attribute('src')
-                play_cnt = video_element.find_elements(By.XPATH, '//wx-view[@class="clickbox videolist--clickbox"]')[index+i].text
-
-                play_cnt = int(float(play_cnt.replace("阅读数", "").strip().split("万")[0]) * 10000)
-                out_video_id = md5(video_title.encode('utf8')).hexdigest()
-                video_dict = {
-                    'video_title': video_title,
-                    'video_id': out_video_id,
-                    'play_cnt': play_cnt,
-                    'comment_cnt': 0,
-                    'like_cnt': 0,
-                    'share_cnt': 0,
-                    'publish_time_stamp': int(time.time()),
-                    'publish_time_str': time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(int(time.time()))),
-                    'user_name': "zhiqingtiantiankan",
-                    'user_id': "zhiqingtiantiankan",
-                    'avatar_url': cover_url,
-                    'cover_url': cover_url,
-                    'session': f"zhiqingtiantiankan-{int(time.time())}"
-                }
-                for k, v in video_dict.items():
-                    Common.logger(log_type, crawler).info(f"{k}:{v}")
-
-                if video_title is None or cover_url is None:
-                    Common.logger(log_type, crawler).info("无效视频\n")
-                elif any(str(word) if str(word) in video_title else False for word in
-                         cls.zhiqingtiantiankan_config(log_type, crawler, "filter", env)) is True:
-                    Common.logger(log_type, crawler).info('已中过滤词\n')
-                elif cls.repeat_out_video_id(log_type, crawler, out_video_id, env) != 0:
-                    Common.logger(log_type, crawler).info('视频已下载\n')
-                else:
-                    video_url = cls.get_video_url(log_type, crawler, driver, video_element)
 
-                    if video_url is None:
-                        Common.logger(log_type, crawler).info("未获取到视频播放地址\n")
-                        driver.press_keycode(AndroidKey.BACK)
-                    elif cls.repeat_video_url(log_type, crawler, video_url, env) != 0:
+                for i, video_element in enumerate(video_element_temp):
+                    if video_element is None:
+                        Common.logger(log_type, crawler).info('到底啦~\n')
+                        return
+                    cls.i += 1
+                    cls.search_elements(driver, '//wx-view[@class="videolist-box videolist--videolist-box"]')
+
+                    Common.logger(log_type, crawler).info(f'拖动"视频"列表第{cls.i}个至屏幕中间')
+                    time.sleep(3)
+                    driver.execute_script("arguments[0].scrollIntoView({block:'center',inline:'center'})", video_element)
+
+                    # video_title = video_element.find_elements(By.XPATH, '//wx-view[@class="video_title videolist--video_title"]')[cls.i-1].text
+                    # cover_url = video_element.find_elements(By.XPATH, '//wx-image[@class="itemimage videolist--itemimage"]')[cls.i-1].get_attribute('src')
+                    # play_cnt = video_element.find_elements(By.XPATH, '//wx-view[@class="clickbox videolist--clickbox"]')[cls.i-1].text
+
+                    video_title = video_element.find_elements(By.XPATH, '//wx-view[@class="video_title videolist--video_title"]')[index+i].text
+                    cover_url = video_element.find_elements(By.XPATH, '//wx-image[@class="itemimage videolist--itemimage"]')[index+i].get_attribute('src')
+                    play_cnt = video_element.find_elements(By.XPATH, '//wx-view[@class="clickbox videolist--clickbox"]')[index+i].text
+
+                    play_cnt = int(float(play_cnt.replace("阅读数", "").strip().split("万")[0]) * 10000)
+                    out_video_id = md5(video_title.encode('utf8')).hexdigest()
+                    video_dict = {
+                        'video_title': video_title,
+                        'video_id': out_video_id,
+                        'play_cnt': play_cnt,
+                        'comment_cnt': 0,
+                        'like_cnt': 0,
+                        'share_cnt': 0,
+                        'publish_time_stamp': int(time.time()),
+                        'publish_time_str': time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(int(time.time()))),
+                        'user_name': "zhiqingtiantiankan",
+                        'user_id': "zhiqingtiantiankan",
+                        'avatar_url': cover_url,
+                        'cover_url': cover_url,
+                        'session': f"zhiqingtiantiankan-{int(time.time())}"
+                    }
+                    for k, v in video_dict.items():
+                        Common.logger(log_type, crawler).info(f"{k}:{v}")
+
+                    if video_title is None or cover_url is None:
+                        Common.logger(log_type, crawler).info("无效视频\n")
+                    elif any(str(word) if str(word) in video_title else False for word in
+                             cls.zhiqingtiantiankan_config(log_type, crawler, "filter", env)) is True:
+                        Common.logger(log_type, crawler).info('已中过滤词\n')
+                    elif cls.repeat_out_video_id(log_type, crawler, out_video_id, env) != 0:
                         Common.logger(log_type, crawler).info('视频已下载\n')
-                        driver.press_keycode(AndroidKey.BACK)
                     else:
-                        video_dict["video_url"] = video_url
-                        Common.logger(log_type, crawler).info(f"video_url:{video_url}\n")
-                        # driver.press_keycode(AndroidKey.BACK)
-                        cls.download_publish(log_type, crawler, video_dict, env, driver)
-            Common.logger(log_type, crawler).info('已抓取完一组视频，休眠10秒\n')
-            time.sleep(10)
-            index = index + len(video_element_temp)
-            # except Exception as e:
-            #     Common.logger(log_type, crawler).info(f"get_videoList:{e}\n")
-            #     cls.i = 0
-            #     return
+                        video_url = cls.get_video_url(log_type, crawler, driver, video_element)
+
+                        if video_url is None:
+                            Common.logger(log_type, crawler).info("未获取到视频播放地址\n")
+                            driver.press_keycode(AndroidKey.BACK)
+                        elif cls.repeat_video_url(log_type, crawler, video_url, env) != 0:
+                            Common.logger(log_type, crawler).info('视频已下载\n')
+                            driver.press_keycode(AndroidKey.BACK)
+                        else:
+                            video_dict["video_url"] = video_url
+                            Common.logger(log_type, crawler).info(f"video_url:{video_url}\n")
+                            # driver.press_keycode(AndroidKey.BACK)
+                            cls.download_publish(log_type, crawler, video_dict, env, driver)
+                Common.logger(log_type, crawler).info('已抓取完一组视频，休眠10秒\n')
+                time.sleep(10)
+                index = index + len(video_element_temp)
+            except Exception as e:
+                Common.logger(log_type, crawler).info(f"get_videoList:{e}\n")
+                cls.i = 0
+                return
 
     @classmethod
     def download_publish(cls, log_type, crawler, video_dict, env, driver: WebDriver):