update

README.MD

@@ -86,10 +86,11 @@ ps aux | grep 微信 | grep -v grep | awk '{print $2}' | xargs kill -9
 
 ```
 
-#### 线下爬虫: 刚刚都传 / 吉祥幸福 / 知青天天看 / 众妙音信 / wechat_search_key
+#### 线下爬虫: 刚刚都传 / 吉祥幸福 / 知青天天看 / 众妙音信 / wechat_search_key / start_appium
 ```commandline
 MacAir 设备, crontab定时任务
 * * * * * /bin/sh /Users/piaoquan/Desktop/piaoquan_crawler/main/process_offline.sh "prod"
+* * * * * /bin/sh /Users/piaoquan/Desktop/piaoquan_crawler/main/start_appium.sh "recommend" "jixiangxingfu" "prod"
 线下调试
 sh /Users/wangkun/Desktop/crawler/piaoquan_crawler/main/process_offline.sh "dev"
 cd /Users/piaoquan/Desktop/piaoquan_crawler/ && nohup python3 -u weixinzhishu/weixinzhishu_key/search_key_mac.py >> weixinzhishu/logs/nohup-search-key.log 2>&1 &

ganggangdouchuan/ganggangdouchuan_main/run_ganggangdouchuan_recommend.py

@@ -11,13 +11,10 @@ from ganggangdouchuan.ganggangdouchuan_recommend.ganggangdouchuan_recommend impo
 
 
 def main(log_type, crawler, env):
-    # oss_endpoint = "inner"  # 阿里云 Windows
-    oss_endpoint = "out"  # 线下 Mac
-
     Common.logger(log_type, crawler).info('开始抓取:刚刚都传小程序\n')
     Common.logging(log_type, crawler, env, '开始抓取:刚刚都传小程序\n')
     ganggangdouchuan_start_time = int(time.time())
-    GanggangdouchuanRecommend.start_wechat(log_type, crawler, oss_endpoint, env)
+    GanggangdouchuanRecommend.start_wechat(log_type, crawler, env)
     Common.del_logs(log_type, crawler)
     Common.logger(log_type, crawler).info('抓取完一轮\n')
     Common.logging(log_type, crawler, env, '抓取完一轮\n')

ganggangdouchuan/ganggangdouchuan_recommend/ganggangdouchuan_recommend.py

@@ -24,77 +24,77 @@ class GanggangdouchuanRecommend:
     i = 0
 
     @classmethod
-    def start_wechat(cls, log_type, crawler, oss_endpoint, env):
-        try:
-            if env == "dev":
-                chromedriverExecutable = '/Users/wangkun/Downloads/chromedriver/chromedriver_v107/chromedriver'
-                # chromedriverExecutable = 'C:\\chromedriver\\chromedriver.exe'  # 阿里云 Windows
-            else:
-                chromedriverExecutable = '/Users/piaoquan/Downloads/chromedriver'  # Mac 爬虫机器
-                # chromedriverExecutable = 'C:\\chromedriver\\chromedriver.exe'  # 阿里云 Windows
+    def start_wechat(cls, log_type, crawler, env):
+        # try:
+        if env == "dev":
+            chromedriverExecutable = '/Users/wangkun/Downloads/chromedriver/chromedriver_v107/chromedriver'
+            # chromedriverExecutable = 'C:\\chromedriver\\chromedriver.exe'  # 阿里云 Windows
+        else:
+            chromedriverExecutable = '/Users/piaoquan/Downloads/chromedriver'  # Mac 爬虫机器
+            # chromedriverExecutable = 'C:\\chromedriver\\chromedriver.exe'  # 阿里云 Windows
 
-            Common.logger(log_type, crawler).info('启动微信')
-            Common.logging(log_type, crawler, env, '启动微信')
-            caps = {
-                "platformName": "Android",  # 手机操作系统 Android / iOS
-                "deviceName": "a0a65126",  # 连接的设备名（模拟器或真机），安卓可以随便写
-                # "udid": "emulator-5554",  # 指定 adb devices 中的哪一台设备
-                "platforVersion": "11",  # 手机对应的系统版本
-                "appPackage": "com.tencent.mm",  # 被测APP的包名，乐活圈 Android
-                "appActivity": ".ui.LauncherUI",  # 启动的Activity名
-                "autoGrantPermissions": "true",  # 让 appium 自动授权 base 权限，
-                # 如果 noReset 为 True，则该条不生效（该参数为 Android 独有），对应的值为 True 或 False
-                "unicodekeyboard": True,  # 使用自带输入法，输入中文时填True
-                "resetkeyboard": True,  # 执行完程序恢复原来输入法
-                "noReset": True,  # 不重置APP
-                "printPageSourceOnFailure": True,  # 找不到元素时，appium log 会完整记录当前页面的 pagesource
-                "newCommandTimeout": 6000,  # 初始等待时间
-                "automationName": "UiAutomator2",  # 使用引擎，默认为 Appium，
-                # 其中 Appium、UiAutomator2、Selendroid、Espresso 用于 Android，XCUITest 用于 iOS
-                "showChromedriverLog": True,
-                'enableWebviewDetailsCollection': True,
-                'setWebContentsDebuggingEnabled': True,
-                'recreateChromeDriverSessions': True,
-                'chromedriverExecutable': chromedriverExecutable,
-                "chromeOptions": {"androidProcess": "com.tencent.mm:appbrand0"},
-                'browserName': ''
-            }
-            driver = webdriver.Remote("http://localhost:4723/wd/hub", caps)  # 指定 Appium 端口号：4723
-            driver.implicitly_wait(30)
-            # 向下滑动页面，展示出小程序选择面板
-            for i in range(120):
-                try:
-                    # 发现微信消息 TAB，代表微信已启动成功
-                    if driver.find_elements(By.ID, 'com.tencent.mm:id/f2s'):
-                        break
-                    # 发现并关闭系统菜单栏
-                    elif driver.find_element(By.ID, 'com.android.systemui:id/dismiss_view'):
-                        Common.logger(log_type, crawler).info('发现并关闭系统下拉菜单栏')
-                        Common.logging(log_type, crawler, env, '发现并关闭系统下拉菜单栏')
-                        driver.find_element(By.ID, 'com.android.systemui:id/dismiss_view').click()
-                    else:
-                        pass
-                except NoSuchElementException:
-                    time.sleep(1)
-            Common.logger(log_type, crawler).info('下滑，展示小程序选择面板')
-            Common.logging(log_type, crawler, env, '下滑，展示小程序选择面板')
-            size = driver.get_window_size()
-            driver.swipe(int(size['width'] * 0.5), int(size['height'] * 0.2),
-                         int(size['width'] * 0.5), int(size['height'] * 0.8), 200)
-            # 打开小程序"刚刚都传"
-            time.sleep(5)
-            Common.logger(log_type, crawler).info('打开小程序"刚刚都传"')
-            Common.logging(log_type, crawler, env, '打开小程序"刚刚都传"')
-            driver.find_elements(By.XPATH, '//*[@text="刚刚都传"]')[-1].click()
-            # time.sleep(40)
-            time.sleep(10)
-            cls.get_videoList(log_type, crawler, oss_endpoint, env, driver)
-            driver.quit()
-            Common.logger(log_type, crawler).info('退出微信成功\n')
-            Common.logging(log_type, crawler, env, '退出微信成功\n')
-        except Exception as e:
-            Common.logger(log_type, crawler).error(f'start_wechat异常:{e}\n')
-            Common.logging(log_type, crawler, env, f'start_wechat异常:{e}\n')
+        Common.logger(log_type, crawler).info('启动微信')
+        Common.logging(log_type, crawler, env, '启动微信')
+        caps = {
+            "platformName": "Android",  # 手机操作系统 Android / iOS
+            "deviceName": "a0a65126",  # 连接的设备名（模拟器或真机），安卓可以随便写
+            # "udid": "emulator-5554",  # 指定 adb devices 中的哪一台设备
+            "platforVersion": "11",  # 手机对应的系统版本
+            "appPackage": "com.tencent.mm",  # 被测APP的包名，乐活圈 Android
+            "appActivity": ".ui.LauncherUI",  # 启动的Activity名
+            "autoGrantPermissions": "true",  # 让 appium 自动授权 base 权限，
+            # 如果 noReset 为 True，则该条不生效（该参数为 Android 独有），对应的值为 True 或 False
+            "unicodekeyboard": True,  # 使用自带输入法，输入中文时填True
+            "resetkeyboard": True,  # 执行完程序恢复原来输入法
+            "noReset": True,  # 不重置APP
+            "printPageSourceOnFailure": True,  # 找不到元素时，appium log 会完整记录当前页面的 pagesource
+            "newCommandTimeout": 6000,  # 初始等待时间
+            "automationName": "UiAutomator2",  # 使用引擎，默认为 Appium，
+            # 其中 Appium、UiAutomator2、Selendroid、Espresso 用于 Android，XCUITest 用于 iOS
+            "showChromedriverLog": True,
+            'enableWebviewDetailsCollection': True,
+            'setWebContentsDebuggingEnabled': True,
+            'recreateChromeDriverSessions': True,
+            'chromedriverExecutable': chromedriverExecutable,
+            "chromeOptions": {"androidProcess": "com.tencent.mm:appbrand0"},
+            'browserName': ''
+        }
+        driver = webdriver.Remote("http://localhost:4723/wd/hub", caps)  # 指定 Appium 端口号：4723
+        driver.implicitly_wait(30)
+        # 向下滑动页面，展示出小程序选择面板
+        for i in range(120):
+            try:
+                # 发现微信消息 TAB，代表微信已启动成功
+                if driver.find_elements(By.ID, 'com.tencent.mm:id/f2s'):
+                    break
+                # 发现并关闭系统菜单栏
+                elif driver.find_element(By.ID, 'com.android.systemui:id/dismiss_view'):
+                    Common.logger(log_type, crawler).info('发现并关闭系统下拉菜单栏')
+                    Common.logging(log_type, crawler, env, '发现并关闭系统下拉菜单栏')
+                    driver.find_element(By.ID, 'com.android.systemui:id/dismiss_view').click()
+                else:
+                    pass
+            except NoSuchElementException:
+                time.sleep(1)
+        Common.logger(log_type, crawler).info('下滑，展示小程序选择面板')
+        Common.logging(log_type, crawler, env, '下滑，展示小程序选择面板')
+        size = driver.get_window_size()
+        driver.swipe(int(size['width'] * 0.5), int(size['height'] * 0.2),
+                     int(size['width'] * 0.5), int(size['height'] * 0.8), 200)
+        # 打开小程序"刚刚都传"
+        time.sleep(5)
+        Common.logger(log_type, crawler).info('打开小程序"刚刚都传"')
+        Common.logging(log_type, crawler, env, '打开小程序"刚刚都传"')
+        driver.find_elements(By.XPATH, '//*[@text="刚刚都传"]')[-1].click()
+        # time.sleep(40)
+        time.sleep(10)
+        cls.get_videoList(log_type, crawler, env, driver)
+        driver.quit()
+        Common.logger(log_type, crawler).info('退出微信成功\n')
+        Common.logging(log_type, crawler, env, '退出微信成功\n')
+        # except Exception as e:
+        #     Common.logger(log_type, crawler).error(f'start_wechat异常:{e}\n')
+        #     Common.logging(log_type, crawler, env, f'start_wechat异常:{e}\n')
 
     @classmethod
     def get_video_url(cls, log_type, crawler, env, driver: WebDriver, video_element):
@@ -177,7 +177,7 @@ class GanggangdouchuanRecommend:
         return len(repeat_video)
 
     @classmethod
-    def get_videoList(cls, log_type, crawler, oss_endpoint, env, driver: WebDriver):
+    def get_videoList(cls, log_type, crawler, env, driver: WebDriver):
         driver.implicitly_wait(20)
         # 切换到小程序
         cls.check_to_applet(log_type, crawler, env, driver)
@@ -185,98 +185,98 @@ class GanggangdouchuanRecommend:
         time.sleep(10)
         index = 0
         while True:
-            try:
-                if cls.search_elements(log_type, crawler, env, driver, '//wx-view[@class="lists"]') is None:
-                    Common.logger(log_type, crawler).info('窗口已销毁\n')
-                    Common.logging(log_type, crawler, env, '窗口已销毁\n')
-                    return
+            # try:
+            if cls.search_elements(log_type, crawler, env, driver, '//wx-view[@class="lists"]') is None:
+                Common.logger(log_type, crawler).info('窗口已销毁\n')
+                Common.logging(log_type, crawler, env, '窗口已销毁\n')
+                return
 
-                Common.logger(log_type, crawler).info('获取视频列表\n')
-                Common.logging(log_type, crawler, env, '获取视频列表\n')
-                video_elements = cls.search_elements(log_type, crawler, env, driver, '//wx-view[@class="list"]')
-                if video_elements is None:
-                    Common.logger(log_type, crawler).warning(f'video_elements:{video_elements}')
-                    Common.logging(log_type, crawler, env, f'video_elements:{video_elements}')
-                    return
+            Common.logger(log_type, crawler).info('获取视频列表\n')
+            Common.logging(log_type, crawler, env, '获取视频列表\n')
+            video_elements = cls.search_elements(log_type, crawler, env, driver, '//wx-view[@class="list"]')
+            if video_elements is None:
+                Common.logger(log_type, crawler).warning(f'video_elements:{video_elements}')
+                Common.logging(log_type, crawler, env, f'video_elements:{video_elements}')
+                return
 
-                video_element_temp = video_elements[index:]
-                if len(video_element_temp) == 0:
-                    Common.logger(log_type, crawler).info('到底啦~~~~~~~~~~~~~\n')
-                    Common.logging(log_type, crawler, env, '到底啦~~~~~~~~~~~~~\n')
-                    return
+            video_element_temp = video_elements[index:]
+            if len(video_element_temp) == 0:
+                Common.logger(log_type, crawler).info('到底啦~~~~~~~~~~~~~\n')
+                Common.logging(log_type, crawler, env, '到底啦~~~~~~~~~~~~~\n')
+                return
 
-                for i, video_element in enumerate(video_element_temp):
-                    if video_element is None:
-                        Common.logger(log_type, crawler).info('到底啦~\n')
-                        Common.logging(log_type, crawler, env, '到底啦~\n')
-                        return
-                    cls.i += 1
-                    cls.search_elements(log_type, crawler, env, driver, '//wx-view[@class="list"]')
-                    Common.logger(log_type, crawler).info(f'拖动"视频"列表第{cls.i}个至屏幕中间')
-                    Common.logging(log_type, crawler, env, f'拖动"视频"列表第{cls.i}个至屏幕中间')
-                    time.sleep(3)
-                    driver.execute_script("arguments[0].scrollIntoView({block:'center',inline:'center'})", video_element)
+            for i, video_element in enumerate(video_element_temp):
+                if video_element is None:
+                    Common.logger(log_type, crawler).info('到底啦~\n')
+                    Common.logging(log_type, crawler, env, '到底啦~\n')
+                    return
+                cls.i += 1
+                cls.search_elements(log_type, crawler, env, driver, '//wx-view[@class="list"]')
+                Common.logger(log_type, crawler).info(f'拖动"视频"列表第{cls.i}个至屏幕中间')
+                Common.logging(log_type, crawler, env, f'拖动"视频"列表第{cls.i}个至屏幕中间')
+                time.sleep(3)
+                driver.execute_script("arguments[0].scrollIntoView({block:'center',inline:'center'})", video_element)
 
-                    # video_title = video_element.find_elements(By.XPATH, '//wx-view[@class="title"]//span[2]')[cls.i - 1].text
-                    # cover_url = video_element.find_elements(By.XPATH, '//wx-image[@class="poster-img"]')[cls.i - 1].get_attribute('src')
+                # video_title = video_element.find_elements(By.XPATH, '//wx-view[@class="title"]//span[2]')[cls.i - 1].text
+                # cover_url = video_element.find_elements(By.XPATH, '//wx-image[@class="poster-img"]')[cls.i - 1].get_attribute('src')
 
-                    video_title = video_element.find_elements(By.XPATH, '//wx-view[@class="title"]//span[2]')[index+i].text
-                    cover_url = video_element.find_elements(By.XPATH, '//wx-image[@class="poster-img"]')[index+i].get_attribute('src')
+                video_title = video_element.find_elements(By.XPATH, '//wx-view[@class="title"]//span[2]')[index+i].text
+                cover_url = video_element.find_elements(By.XPATH, '//wx-image[@class="poster-img"]')[index+i].get_attribute('src')
 
-                    out_video_id = md5(video_title.encode('utf8')).hexdigest()
-                    video_dict = {
-                        'video_title': video_title,
-                        'video_id': out_video_id,
-                        'play_cnt': 0,
-                        'comment_cnt': 0,
-                        'like_cnt': 0,
-                        'share_cnt': 0,
-                        'publish_time_stamp': int(time.time()),
-                        'publish_time_str': time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(int(time.time()))),
-                        'user_name': "ganggangdouchuan",
-                        'user_id': "ganggangdouchuan",
-                        'avatar_url': cover_url,
-                        'cover_url': cover_url,
-                        'session': f"ganggangdouchuan-{int(time.time())}"
-                    }
-                    for k, v in video_dict.items():
-                        Common.logger(log_type, crawler).info(f"{k}:{v}")
-                    Common.logging(log_type, crawler, env, f'video_dict:{video_dict}')
+                out_video_id = md5(video_title.encode('utf8')).hexdigest()
+                video_dict = {
+                    'video_title': video_title,
+                    'video_id': out_video_id,
+                    'play_cnt': 0,
+                    'comment_cnt': 0,
+                    'like_cnt': 0,
+                    'share_cnt': 0,
+                    'publish_time_stamp': int(time.time()),
+                    'publish_time_str': time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(int(time.time()))),
+                    'user_name': "ganggangdouchuan",
+                    'user_id': "ganggangdouchuan",
+                    'avatar_url': cover_url,
+                    'cover_url': cover_url,
+                    'session': f"ganggangdouchuan-{int(time.time())}"
+                }
+                for k, v in video_dict.items():
+                    Common.logger(log_type, crawler).info(f"{k}:{v}")
+                Common.logging(log_type, crawler, env, f'video_dict:{video_dict}')
 
-                    if video_title is None or cover_url is None:
-                        Common.logger(log_type, crawler).info("无效视频\n")
-                        Common.logging(log_type, crawler, env, "无效视频\n")
-                    elif cls.repeat_out_video_id(log_type, crawler, out_video_id, env) != 0:
+                if video_title is None or cover_url is None:
+                    Common.logger(log_type, crawler).info("无效视频\n")
+                    Common.logging(log_type, crawler, env, "无效视频\n")
+                elif cls.repeat_out_video_id(log_type, crawler, out_video_id, env) != 0:
+                    Common.logger(log_type, crawler).info('视频已下载\n')
+                    Common.logging(log_type, crawler, env, '视频已下载\n')
+                else:
+                    video_url = cls.get_video_url(log_type, crawler, env, driver, video_element)
+                    Common.logger(log_type, crawler).info(f'video_url:{video_url}')
+                    Common.logging(log_type, crawler, env, f'video_url:{video_url}')
+                    if video_url is None:
+                        Common.logger(log_type, crawler).info("未获取到视频播放地址\n")
+                        Common.logging(log_type, crawler, env, "未获取到视频播放地址\n")
+                        driver.press_keycode(AndroidKey.BACK)
+                    elif cls.repeat_video_url(log_type, crawler, video_url, env) != 0:
                         Common.logger(log_type, crawler).info('视频已下载\n')
                         Common.logging(log_type, crawler, env, '视频已下载\n')
+                        driver.press_keycode(AndroidKey.BACK)
                     else:
-                        video_url = cls.get_video_url(log_type, crawler, env, driver, video_element)
-                        Common.logger(log_type, crawler).info(f'video_url:{video_url}')
-                        Common.logging(log_type, crawler, env, f'video_url:{video_url}')
-                        if video_url is None:
-                            Common.logger(log_type, crawler).info("未获取到视频播放地址\n")
-                            Common.logging(log_type, crawler, env, "未获取到视频播放地址\n")
-                            driver.press_keycode(AndroidKey.BACK)
-                        elif cls.repeat_video_url(log_type, crawler, video_url, env) != 0:
-                            Common.logger(log_type, crawler).info('视频已下载\n')
-                            Common.logging(log_type, crawler, env, '视频已下载\n')
-                            driver.press_keycode(AndroidKey.BACK)
-                        else:
-                            video_dict["video_url"]=video_url
-                            cls.download_publish(log_type, crawler, video_dict, oss_endpoint, env, driver)
-                Common.logger(log_type, crawler).info('已抓取完一组视频，休眠10秒\n')
-                Common.logging(log_type, crawler, env, '已抓取完一组视频，休眠10秒\n')
-                time.sleep(10)
-                index = index + len(video_element_temp)
+                        video_dict["video_url"]=video_url
+                        cls.download_publish(log_type, crawler, video_dict, env, driver)
+            Common.logger(log_type, crawler).info('已抓取完一组视频，休眠10秒\n')
+            Common.logging(log_type, crawler, env, '已抓取完一组视频，休眠10秒\n')
+            time.sleep(10)
+            index = index + len(video_element_temp)
 
-            except Exception as e:
-                Common.logger(log_type, crawler).error(f'get_videoList异常:{e}\n')
-                Common.logging(log_type, crawler, env, f'get_videoList异常:{e}\n')
-                cls.i = 0
-                return
+            # except Exception as e:
+            #     Common.logger(log_type, crawler).error(f'get_videoList异常:{e}\n')
+            #     Common.logging(log_type, crawler, env, f'get_videoList异常:{e}\n')
+            #     cls.i = 0
+            #     return
 
     @classmethod
-    def download_publish(cls, log_type, crawler, video_dict, oss_endpoint, env, driver: WebDriver):
+    def download_publish(cls, log_type, crawler, video_dict, env, driver: WebDriver):
         # 下载视频
         Common.download_method(log_type=log_type, crawler=crawler, text='video', title=video_dict['video_title'], url=video_dict['video_url'])
         ffmpeg_dict = Common.ffmpeg(log_type, crawler, f"./{crawler}/videos/{video_dict['video_title']}/video.mp4")
@@ -301,7 +301,7 @@ class GanggangdouchuanRecommend:
                                                   strategy="推荐榜爬虫策略",
                                                   our_uid="recommend",
                                                   env=env,
-                                                  oss_endpoint=oss_endpoint)
+                                                  oss_endpoint="out")
         if env == 'dev':
             our_video_link = f"https://testadmin.piaoquantv.com/cms/post-detail/{our_video_id}/info"
         else:
@@ -367,5 +367,5 @@ class GanggangdouchuanRecommend:
 
 
 if __name__ == '__main__':
-    GanggangdouchuanRecommend.start_wechat('recommend', 'ganggangdouchuan', 'inner', 'prod')
+    GanggangdouchuanRecommend.start_wechat('recommend', 'inner', 'prod')
     pass