update

ganggangdouchuan/ganggangdouchuan_main/run_ganggangdouchuan_recommend.py

@@ -11,7 +11,9 @@ from ganggangdouchuan.ganggangdouchuan_recommend.ganggangdouchuan_recommend impo
 
 
 def main(log_type, crawler, env):
-    oss_endpoint = "inner"
+    oss_endpoint = "inner"  # 阿里云 Windows
+    oss_endpoint = "out"  # 线下 Mac
+
     Common.logger(log_type, crawler).info('开始抓取:刚刚都传小程序\n')
     Common.logging(log_type, crawler, env, '开始抓取:刚刚都传小程序\n')
     ganggangdouchuan_start_time = int(time.time())

ganggangdouchuan/ganggangdouchuan_recommend/ganggangdouchuan_recommend.py

@@ -27,18 +27,18 @@ class GanggangdouchuanRecommend:
     def start_wechat(cls, log_type, crawler, oss_endpoint, env):
         try:
             if env == "dev":
-                # chromedriverExecutable = '/Users/wangkun/Downloads/chromedriver/chromedriver_v107/chromedriver'
-                chromedriverExecutable = 'C:\\chromedriver\\chromedriver.exe'  # 阿里云 Windows
+                chromedriverExecutable = '/Users/wangkun/Downloads/chromedriver/chromedriver_v107/chromedriver'
+                # chromedriverExecutable = 'C:\\chromedriver\\chromedriver.exe'  # 阿里云 Windows
             else:
-                # chromedriverExecutable = '/Users/piaoquan/Downloads/chromedriver'  # Mac 爬虫机器
-                chromedriverExecutable = 'C:\\chromedriver\\chromedriver.exe'  # 阿里云 Windows
+                chromedriverExecutable = '/Users/piaoquan/Downloads/chromedriver'  # Mac 爬虫机器
+                # chromedriverExecutable = 'C:\\chromedriver\\chromedriver.exe'  # 阿里云 Windows
 
             Common.logger(log_type, crawler).info('启动微信')
             Common.logging(log_type, crawler, env, '启动微信')
             caps = {
                 "platformName": "Android",  # 手机操作系统 Android / iOS
                 "deviceName": "a0a65126",  # 连接的设备名（模拟器或真机），安卓可以随便写
-                "udid": "emulator-5554",  # 指定 adb devices 中的哪一台设备
+                # "udid": "emulator-5554",  # 指定 adb devices 中的哪一台设备
                 "platforVersion": "12",  # 手机对应的系统版本
                 "appPackage": "com.tencent.mm",  # 被测APP的包名，乐活圈 Android
                 "appActivity": ".ui.LauncherUI",  # 启动的Activity名
@@ -86,7 +86,8 @@ class GanggangdouchuanRecommend:
             Common.logger(log_type, crawler).info('打开小程序"刚刚都传"')
             Common.logging(log_type, crawler, env, '打开小程序"刚刚都传"')
             driver.find_elements(By.XPATH, '//*[@text="刚刚都传"]')[-1].click()
-            time.sleep(40)
+            # time.sleep(40)
+            time.sleep(10)
             cls.get_videoList(log_type, crawler, oss_endpoint, env, driver)
             driver.quit()
             Common.logger(log_type, crawler).info('退出微信成功\n')
@@ -171,7 +172,6 @@ class GanggangdouchuanRecommend:
 
     @classmethod
     def repeat_video_url(cls, log_type, crawler, video_url, env):
-        # sql = f""" select * from crawler_video where platform="{cls.platform}" and video_url="{video_url}"; """
         sql = f""" select * from crawler_video where platform in ("{cls.platform}", "{crawler}") and video_url="{video_url}"; """
         repeat_video = MysqlHelper.get_values(log_type, crawler, sql, env)
         return len(repeat_video)
@@ -367,5 +367,5 @@ class GanggangdouchuanRecommend:
 
 
 if __name__ == '__main__':
-    GanggangdouchuanRecommend.start_wechat('recommend', 'ganggangdouchuan', 'inner', 'dev')
+    GanggangdouchuanRecommend.start_wechat('recommend', 'ganggangdouchuan', 'inner', 'prod')
     pass

jixiangxingfu/jixiangxingfu_recommend/jixiangxingfu_recommend.py

@@ -51,17 +51,17 @@ class JixiangxingfuRecommend:
     def start_wechat(cls, log_type, crawler, env):
         try:
             if env == "dev":
-                # chromedriverExecutable = '/Users/wangkun/Downloads/chromedriver/chromedriver_v107/chromedriver'
-                chromedriverExecutable = 'C:\\chromedriver\\chromedriver.exe'  # 阿里云 Windows
+                chromedriverExecutable = '/Users/wangkun/Downloads/chromedriver/chromedriver_v107/chromedriver'
+                # chromedriverExecutable = 'C:\\chromedriver\\chromedriver.exe'  # 阿里云 Windows
             else:
-                # chromedriverExecutable = '/Users/piaoquan/Downloads/chromedriver'  # Mac 爬虫机器
-                chromedriverExecutable = 'C:\\chromedriver\\chromedriver.exe'  # 阿里云 Windows
+                chromedriverExecutable = '/Users/piaoquan/Downloads/chromedriver'  # Mac 爬虫机器
+                # chromedriverExecutable = 'C:\\chromedriver\\chromedriver.exe'  # 阿里云 Windows
 
             Common.logger(log_type, crawler).info('启动微信')
             caps = {
                 "platformName": "Android",  # 手机操作系统 Android / iOS
                 "deviceName": "a0a65126",  # 连接的设备名（模拟器或真机），安卓可以随便写
-                "udid": "emulator-5554",  # 指定 adb devices 中的哪一台设备
+                # "udid": "emulator-5554",  # 指定 adb devices 中的哪一台设备
                 "platforVersion": "12",  # 手机对应的系统版本
                 "appPackage": "com.tencent.mm",  # 被测APP的包名，乐活圈 Android
                 "appActivity": ".ui.LauncherUI",  # 启动的Activity名
@@ -109,7 +109,8 @@ class JixiangxingfuRecommend:
             driver.find_elements(By.XPATH, '//*[@text="祝福每天好运来相伴"]')[-1].click()
 
             # 获取视频信息
-            time.sleep(60)
+            time.sleep(20)
+            # time.sleep(60)
             cls.get_videoList(log_type, crawler, driver, env)
 
             # 退出微信
@@ -222,7 +223,7 @@ class JixiangxingfuRecommend:
         # 切换到小程序
         cls.check_to_applet(log_type, crawler, driver)
 
-        time.sleep(20)
+        time.sleep(10)
         index = 0
         while True:
             try:
@@ -340,7 +341,7 @@ class JixiangxingfuRecommend:
                                                   strategy="推荐榜爬虫策略",
                                                   our_uid="recommend",
                                                   env=env,
-                                                  oss_endpoint="inner")
+                                                  oss_endpoint="out")
         if env == 'dev':
             our_video_link = f"https://testadmin.piaoquantv.com/cms/post-detail/{our_video_id}/info"
         else:
@@ -407,6 +408,5 @@ class JixiangxingfuRecommend:
 
 
 if __name__ == "__main__":
-    JixiangxingfuRecommend.start_wechat('recommend', 'jixiangxingfu', 'dev')
-    # print(JixiangxingfuRecommend.jixiangxingfu_config("recommend", "jixiangxingfu", "filter", "dev"))
+    JixiangxingfuRecommend.start_wechat('recommend', 'jixiangxingfu', 'prod')
     pass

main/process_offline.sh

@@ -32,8 +32,6 @@ fi
 # 知青天天看
 if [[ "$time" > "00:00:00" ]] && [[ "$time" < "08:59:59" ]]; then
   echo "$(date "+%Y-%m-%d %H:%M:%S") 开始启动 知青天天看 爬虫脚本任务" >> ${log_path}
-#  adb kill-server
-#  adb start-server
   ps aux | grep run_zhongmiaoyinxin | grep -v grep | awk '{print $2}' | xargs kill -9
   ps aux | grep run_ganggangdouchuan | grep -v grep | awk '{print $2}' | xargs kill -9
   ps aux | grep run_jixiangxingfu | grep -v grep | awk '{print $2}' | xargs kill -9
@@ -53,8 +51,6 @@ fi
 # 刚刚都传
 if [[ "$time" > "09:00:00" ]] && [[ "$time" < "12:59:59" ]]; then
   echo "$(date "+%Y-%m-%d %H:%M:%S") 开始启动 刚刚都传 爬虫脚本任务" >> ${log_path}
-#  adb kill-server
-#  adb start-server
   ps aux | grep run_zhongmiaoyinxin | grep -v grep | awk '{print $2}' | xargs kill -9
   ps aux | grep run_zhiqingtiantiankan | grep -v grep | awk '{print $2}' | xargs kill -9
   ps aux | grep run_jixiangxingfu | grep -v grep | awk '{print $2}' | xargs kill -9
@@ -74,8 +70,6 @@ fi
 # 吉祥幸福
 if [[ "$time" > "13:00:00" ]] && [[ "$time" < "16:59:59" ]]; then
   echo "$(date "+%Y-%m-%d %H:%M:%S") 开始启动 吉祥幸福 爬虫脚本任务" >> ${log_path}
-#  adb kill-server
-#  adb start-server
   ps aux | grep run_zhongmiaoyinxin | grep -v grep | awk '{print $2}' | xargs kill -9
   ps aux | grep run_zhiqingtiantiankan | grep -v grep | awk '{print $2}' | xargs kill -9
   ps aux | grep run_ganggangdouchuan | grep -v grep | awk '{print $2}' | xargs kill -9
@@ -95,8 +89,6 @@ fi
 # 众妙音信
 if [[ "$time" > "17:00:00" ]] && [[ "$time" < "23:59:59" ]]; then
   echo "$(date "+%Y-%m-%d %H:%M:%S") 开始启动 众妙音信 爬虫脚本任务" >> ${log_path}
-#  adb kill-server
-#  adb start-server
   ps aux | grep run_ganggangdouchuan | grep -v grep | awk '{print $2}' | xargs kill -9
   ps aux | grep run_zhiqingtiantiankan | grep -v grep | awk '{print $2}' | xargs kill -9
   ps aux | grep run_jixiangxingfu | grep -v grep | awk '{print $2}' | xargs kill -9

zhiqingtiantiankan/zhiqingtiantiankan_recommend/zhiqingtiantiankan_recommend.py

@@ -52,15 +52,15 @@ class ZhiqingtiantiankanRecommend:
         try:
             Common.logger(log_type, crawler).info('启动微信')
             if env == "dev":
-                # chromedriverExecutable = '/Users/wangkun/Downloads/chromedriver/chromedriver_v107/chromedriver'
-                chromedriverExecutable = 'C:\\chromedriver\\chromedriver.exe'  # 阿里云 Windows
+                chromedriverExecutable = '/Users/wangkun/Downloads/chromedriver/chromedriver_v107/chromedriver'
+                # chromedriverExecutable = 'C:\\chromedriver\\chromedriver.exe'  # 阿里云 Windows
             else:
-                # chromedriverExecutable = '/Users/piaoquan/Downloads/chromedriver'  # Mac 爬虫机器
-                chromedriverExecutable = 'C:\\chromedriver\\chromedriver.exe'  # 阿里云 Windows
+                chromedriverExecutable = '/Users/piaoquan/Downloads/chromedriver'  # Mac 爬虫机器
+                # chromedriverExecutable = 'C:\\chromedriver\\chromedriver.exe'  # 阿里云 Windows
             caps = {
                 "platformName": "Android",  # 手机操作系统 Android / iOS
                 "deviceName": "Android",  # 连接的设备名（模拟器或真机），安卓可以随便写
-                "udid": "emulator-5554",  # 指定 adb devices 中的哪一台设备
+                # "udid": "emulator-5554",  # 指定 adb devices 中的哪一台设备
                 "platforVersion": "12",  # 手机对应的系统版本
                 "appPackage": "com.tencent.mm",  # 被测APP的包名
                 "appActivity": ".ui.LauncherUI",  # 启动的Activity名
@@ -108,7 +108,8 @@ class ZhiqingtiantiankanRecommend:
             driver.find_elements(By.XPATH, '//*[@text="知青天天看"]')[-1].click()
 
             # 获取视频信息
-            time.sleep(30)
+            # time.sleep(30)
+            time.sleep(10)
             cls.get_videoList(log_type, crawler, driver, env)
 
             # 退出微信
@@ -342,7 +343,7 @@ class ZhiqingtiantiankanRecommend:
                                                   strategy="推荐榜爬虫策略",
                                                   our_uid="recommend",
                                                   env=env,
-                                                  oss_endpoint="inner")
+                                                  oss_endpoint="out")
         if env == 'dev':
             our_video_link = f"https://testadmin.piaoquantv.com/cms/post-detail/{our_video_id}/info"
         else:
@@ -409,7 +410,6 @@ class ZhiqingtiantiankanRecommend:
 
 
 if __name__ == '__main__':
-    ZhiqingtiantiankanRecommend.start_wechat('recommend', 'zhiqingtiantiankan', 'dev')
-    # print(ZhiqingtiantiankanRecommend.zhiqingtiantiankan_config('recommend', 'zhiqingtiantiankan', 'filter', 'dev'))
+    ZhiqingtiantiankanRecommend.start_wechat('recommend', 'zhiqingtiantiankan', 'prod')
 
     pass

zhongmiaoyinxin/zhongmiaoyinxin_main/run_zhongmiaoyinxin_recommend.py

@@ -12,7 +12,7 @@ from zhongmiaoyinxin.zhongmiaoyinxin_recommend.zhongmiaoyinxin_recommend import
 def main(log_type, crawler, env):
     Common.logger(log_type, crawler).info('开始抓取:众妙音信小程序\n')
     ZhongmiaoyinxinRecommend.start_wechat(log_type, crawler, env)
-    # Common.del_logs(log_type, crawler)
+    Common.del_logs(log_type, crawler)
     Common.logger(log_type, crawler).info('抓取完一轮\n')
 
 

zhongmiaoyinxin/zhongmiaoyinxin_recommend/zhongmiaoyinxin_recommend.py

@@ -51,17 +51,17 @@ class ZhongmiaoyinxinRecommend:
     def start_wechat(cls, log_type, crawler, env):
         try:
             if env == "dev":
-                # chromedriverExecutable = "/Users/wangkun/Downloads/chromedriver/chromedriver_v107/chromedriver"
-                chromedriverExecutable = 'C:\\chromedriver\\chromedriver.exe'  # 阿里云 Windows
+                chromedriverExecutable = "/Users/wangkun/Downloads/chromedriver/chromedriver_v107/chromedriver"
+                # chromedriverExecutable = 'C:\\chromedriver\\chromedriver.exe'  # 阿里云 Windows
             else:
-                # chromedriverExecutable = '/Users/piaoquan/Downloads/chromedriver'  # Mac 爬虫机器
-                chromedriverExecutable = 'C:\\chromedriver\\chromedriver.exe'  # 阿里云 Windows
+                chromedriverExecutable = '/Users/piaoquan/Downloads/chromedriver'  # Mac 爬虫机器
+                # chromedriverExecutable = 'C:\\chromedriver\\chromedriver.exe'  # 阿里云 Windows
 
             Common.logger(log_type, crawler).info('启动微信')
             caps = {
                 "platformName": "Android",  # 手机操作系统 Android / iOS
                 "deviceName": "a0a65126",  # 连接的设备名（模拟器或真机），安卓可以随便写
-                "udid": "emulator-5554",  # 指定 adb devices 中的哪一台设备
+                # "udid": "emulator-5554",  # 指定 adb devices 中的哪一台设备
                 "platforVersion": "12",  # 手机对应的系统版本
                 "appPackage": "com.tencent.mm",  # 被测APP的包名，乐活圈 Android
                 "appActivity": ".ui.LauncherUI",  # 启动的Activity名
@@ -108,7 +108,8 @@ class ZhongmiaoyinxinRecommend:
             Common.logger(log_type, crawler).info('打开小程序"西瓜悦"')
             driver.find_elements(By.XPATH, '//*[@text="西瓜悦"]')[-1].click()
 
-            time.sleep(40)
+            # time.sleep(40)
+            time.sleep(10)
             cls.get_videoList(log_type, crawler, driver, env)
 
             cls.quit(log_type, crawler, driver)
@@ -336,7 +337,7 @@ class ZhongmiaoyinxinRecommend:
                                                   strategy="推荐榜爬虫策略",
                                                   our_uid="recommend",
                                                   env=env,
-                                                  oss_endpoint="inner")
+                                                  oss_endpoint="out")
         if env == 'dev':
             our_video_link = f"https://testadmin.piaoquantv.com/cms/post-detail/{our_video_id}/info"
         else:
@@ -405,6 +406,5 @@ class ZhongmiaoyinxinRecommend:
 
 
 if __name__ == '__main__':
-    # print(ZhongmiaoyinxinRecommend.zhongmiaoyinxin_config('recommend', 'zhongmiaoyinxin', "filter", "dev"))
-    ZhongmiaoyinxinRecommend.start_wechat("recommend", "zhongmiaoyinxin", "dev")
+    ZhongmiaoyinxinRecommend.start_wechat("recommend", "zhongmiaoyinxin", "prod")
     pass