update

README.MD

@@ -112,7 +112,7 @@ ps aux | grep shipinhao_search
 ps aux | grep shipinhao_search | grep -v grep | awk '{print $2}' | xargs kill -9
 ```
 
-#### CPU/MEMORY 监控
+#### 207 服务器，CPU/MEMORY 监控
 ```commandline
 正式环境
 * * * * * /usr/bin/sh /root/piaoquan_crawler/monitor/monitor_main/run_monitor.sh monitor/monitor_main/run_cpu_memory.py "cpumemory" "monitor" "prod"
@@ -142,26 +142,41 @@ ps aux | grep run_monitor | grep -v grep | awk '{print $2}' | xargs kill -9
 /bin/sh /Users/wangkun/Desktop/crawler/piaoquan_crawler/main/process_mq.sh "xng" "xiaoniangao" "author" "dev"
 /bin/sh /Users/wangkun/Desktop/crawler/piaoquan_crawler/main/process_mq.sh "kykjk" "kanyikan" "recommend" "dev"
 
-207 服务器
-# 调用 MQ 爬虫守护进程
+207 服务器, 调用 MQ 爬虫守护进程
+# 岁岁年年迎福气
 * * * * * /usr/bin/sh /root/piaoquan_crawler/main/process_mq.sh "ssnnyfq" "suisuiniannianyingfuqi" "recommend" "prod"
+# 公众号（根据抓取目标用户数，自动计算需要启动 X 个进程同时抓取。每 100 个目标抓取用户，占用一个进程）
 * * * * * /usr/bin/sh /root/piaoquan_crawler/main/process_mq.sh "gzh" "gongzhonghao" "author" "prod"
+# 西瓜账号
 * * * * * /usr/bin/sh /root/piaoquan_crawler/main/process_mq.sh "xg" "xigua" "author" "prod"
+# 西瓜搜索
 * * * * * /usr/bin/sh /root/piaoquan_crawler/main/process_mq.sh "xg" "xigua" "search" "prod"
+# 本山祝福
 * * * * * /usr/bin/sh /root/piaoquan_crawler/main/process_mq.sh "bszf" "benshanzhufu" "recommend" "prod"
+# 快手推荐
 * * * * * /usr/bin/sh /root/piaoquan_crawler/main/process_mq.sh "ks" "kuaishou" "recommend" "prod"
+# 快手账号
 * * * * * /usr/bin/sh /root/piaoquan_crawler/main/process_mq.sh "ks" "kuaishou" "author" "prod"
+# 抖音推荐
 * * * * * /usr/bin/sh /root/piaoquan_crawler/main/process_mq.sh "dy" "douyin" "recommend" "prod"
+# 抖音账号
 * * * * * /usr/bin/sh /root/piaoquan_crawler/main/process_mq.sh "dy" "douyin" "author" "prod"
+# 小年糕播放榜
 * * * * * /usr/bin/sh /root/piaoquan_crawler/main/process_mq.sh "xng" "xiaoniangao" "play" "prod"
+# 小年糕上升榜
 * * * * * /usr/bin/sh /root/piaoquan_crawler/main/process_mq.sh "xng" "xiaoniangao" "hour" "prod"
+# 小年糕账号
 * * * * * /usr/bin/sh /root/piaoquan_crawler/main/process_mq.sh "xng" "xiaoniangao" "author" "prod"
+# 看一看推荐 1
 * * * * * /bin/sh /Users/lieyunye/Desktop/crawler/piaoquan_crawler/main/process_mq.sh "kyk" "kanyikan" "recommend" "prod"
+# 看一看推荐健康类
 * * * * * /bin/sh /Users/kanyikan/Desktop/crawler/piaoquan_crawler/main/process_mq.sh "kykjk" "kanyikan" "recommend" "prod"
+# 西瓜推荐 1
 * * * * * /bin/sh /Users/kanyikan/Desktop/crawler/piaoquan_crawler/main/process_mq.sh "xg" "xigua" "recommend" "prod"
-# 启动Appium /bin/sh /Users/wangkun/Desktop/crawler/piaoquan_crawler/main/start_appium.sh "local" "local" "dev"
-
-线下服务器
+# 西瓜推荐民生类
+* * * * * /bin/sh /Users/piaoquan/Desktop/piaoquan_crawler/main/process_mq.sh "xgms" "xigua" "recommend" "prod"
+# 线下 Mac，启动 Appium 
+/bin/sh /Users/wangkun/Desktop/crawler/piaoquan_crawler/main/start_appium.sh "local" "local" "dev"
 
 杀进程
 ps aux | grep suisuiniannianyingfuqi | grep -v grep | awk '{print $2}' | xargs kill -9

main/process_mq.sh

@@ -25,6 +25,11 @@ elif [ ${crawler} = "kyk" ];then
   profile_path=./base_profile
   python=python3
   log_path=${piaoquan_crawler_dir}main/main_logs/process-mq-$(date +%Y-%m-%d).log
+elif [ ${crawler} = "xgms" ];then
+  piaoquan_crawler_dir=/Users/piaoquan/Desktop/piaoquan_crawler/
+  profile_path=/etc/profile
+  python=python3
+  log_path=${piaoquan_crawler_dir}main/main_logs/process-mq-$(date +%Y-%m-%d).log
 elif [ ${crawler} = "xg" ] && [ ${log_type} = "recommend" ];then
   piaoquan_crawler_dir=/Users/kanyikan/Desktop/crawler/piaoquan_crawler/
   profile_path=/.base_profile

shipinhao/shipinhao_main/run_sph_recommend_dev.py

@@ -4,7 +4,6 @@
 import os
 import sys
 import time
-
 sys.path.append(os.getcwd())
 from common.common import Common
 from shipinhao.shipinhao_recommend.recommend_h5 import RecommendH5
@@ -39,4 +38,4 @@ class ShipinhaoRecommendMain:
 
 
 if __name__ == "__main__":
-    ShipinhaoRecommendMain.shipinhao_recommend_main("recommend", "shipinhao", "dev")
+    ShipinhaoRecommendMain.shipinhao_recommend_main("recommend", "shipinhao", "prod")

shipinhao/shipinhao_recommend/recommend_h5.py

@@ -27,7 +27,7 @@ class RecommendH5:
         if env == "dev":
             chromedriverExecutable = "/Users/wangkun/Downloads/chromedriver/chromedriver_v107/chromedriver"
         else:
-            chromedriverExecutable = '/Users/piaoquan/Downloads/chromedriver'
+            chromedriverExecutable = '/Users/lieyunye/Downloads/chromedriver/chromedriver_v107/chromedriver'
         caps = {
             "platformName": "Android",  # 手机操作系统 Android / iOS
             "deviceName": "Android",  # 连接的设备名（模拟器或真机），安卓可以随便写

shipinhao/shipinhao_recommend/shipinhao_recommend.py

@@ -34,7 +34,7 @@ class ShipinhaoRecommend:
         if env == "dev":
             chromedriverExecutable = "/Users/wangkun/Downloads/chromedriver/chromedriver_v107/chromedriver"
         else:
-            chromedriverExecutable = '/Users/piaoquan/Downloads/chromedriver'
+            chromedriverExecutable = '/Users/lieyunye/Downloads/chromedriver/chromedriver_v107/chromedriver'
         caps = {
             "platformName": "Android",  # 手机操作系统 Android / iOS
             "deviceName": "Android",  # 连接的设备名（模拟器或真机），安卓可以随便写

xigua/xigua_main/run_xg_recommend2.py → xigua/xigua_main/run_xgms_recommend.py

@@ -12,7 +12,7 @@ sys.path.append(os.getcwd())
 from common.common import Common
 from common.public import get_consumer, ack_message, task_fun_mq
 from common.scheduling_db import MysqlHelper
-from xigua.xigua_recommend.xg_recommend2 import XiguaRecommend
+from xigua.xigua_recommend.xgms_recommend import XiguaRecommend
 
 
 def main(log_type, crawler, topic_name, group_id, env):

xigua/xigua_recommend/xg_recommend2.py → xigua/xigua_recommend/xgms_recommend.py

@@ -7,7 +7,6 @@ import json
 import os
 import random
 import string
-import subprocess
 import sys
 import time
 import requests
@@ -304,6 +303,7 @@ class XiguaRecommend:
                 video_dict["out_user_id"] = video_dict["user_id"]
                 video_dict["platform"] = crawler
                 video_dict["strategy"] = log_type
+                video_dict["strategy_type"] = "ms"
                 video_dict["out_video_id"] = video_dict["video_id"]
                 video_dict["width"] = video_dict["video_width"]
                 video_dict["height"] = video_dict["video_height"]