Browse Source

初始化项目

罗俊辉 1 year ago
parent
commit
657d1c811e

+ 22 - 2
README.md

@@ -1,3 +1,23 @@
-# automatic_crawler
+## automatic_crawler
 
-python+appium+chromedriver 模拟人类新闻采集爬虫
+###### python+appium+chromedriver 模拟人类新闻采集爬虫
+
+### common   
+通用功能
+###### aliyun  阿里云api
+###### mysql  mysql api
+###### proxies  代理
+### config  
+配置文件
+
+### functions   
+特殊功能方法
+
+### items   
+规范化item,保证送往ETL的消息格式正确
+
+### pipeline   
+管道,所有视频信息在发生之前需要过管道的规则判断
+
+### spider   
+爬虫代码

+ 0 - 0
application/__init__.py


+ 0 - 0
application/common/__init__.py


+ 0 - 0
application/common/aliyun/__init__.py


+ 0 - 0
application/common/mysql/__init__.py


+ 0 - 0
application/common/proxies/__init__.py


+ 0 - 0
application/config/__init__.py


+ 0 - 0
application/functions/__init__.py


+ 0 - 0
application/items/__init__.py


+ 0 - 0
application/items/items.py


+ 0 - 0
application/pipeline/__init__.py


+ 0 - 0
application/pipeline/pipeline.py


+ 0 - 0
application/pipeline/pipeline_dev.py


+ 0 - 0
application/spider/__init__.py


+ 0 - 0
application/spider/ad_click/__init__.py


+ 1 - 0
application/spider/ad_click/piaoquan_tv_v1.py

@@ -0,0 +1 @@
+import time

+ 0 - 0
application/spider/crawler/__init__.py


+ 37 - 0
docker-compose.yml

@@ -0,0 +1,37 @@
+version: '3.9'
+
+services:
+  appium_ad_1:
+    image: appium/appium:1.18.0-p0ad
+    container_name: appium_piaoquan_150
+    environment:
+      - REMOTE_ADB=true
+      - ANDROID_DEVICES=192.168.81.92:5555
+    volumes:
+      - /Users/luojunhui/Downloads/chromedriver-linux64-116/chromedriver:/usr/bin/chromedriver
+    restart: always
+    ports:
+      - '4750:4723'
+  appium_ad_2:
+    image: appium/appium:1.18.0-p0ad
+    container_name: appium_piaoquan_151
+    environment:
+      - REMOTE_ADB=true
+      - ANDROID_DEVICES=192.168.81.92:5555
+    volumes:
+      - /Users/luojunhui/Downloads/chromedriver-linux64-116/chromedriver:/usr/bin/chromedriver
+    restart: always
+    ports:
+      - '4751:4723'
+
+  appium_ad_3:
+    image: appium/appium:1.18.0-p0ad
+    container_name: appium_piaoquan_152
+    environment:
+      - REMOTE_ADB=true
+      - ANDROID_DEVICES=192.168.81.92:5555
+    volumes:
+      - /Users/luojunhui/Downloads/chromedriver-linux64-116/chromedriver:/usr/bin/chromedriver
+    restart: always
+    ports:
+      - '4752:4723'

+ 0 - 0
scheduler/__init__.py