| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293 |
- default:
- base_url: http://8.217.192.46:8889
- request_timeout: 30
- headers:
- {"Content-Type": "application/json"}
- benshanzhufurecommend:
- platform: benshanzhufu
- mode: recommend
- path: /crawler/ben_shan_zhu_fu/recommend
- method: post
- request_body:
- cursor: "1"
- loop_times: 2
- loop_interval: 5
- response_parse:
- next_cursor: "$.data.next_cursor"
- data_path: "$.data.data"
- fields:
- video_id: "$.nid"
- video_title: "$.title"
- play_cnt: 0
- publish_time_stamp: "$.update_time"
- out_user_id: "$.nid"
- cover_url: "$.video_cover"
- like_cnt: 0
- video_url: "$.video_url"
- out_video_id: "$.nid"
- xngtjl_recommend_prod:
- platform: xiaoniangaotuijianliu
- mode: recommend
- path: /crawler/ben_shan_zhu_fu/recommend
- method: post
- request_body:
- cursor: "1"
- loop_times: 2
- etl_hook: "process_video_obj"
- response_parse:
- next_cursor: "$.data.next_cursor"
- data_path: "$.data.data"
- fields:
- video_id: "$.nid"
- video_title: "$.title"
- play_cnt: 0
- publish_time_stamp: "$.update_time"
- out_user_id: "$.nid"
- cover_url: "$.video_cover"
- like_cnt: 0
- video_url: "$.video_url"
- out_video_id: "$.nid"
- post_actions:
- - trigger: after_video_processed
- endpoint: "http://example.com/notify"
- payload:
- zhongqingkandian:
- mode: recommend
- path: "/zqkd"
- paging: true
- max_pages: 5
- db_config:
- table: "zhongqingkandian"
- etl_hook: "process_video_obj"
- parse:
- data_path: "$.data[*]"
- fields:
- title: "$.title"
- vid: "$.id"
- cover: "$.cover"
- url: "$.video_url"
- custom_class: my_crawlers.ZhongqingKandianCrawler
- fuqihaoyundao:
- url: "/fuqi"
- method: "POST"
- paging: false
- retry_times: 2
- etl_hook: "process_video_obj"
- parse:
- data_path: "$.videos[*]"
- fields:
- id: "$.id"
- name: "$.name"
- mp4: "$.url"
|
