| 1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192 |
- default:
- base_url: http://8.217.192.46:8889
- request_timeout: 30
- headers:
- {"Content-Type": "application/json"}
- benshanzhufurecommend:
- platform: benshanzhufu
- mode: recommend
- path: /crawler/ben_shan_zhu_fu/recommend
- method: post
- request_body:
- cursor: "{{next_cursor|'1'}}"
- loop_times: 200
- loop_interval: 5
- feishu_sheetid: "aTSJH4"
- response_parse:
- next_cursor: "$.data.next_cursor"
- data_path: "$.data.data"
- fields:
- video_id: "$.nid"
- video_title: "$.title"
- play_cnt: 0
- publish_time_stamp: "$.update_time"
- out_user_id: "$.nid"
- cover_url: "$.video_cover"
- like_cnt: 0
- video_url: "$.video_url"
- out_video_id: "$.nid"
- xngtjl_recommend_prod:
- platform: xiaoniangaotuijianliu
- mode: recommend
- path: /crawler/ben_shan_zhu_fu/recommend
- method: post
- request_body:
- cursor: "1"
- loop_times: 2
- etl_hook: "process_video_obj"
- response_parse:
- next_cursor: "$.data.next_cursor"
- data_path: "$.data.data"
- fields:
- video_id: "$.nid"
- video_title: "$.title"
- play_cnt: 0
- publish_time_stamp: "$.update_time"
- out_user_id: "$.nid"
- cover_url: "$.video_cover"
- like_cnt: 0
- video_url: "$.video_url"
- out_video_id: "$.nid"
- post_actions:
- - trigger: after_video_processed
- endpoint: "http://example.com/notify"
- payload:
- zhongqingkandian:
- mode: recommend
- path: "/zqkd"
- paging: true
- max_pages: 5
- db_config:
- table: "zhongqingkandian"
- etl_hook: "process_video_obj"
- parse:
- data_path: "$.data[*]"
- fields:
- title: "$.title"
- vid: "$.id"
- cover: "$.cover"
- url: "$.video_url"
- custom_class: my_crawlers.ZhongqingKandianCrawler
- fuqihaoyundao:
- url: "/fuqi"
- method: "POST"
- paging: false
- retry_times: 2
- etl_hook: "process_video_obj"
- parse:
- data_path: "$.videos[*]"
- fields:
- id: "$.id"
- name: "$.name"
- mp4: "$.url"
|
