AutoScraperX/configs/spiders_config.yaml

default:
  base_url: http://8.217.192.46:8889
  request_timeout: 30
  headers:
    {"Content-Type": "application/json"}
benshanzhufu:
  mode: recommend
  path: /crawler/ben_shan_zhu_fu/recommend
  method: post
  request_body:
    cursor: "1"
  paging: true
  max_pages: 5
  etl_hook: "process_video_obj"
  response_parse:
    next_cursor: "$.data.next_cursor"
    data_path: "$.data.data"
    fields:
      video_id: "$.nid"
      video_title: "$.title"
      play_cnt: 0
      publish_time_stamp: "$.update_time"
      out_user_id: "$.nid"
      cover_url: "$.video_cover"
      like_cnt: 0
      video_url: "$.video_url"
      out_video_id: "$.nid"


zhongqingkandian:
  mode: recommend
  path: "/zqkd"
  paging: true
  max_pages: 5
  db_config:
    table: "zhongqingkandian"
  etl_hook: "process_video_obj"
  parse:
    data_path: "$.data[*]"
    fields:
      title: "$.title"
      vid: "$.id"
      cover: "$.cover"
      url: "$.video_url"
  custom_class: my_crawlers.ZhongqingKandianCrawler

fuqihaoyundao:
  url: "/fuqi"
  method: "POST"
  paging: false
  retry_times: 2
  etl_hook: "process_video_obj"
  parse:
    data_path: "$.videos[*]"
    fields:
      id: "$.id"
      name: "$.name"
      mp4: "$.url"