AutoScraperX/config/spiders_config.yaml

default:
  base_url: http://8.217.192.46:8889
  request_timeout: 30
  max_retries: 3
  headers:
    {"Content-Type": "application/json"}

benshanzhufurecommend:
  platform: benshanzhufu
  mode: recommend
  path: /crawler/ben_shan_zhu_fu/recommend
  method: post
  request_body:
    cursor: "{{next_cursor}}"
  loop_times: 7
  loop_interval:
    min: 30
    max: 60
  feishu_sheetid: "aTSJH4"
  response_parse:
    data: "$.data"
    next_cursor: "$.data.next_cursor"
    data_path: "$.data.data"
    fields:
      video_id: "$.nid"
      video_title: "$.title"
      play_cnt: 0
      publish_time_stamp: "$.update_time"
      out_user_id: "$.nid"
      cover_url: "$.video_cover"
      like_cnt: 0
      video_url: "$.video_url"
      out_video_id: "$.nid"


yuannifuqimanmanrecommend:
  platform: yuannifuqimanman
  mode: recommend
  path: /crawler/yuan_ni_fu_qi_man_man/recommend
  method: post
  request_body:
    cursor: "{{next_cursor}}"
  loop_times: 100
  loop_interval:
    min: 30
    max: 60
  feishu_sheetid: "golXy9"
  response_parse:
    data: "$.data"
    next_cursor: "$.data.next_cursor"
    data_path: "$.data.data"
    fields:
      video_id: "$.nid"
      video_title: "$.title"
      out_user_id: "$.nid"
      cover_url: "$.video_cover"
      video_url: "$.video_url"
      out_video_id: "$.nid"

xiaoniangaoauthor:
  platform: xiaoniangao
  mode: author
  path: /crawler/xiao_nian_gao_plus/blogger
  method: post
  request_body:
      cursor: "{{next_cursor}}"
      account_id: "{{uid}}" # 数据库的uid
  loop_times: 100
  loop_interval:
    min: 5
    max: 10
  feishu_sheetid: "K0gA9Y"
  response_parse:
    uid: "$.uid" # 数据库的uid
    next_cursor: "$.data.next_cursor"
    data: "$.data"
    has_more: "$.data.has_more"
    data_path: "$.data.data"
    fields:
      video_title: "$.title"
      duration: "$.du"
      play_cnt: "$.play_pv"
      like_cnt: "$.favor.total"
      comment_cnt: "$.comment_count"
      share_cnt: "$.share"
      width: "$.w"
      height: "$.h"
      avatar_url: "$.user.hurl"
      cover_url: "$.url"
      video_url: "$.v_url"
      out_user_id: "$.user.mid"
      out_video_id: "$.vid"
      publish_time_stamp: "$.t"
#
#xiaoniangaorecommend:
#  platform: xiaoniangao
#  mode: recommend
#  path: crawler/xiao_nian_gao_plus/recommend
#  method: post
#  request_body:
#
#  loop_times: 100
#  loop_interval:
#    min: 5
#    max: 20
#  feishu_sheetid: "D1nVxQ"
#  response_parse:
#    uid: "$.uid" # 数据库的uid
#    next_cursor: "$.cursor"
#    data: "$.data"
#    has_more: "$.data.has_more"
#    data_path: "$.data.data"
#    fields:
#      video_title: "$.title"
#      duration: 0
#      play_cnt: "$.play_pv"
#      like_cnt: 0
#      comment_cnt: "$.comment_count"
#      share_cnt: "$.share"
#      width: "$.w"
#      height: "$.h"
#      avatar_url: "$.user.hurl"
#      cover_url: "$.url"
#      video_url: "$.v_url"
#      out_user_id:
#      out_video_id: "$.id"
#      publish_time_stamp: "$.t"