spiders_config.yaml 1.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354
  1. default:
  2. base_url: http://8.217.192.46:8889
  3. request_timeout: 30
  4. headers:
  5. {"Content-Type": "application/json"}
  6. benshanzhufurecommend:
  7. platform: benshanzhufu
  8. mode: recommend
  9. path: /crawler/ben_shan_zhu_fu/recommend
  10. method: post
  11. request_body:
  12. cursor: "{{next_cursor}}"
  13. loop_times: 200
  14. loop_interval: 5
  15. feishu_sheetid: "aTSJH4"
  16. response_parse:
  17. data: "$.data"
  18. next_cursor: "$.data.next_cursor"
  19. data_path: "$.data.data"
  20. fields:
  21. video_id: "$.nid"
  22. video_title: "$.title"
  23. play_cnt: 0
  24. publish_time_stamp: "$.update_time"
  25. out_user_id: "$.nid"
  26. cover_url: "$.video_cover"
  27. like_cnt: 0
  28. video_url: "$.video_url"
  29. out_video_id: "$.nid"
  30. yuannifuqimanmanrecommend:
  31. platform: yuannifuqimanman
  32. mode: recommend
  33. path: /crawler/yuan_ni_fu_qi_man_man/recommend
  34. method: post
  35. request_body:
  36. cursor: "{{next_cursor}}"
  37. loop_times: 50
  38. loop_interval: 5
  39. feishu_sheetid: "golXy9"
  40. response_parse:
  41. data: "$.data"
  42. next_cursor: "$.data.next_cursor"
  43. data_path: "$.data.data"
  44. fields:
  45. video_id: "$.nid"
  46. video_title: "$.title"
  47. out_user_id: "$.nid"
  48. cover_url: "$.video_cover"
  49. video_url: "$.video_url"
  50. out_video_id: "$.nid"