spiders_config.yaml 1.2 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455
  1. default:
  2. base_url: http://8.217.192.46:8889
  3. request_timeout: 30
  4. max_retries: 3
  5. headers:
  6. {"Content-Type": "application/json"}
  7. benshanzhufurecommend:
  8. platform: benshanzhufu
  9. mode: recommend
  10. path: /crawler/ben_shan_zhu_fu/recommend
  11. method: post
  12. request_body:
  13. cursor: "{{next_cursor}}"
  14. loop_times: 200
  15. loop_interval: 5
  16. feishu_sheetid: "aTSJH4"
  17. response_parse:
  18. data: "$.data"
  19. next_cursor: "$.data.next_cursor"
  20. data_path: "$.data.data"
  21. fields:
  22. video_id: "$.nid"
  23. video_title: "$.title"
  24. play_cnt: 0
  25. publish_time_stamp: "$.update_time"
  26. out_user_id: "$.nid"
  27. cover_url: "$.video_cover"
  28. like_cnt: 0
  29. video_url: "$.video_url"
  30. out_video_id: "$.nid"
  31. yuannifuqimanmanrecommend:
  32. platform: yuannifuqimanman
  33. mode: recommend
  34. path: /crawler/yuan_ni_fu_qi_man_man/recommend
  35. method: post
  36. request_body:
  37. cursor: "{{next_cursor}}"
  38. loop_times: 200
  39. loop_interval: 5
  40. feishu_sheetid: "golXy9"
  41. response_parse:
  42. data: "$.data"
  43. next_cursor: "$.data.next_cursor"
  44. data_path: "$.data.data"
  45. fields:
  46. video_id: "$.nid"
  47. video_title: "$.title"
  48. out_user_id: "$.nid"
  49. cover_url: "$.video_cover"
  50. video_url: "$.video_url"
  51. out_video_id: "$.nid"