dy_ls.py 3.6 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889
  1. import random
  2. import time
  3. import requests
  4. import json
  5. from common import Common
  6. from common.sql_help import sqlCollect
  7. class DYLS:
  8. @classmethod
  9. def get_dyls_list(cls, task_mark, url_id, number, mark):
  10. next_cursor = ""
  11. for i in range(50):
  12. url = "http://8.217.190.241:8888/crawler/dou_yin/blogger"
  13. payload = json.dumps({
  14. "account_id": url_id,
  15. "source": "抖查查",
  16. "cursor": next_cursor
  17. })
  18. headers = {
  19. 'Content-Type': 'application/json'
  20. }
  21. time.sleep(random.randint(1, 5))
  22. response = requests.request("POST", url, headers=headers, data=payload)
  23. response = response.json()
  24. list = []
  25. data_all_list = response["data"]
  26. has_more = data_all_list["has_more"]
  27. next_cursor = str(data_all_list["next_cursor"])
  28. try:
  29. data_list = data_all_list["data"]
  30. for data in data_list:
  31. # comment_count = data["comment_count"]
  32. # download_count = data["download_count"]
  33. share_count = data["share_count"]
  34. good_count = data["good_count"]
  35. # collect_count = data["collect_count"]
  36. duration = data["duration"]
  37. video_id = data["video_id"]
  38. old_title = data["video_desc"]
  39. status = sqlCollect.is_used(task_mark, video_id, mark, "抖音")
  40. if status:
  41. video_percent = '%.2f' % (int(share_count) / int(good_count))
  42. special = float(0.25)
  43. duration = duration / 1000
  44. if int(share_count) < 500 or float(video_percent) < special or int(duration) < 30 or int(duration) > 720:
  45. Common.logger("dy-ls").info(
  46. f"不符合规则:{task_mark},用户主页id:{url_id},视频id{video_id} ,分享:{share_count},点赞{good_count} ,时长:{int(duration)} ")
  47. continue
  48. video_url, image_url = cls.get_video(video_id)
  49. if video_url:
  50. all_data = {"video_id": video_id, "cover": image_url, "video_url": video_url, "rule": video_percent,
  51. "old_title": old_title}
  52. list.append(all_data)
  53. if len(list) == int(number):
  54. Common.logger("dy-ls").info(f"获取抖音历史视频总数:{len(list)}\n")
  55. return list
  56. if has_more == False:
  57. return list
  58. except Exception as exc:
  59. Common.logger("dy-ls").info(f"抖音历史数据获取失败:{exc}\n")
  60. return list
  61. @classmethod
  62. def get_video(cls, video_id):
  63. url = "http://8.217.190.241:8888/crawler/dou_yin/detail"
  64. payload = json.dumps({
  65. "content_id": str(video_id)
  66. })
  67. headers = {
  68. 'Content-Type': 'application/json'
  69. }
  70. response = requests.request("POST", url, headers=headers, data=payload)
  71. response = response.json()
  72. data = response["data"]["data"]
  73. video_url = data["video_url_list"][0]["video_url"]
  74. image_url = data["image_url_list"][0]["image_url"]
  75. return video_url, image_url
  76. if __name__ == '__main__':
  77. # DYLS.get_video("7314923922602954022")
  78. DYLS.get_dyls_list("1","MS4wLjABAAAADUObyc_aoKXnXnV01JEcZMdvU0_ZFvFnVQAU-weztOgHubCQont1aDrDASxWu8B6",1,"1")