jierizhufuzanmenwugufengdeng_recommend.py 6.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148
  1. import os
  2. import json
  3. import random
  4. import sys
  5. import time
  6. import uuid
  7. import requests
  8. sys.path.append(os.getcwd())
  9. from common.video_item import VideoItem
  10. from common import PiaoQuanPipeline, AliyunLogger, tunnel_proxies, get_redirect_url
  11. from common.mq import MQ
  12. class Jrzfzmwgfdspcheduling(object):
  13. def __init__(self, platform, mode, rule_dict, user_list, env):
  14. self.platform = platform
  15. self.mode = mode
  16. self.rule_dict = rule_dict
  17. self.user_list = user_list
  18. self.env = env
  19. self.download_cnt = 0
  20. self.mq = MQ(topic_name="topic_crawler_etl_" + self.env)
  21. self.limit_flag = False
  22. def get_video_list(self):
  23. headers = {
  24. 'Host': 'api.qianjiegogogo.top',
  25. 'Content-Type': 'application/json',
  26. 'Accept-Language': 'zh-cn',
  27. 'Accept': '*/*',
  28. 'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E217 MicroMessenger/6.8.0(0x16080000) NetType/WIFI Language/en Branch/Br_trunk MiniProgramEnv/Mac',
  29. 'Referer': 'https://servicewechat.com/wxd811c29b02c88084/11/page-frame.html',
  30. 'token': 'eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpYXQiOjE3MDI5NjM2ODEsIm5iZiI6MTcwMjk2MzY4MSwiZXhwIjoxNzAyOTcwODgxLCJkYXRhIjp7InVzZXJfaWQiOjI1MzI4NDg5NX19.CmBIeZYuChwF216JUYXVQzWyD9kMySdIYERuJmjla6o',
  31. 'ik': 'b326b5062b2f0e69046810717534cb09'
  32. }
  33. page_index = 1
  34. while True:
  35. time.sleep(random.randint(1, 10))
  36. try:
  37. if self.limit_flag:
  38. AliyunLogger.logging(
  39. code="2000",
  40. platform=self.platform,
  41. mode=self.mode,
  42. env=self.env,
  43. message="本轮已经抓取到足够的数据,自动退出\t{}".format(self.download_cnt),
  44. )
  45. return
  46. else:
  47. url = "https://api.qianjiegogogo.top/index.php?s=mobile/Video/getList&cid=1&page={}&api_version=4&appid=wxd811c29b02c88084&version=1.9.1&env_version=release".format(page_index)
  48. response = requests.post(url, headers=headers)
  49. video_list = response.json()["data"]["list"]
  50. if video_list:
  51. for index, video_obj in enumerate(video_list, 1):
  52. try:
  53. if video_obj.get("title"):
  54. AliyunLogger.logging(
  55. code="1001",
  56. platform=self.platform,
  57. mode=self.mode,
  58. env=self.env,
  59. message="扫描到一条视频",
  60. data=video_obj,
  61. )
  62. self.process_video_obj(video_obj)
  63. except Exception as e:
  64. AliyunLogger.logging(
  65. code="3000",
  66. platform=self.platform,
  67. mode=self.mode,
  68. env=self.env,
  69. data=video_obj,
  70. message="抓取第{}条的时候出现问题, 报错信息是{}".format(index, e),
  71. )
  72. page_index += 1
  73. else:
  74. AliyunLogger.logging(
  75. code="2000",
  76. platform=self.platform,
  77. mode=self.mode,
  78. env=self.env,
  79. message="已经抓完了,自动退出"
  80. )
  81. return
  82. except Exception as e:
  83. AliyunLogger.logging(
  84. code="3000",
  85. platform=self.platform,
  86. mode=self.mode,
  87. env=self.env,
  88. message="抓取第{}页时候出现错误, 报错信息是{}".format(page_index + 1, e),
  89. )
  90. def process_video_obj(self, video_obj):
  91. trace_id = self.platform + str(uuid.uuid1())
  92. our_user = random.choice(self.user_list)
  93. video_url = video_obj['video_url']
  94. video_url = get_redirect_url(video_url)
  95. item = VideoItem()
  96. item.add_video_info("user_id", our_user["uid"])
  97. item.add_video_info("user_name", our_user["nick_name"])
  98. item.add_video_info("video_id", video_obj["id"])
  99. item.add_video_info("video_title", video_obj["title"])
  100. item.add_video_info("publish_time_stamp", int(time.time()))
  101. item.add_video_info("video_url", video_url)
  102. item.add_video_info("cover_url", video_obj["images"])
  103. item.add_video_info("out_video_id", video_obj["id"])
  104. item.add_video_info("platform", self.platform)
  105. item.add_video_info("strategy", self.mode)
  106. item.add_video_info("session", "{}-{}".format(self.platform, int(time.time())))
  107. mq_obj = item.produce_item()
  108. pipeline = PiaoQuanPipeline(
  109. platform=self.platform,
  110. mode=self.mode,
  111. rule_dict=self.rule_dict,
  112. env=self.env,
  113. item=mq_obj,
  114. trace_id=trace_id,
  115. )
  116. if pipeline.process_item():
  117. self.download_cnt += 1
  118. self.mq.send_msg(mq_obj)
  119. AliyunLogger.logging(
  120. code="1002",
  121. platform=self.platform,
  122. mode=self.mode,
  123. env=self.env,
  124. message="成功发送至 ETL",
  125. data=mq_obj,
  126. )
  127. if self.download_cnt >= int(
  128. self.rule_dict.get("videos_cnt", {}).get("min", 200)
  129. ):
  130. self.limit_flag = True
  131. if __name__ == '__main__':
  132. S = Jrzfzmwgfdspcheduling(
  133. platform="Jrzfzmwgfd",
  134. mode="recommend",
  135. env="dev",
  136. rule_dict={},
  137. user_list=[{'nick_name': "Ivring", 'uid': "1997"}, {'nick_name': "paul", 'uid': "1998"}]
  138. )
  139. S.get_video_list()