zfsh_recommend.py 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192
  1. import json
  2. import time
  3. import uuid
  4. import requests
  5. from common.common import Common
  6. from common.scheduling_db import MysqlHelper
  7. from common.mq import MQ
  8. from common.public import download_rule, get_config_from_mysql
  9. proxies = {"http": None, "https": None}
  10. headers = {
  11. 'Host': 'mini.vvuiiu.cn',
  12. 'accept': '*/*',
  13. 'content-type': 'application/json',
  14. 'accept-language': 'zh-cn',
  15. 'user-agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E217 MicroMessenger/6.8.0(0x16080000) NetType/WIFI Language/en Branch/Br_trunk MiniProgramEnv/Mac',
  16. 'referer': 'https://servicewechat.com/wxa903dc775e85eb5b/11/page-frame.html'
  17. }
  18. class ZfshRecommend:
  19. platform = ("祝福生活视频")
  20. @classmethod
  21. def get_video_url(cls, id, uuid1, log_type, crawler, env):
  22. try:
  23. url = "https://mini.vvuiiu.cn/article/detail"
  24. payload = json.dumps({
  25. "id": id,
  26. "uuid": uuid1,
  27. "platform": 35,
  28. "appid": "wxa903dc775e85eb5b"
  29. })
  30. r = requests.post(url=url, headers=headers, data=payload, proxies=proxies, verify=False)
  31. if "data" not in r.text or r.status_code != 200:
  32. Common.logger(log_type, crawler).warning(f"get_videoList:{r.text}\n")
  33. Common.logging(log_type, crawler, env, f"get_videoList:{r.text}\n")
  34. return
  35. elif "data" not in r.json():
  36. Common.logger(log_type, crawler).info(f"get_videoList:{r.json()}\n")
  37. Common.logging(log_type, crawler, env, f"get_videoList:{r.json()}\n")
  38. return
  39. elif len(r.json()["data"]) == 0:
  40. Common.logger(log_type, crawler).warning(f"get_videoList:{r.json()['data']['list']}\n")
  41. Common.logging(log_type, crawler, env, f"get_videoList:{r.json()['data']['list']}\n")
  42. return
  43. else:
  44. # 获取视频url
  45. video_url = r.json()["data"].get("video_url")
  46. return video_url
  47. except Exception as e:
  48. Common.logger(log_type, crawler).error(f" 获取视频链接异常:{e}\n")
  49. Common.logging(log_type, crawler, env, f"获取视频链接异常:{e}\n")
  50. @classmethod
  51. def repeat_video(cls, log_type, crawler, video_id, env):
  52. sql = f""" select * from crawler_video where platform in ("{crawler}","{cls.platform}") and create_time>='2023-06-26' and out_video_id="{video_id}"; """
  53. repeat_video = MysqlHelper.get_values(log_type, crawler, sql, env)
  54. return len(repeat_video)
  55. @classmethod
  56. def get_videoList(cls, log_type, crawler, our_uid, rule_dict, env):
  57. mq = MQ(topic_name="topic_crawler_etl_" + env)
  58. uuid1 = str(uuid.uuid1())
  59. for page in range(1, 6):
  60. try:
  61. Common.logger(log_type, crawler).info(f"正在抓取第{page}页")
  62. Common.logging(log_type, crawler, env, f"正在抓取第{page}页")
  63. url = "https://mini.vvuiiu.cn/article/getArticleList"
  64. payload = json.dumps({
  65. "page": page,
  66. "size": 5,
  67. "category_id": "393774", # 每日推荐id
  68. "from_type": 1,
  69. "uuid": uuid1,
  70. # "openid": "oY5tI5FgvH9Jmc8cntj81t5Ugsds",
  71. "platform": 35,
  72. "appid": "wxa903dc775e85eb5b"
  73. })
  74. r = requests.post(url=url, headers=headers, data=payload, proxies=proxies, verify=False)
  75. if "data" not in r.text or r.status_code != 200:
  76. Common.logger(log_type, crawler).warning(f"get_videoList:{r.text}\n")
  77. Common.logging(log_type, crawler, env, f"get_videoList:{r.text}\n")
  78. return
  79. elif "data" not in r.json():
  80. Common.logger(log_type, crawler).info(f"get_videoList:{r.json()}\n")
  81. Common.logging(log_type, crawler, env, f"get_videoList:{r.json()}\n")
  82. return
  83. elif "list" not in r.json()["data"]:
  84. Common.logger(log_type, crawler).warning(f"get_videoList:{r.json()['data']}\n")
  85. Common.logging(log_type, crawler, env, f"get_videoList:{r.json()['data']}\n")
  86. return
  87. elif len(r.json()["data"]["list"]) == 0:
  88. Common.logger(log_type, crawler).warning(f"get_videoList:{r.json()['data']['list']}\n")
  89. Common.logging(log_type, crawler, env, f"get_videoList:{r.json()['data']['list']}\n")
  90. return
  91. else:
  92. # 视频列表
  93. feeds = r.json()["data"]["list"]
  94. for i in range(len(feeds)):
  95. try:
  96. video_title = feeds[i].get("title", "").strip().replace("\n", "") \
  97. .replace("/", "").replace("\\", "").replace("\r", "") \
  98. .replace(":", "").replace("*", "").replace("?", "") \
  99. .replace("?", "").replace('"', "").replace("<", "") \
  100. .replace(">", "").replace("|", "").replace(" ", "") \
  101. .replace("&NBSP", "").replace(".", "。").replace(" ", "") \
  102. .replace("'", "").replace("#", "").replace("Merge", "")
  103. publish_time_stamp = int(feeds[i].get("article_id", 0))
  104. publish_time_str = time.strftime("%Y/%m/%d %H:%M:%S", time.localtime(publish_time_stamp))
  105. id = feeds[i]["id"]
  106. # 获取视频链接
  107. video_url = cls.get_video_url(id, uuid1, log_type, crawler, env)
  108. video_dict = {
  109. "video_title": video_title,
  110. "video_id": feeds[i]["id"], # 视频id
  111. "publish_time_stamp": publish_time_stamp,
  112. "publish_time_str": publish_time_str,
  113. "is_video": int(feeds[i].get("is_video", 0)), # 类型
  114. "category_id": int(feeds[i].get("category_id", 0)), # 视频来源(精彩推荐)
  115. "image_path": feeds[i].get("image_path", ""), # 视频封面
  116. "video_url": video_url, # 视频链接
  117. "click": int(feeds[i].get("click", 0)), # 点击数
  118. "video_width": int(feeds[i].get("vw", 0)),
  119. "video_height": int(feeds[i].get("vh", 0)),
  120. "user_name": feeds[i].get("source", "").strip().replace("\n", ""),
  121. "user_id": feeds[i].get("openid", ""),
  122. "play_cnt": 0,
  123. "like_cnt": 0,
  124. "comment_cnt": 0,
  125. "share_cnt": 0,
  126. # "duration": feeds[i].get("mediaDuration", 0),
  127. "session": ""
  128. }
  129. for k, v in video_dict.items():
  130. Common.logger(log_type, crawler).info(f"{k}:{v}")
  131. Common.logging(log_type, crawler, env, f"video_dict:{video_dict}")
  132. if video_dict["video_id"] == "" or video_dict["video_title"] == "" or video_dict[
  133. "video_url"] == "":
  134. Common.logger(log_type, crawler).info("无效视频\n")
  135. Common.logging(log_type, crawler, env, "无效视频\n")
  136. elif download_rule(log_type=log_type, crawler=crawler, video_dict=video_dict,
  137. rule_dict=rule_dict) is False:
  138. Common.logger(log_type, crawler).info("不满足抓取规则\n")
  139. Common.logging(log_type, crawler, env, "不满足抓取规则\n")
  140. elif any(str(word) if str(word) in video_dict["video_title"] else False
  141. for word in get_config_from_mysql(log_type=log_type,
  142. source=crawler,
  143. env=env,
  144. text="filter",
  145. action="")) is True:
  146. Common.logger(log_type, crawler).info('已中过滤词\n')
  147. Common.logging(log_type, crawler, env, '已中过滤词\n')
  148. elif cls.repeat_video(log_type, crawler, video_dict["video_id"], env) != 0:
  149. Common.logger(log_type, crawler).info('视频已下载\n')
  150. Common.logging(log_type, crawler, env, '视频已下载\n')
  151. else:
  152. video_dict["out_user_id"] = video_dict["user_id"]
  153. video_dict["platform"] = crawler
  154. video_dict["strategy"] = log_type
  155. video_dict["out_video_id"] = video_dict["video_id"]
  156. video_dict["width"] = video_dict["video_width"]
  157. video_dict["height"] = video_dict["video_height"]
  158. video_dict["crawler_rule"] = json.dumps(rule_dict)
  159. video_dict["user_id"] = our_uid
  160. video_dict["publish_time"] = video_dict["publish_time_str"]
  161. mq.send_msg(video_dict)
  162. except Exception as e:
  163. Common.logger(log_type, crawler).error(f"抓取单条视频异常:{e}\n")
  164. Common.logging(log_type, crawler, env, f"抓取单条视频异常:{e}\n")
  165. except Exception as e:
  166. Common.logger(log_type, crawler).error(f"抓取第{page}页时异常:{e}\n")
  167. Common.logging(log_type, crawler, env, f"抓取第{page}页时异常:{e}\n")
  168. if __name__ == "__main__":
  169. rule_dict1 = {"period": {"min": 365, "max": 365},
  170. "duration": {"min": 30, "max": 1800},
  171. "favorite_cnt": {"min": 0, "max": 0},
  172. "videos_cnt": {"min": 10, "max": 20},
  173. "share_cnt": {"min": 0, "max": 0}}
  174. ZfshRecommend.get_videoList("recommend", "zhufushenghuo,", "16QspO", rule_dict1, 'dev')