common.py 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290
  1. # -*- coding: utf-8 -*-
  2. # @Author: wangkun
  3. # @Time: 2022/4/18
  4. """
  5. 公共方法,包含:生成log / 删除log / 获取session / 下载方法 / 读取文件 / 统计下载数
  6. """
  7. import json
  8. from datetime import date, timedelta
  9. import datetime
  10. import logging
  11. import os
  12. import time
  13. import requests
  14. import urllib3
  15. from loguru import logger
  16. proxies = {"http": None, "https": None}
  17. class Common:
  18. # 统一获取当前时间 <class 'datetime.datetime'> 2022-04-14 20:13:51.244472
  19. now = datetime.datetime.now()
  20. # 昨天 <class 'str'> 2022-04-13
  21. yesterday = (date.today() + timedelta(days=-1)).strftime("%Y-%m-%d")
  22. # 今天 <class 'datetime.date'> 2022-04-14
  23. today = date.today()
  24. # 明天 <class 'str'> 2022-04-15
  25. tomorrow = (date.today() + timedelta(days=1)).strftime("%Y-%m-%d")
  26. # 使用 logging 模块生成日志
  27. @staticmethod
  28. def crawler_log():
  29. """
  30. 生成 log 日志
  31. """
  32. # 日志路径
  33. log_dir = r"./logs/"
  34. log_path = os.getcwd() + os.sep + log_dir
  35. if not os.path.isdir(log_path):
  36. os.makedirs(log_path)
  37. # 日志参数
  38. log_format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
  39. date_format = "%Y-%m-%d %p %H:%M:%S"
  40. log_name = time.strftime("%Y-%m-%d", time.localtime(time.time())) + '.log'
  41. # 日志初始化
  42. logging.basicConfig(filename=log_path + log_name, level=logging.INFO, format=log_format, datefmt=date_format)
  43. crawler_logger = logging.getLogger("crawler-log")
  44. return crawler_logger
  45. # 使用 logger 模块生成日志
  46. @staticmethod
  47. def logger():
  48. """
  49. 使用 logger 模块生成日志
  50. """
  51. # 日志路径
  52. log_dir = r"./logs/"
  53. log_path = os.getcwd() + os.sep + log_dir
  54. if not os.path.isdir(log_path):
  55. os.makedirs(log_path)
  56. # 日志文件名
  57. log_name = time.strftime("%Y-%m-%d", time.localtime(time.time())) + '.log'
  58. # 日志不打印到控制台
  59. logger.remove(handler_id=None)
  60. # rotation="500 MB",实现每 500MB 存储一个文件
  61. # rotation="12:00",实现每天 12:00 创建一个文件
  62. # rotation="1 week",每周创建一个文件
  63. # retention="10 days",每隔10天之后就会清理旧的日志
  64. # 初始化日志
  65. logger.add(log_dir + log_name, level="INFO", rotation='00:00')
  66. return logger
  67. @classmethod
  68. def del_logs(cls):
  69. """
  70. 清除冗余日志文件
  71. :return: 保留最近 7 个日志
  72. """
  73. log_dir = r"./logs/"
  74. all_files = sorted(os.listdir(log_dir))
  75. all_logs = []
  76. for log in all_files:
  77. name = os.path.splitext(log)[-1]
  78. if name == ".log":
  79. all_logs.append(log)
  80. if len(all_logs) <= 7:
  81. pass
  82. else:
  83. for file in all_logs[:len(all_logs) - 7]:
  84. os.remove(log_dir + file)
  85. cls.logger().info("清除冗余日志成功")
  86. # 删除 charles 缓存文件,只保留最近的两个文件
  87. @classmethod
  88. def del_charles_files(cls):
  89. # 目标文件夹下所有文件
  90. all_file = sorted(os.listdir(r"./chlsfiles/"))
  91. for file in all_file[0:-2]:
  92. os.remove(r"./chlsfiles/" + file)
  93. cls.logger().info("删除 charles 缓存文件成功")
  94. @classmethod
  95. def download_method(cls, text, d_name, d_url):
  96. """
  97. 下载封面:text == "cover" ; 下载视频:text == "video"
  98. 需要下载的视频标题:d_title
  99. 视频封面,或视频播放地址:d_url
  100. 下载保存路径:"./videos/{d_title}/"
  101. """
  102. # 首先创建一个保存该视频相关信息的文件夹
  103. video_dir = "./videos/" + d_name + "/"
  104. if not os.path.exists(video_dir):
  105. os.mkdir(video_dir)
  106. # 下载视频
  107. if text == "video":
  108. # 需要下载的视频地址
  109. video_url = d_url
  110. # 视频名
  111. video_name = "video.mp4"
  112. # 下载视频
  113. urllib3.disable_warnings()
  114. response = requests.get(video_url, stream=True, proxies=proxies, verify=False)
  115. try:
  116. with open(video_dir + video_name, "wb") as f:
  117. for chunk in response.iter_content(chunk_size=10240):
  118. f.write(chunk)
  119. cls.logger().info("==========视频下载完成==========")
  120. except Exception as e:
  121. cls.logger().exception("视频下载失败:{}", e)
  122. # 下载封面
  123. elif text == "cover":
  124. # 需要下载的封面地址
  125. cover_url = d_url
  126. # 封面名
  127. cover_name = "image.jpg"
  128. # 下载封面
  129. urllib3.disable_warnings()
  130. response = requests.get(cover_url, proxies=proxies, verify=False)
  131. try:
  132. with open(video_dir + cover_name, "wb") as f:
  133. f.write(response.content)
  134. cls.logger().info("==========封面下载完成==========")
  135. except Exception as e:
  136. cls.logger().exception("封面下载失败:{}", e)
  137. @staticmethod
  138. def read_txt(t_name):
  139. """
  140. 读取 txt 文件
  141. :param t_name: 文件名
  142. :return: 文件内容
  143. """
  144. with open(r"./txt/" + t_name, "r", encoding="utf8") as f:
  145. return f.readlines()
  146. @classmethod
  147. def get_session(cls):
  148. # charles 抓包文件保存目录
  149. charles_file_dir = r"./chlsfiles/"
  150. if int(len(os.listdir(charles_file_dir))) == 1:
  151. Common.logger().info("未找到chlsfile文件,等待60s")
  152. time.sleep(60)
  153. else:
  154. try:
  155. # 目标文件夹下所有文件
  156. all_file = sorted(os.listdir(charles_file_dir))
  157. # 获取到目标文件
  158. old_file = all_file[-1]
  159. # 分离文件名与扩展名
  160. new_file = os.path.splitext(old_file)
  161. # 重命名文件后缀
  162. os.rename(os.path.join(charles_file_dir, old_file),
  163. os.path.join(charles_file_dir, new_file[0] + ".txt"))
  164. with open(charles_file_dir + new_file[0] + ".txt", encoding='utf-8-sig', errors='ignore') as f:
  165. contents = json.load(f, strict=False)
  166. if "search.weixin.qq.com" in [text['host'] for text in contents]:
  167. for text in contents:
  168. if text["host"] == "search.weixin.qq.com" \
  169. and text["path"] == "/cgi-bin/recwxa/recwxagetunreadmessagecnt":
  170. sessions = text["query"].split("session=")[-1].split("&wxaVersion=")[0]
  171. if "&vid" in sessions:
  172. session = sessions.split("&vid")[0]
  173. return session
  174. elif "&offset" in sessions:
  175. session = sessions.split("&offset")[0]
  176. return session
  177. elif "&wxaVersion" in sessions:
  178. session = sessions.split("&wxaVersion")[0]
  179. return session
  180. elif "&limit" in sessions:
  181. session = sessions.split("&limit")[0]
  182. return session
  183. elif "&scene" in sessions:
  184. session = sessions.split("&scene")[0]
  185. return session
  186. elif "&count" in sessions:
  187. session = sessions.split("&count")[0]
  188. return session
  189. elif "&channelid" in sessions:
  190. session = sessions.split("&channelid")[0]
  191. return session
  192. elif "&subscene" in sessions:
  193. session = sessions.split("&subscene")[0]
  194. return session
  195. elif "&clientVersion" in sessions:
  196. session = sessions.split("&clientVersion")[0]
  197. return session
  198. elif "&sharesearchid" in sessions:
  199. session = sessions.split("&sharesearchid")[0]
  200. return session
  201. elif "&nettype" in sessions:
  202. session = sessions.split("&nettype")[0]
  203. return session
  204. elif "&switchprofile" in sessions:
  205. session = sessions.split("&switchprofile")[0]
  206. return session
  207. elif "&switchnewuser" in sessions:
  208. session = sessions.split("&switchnewuser")[0]
  209. return session
  210. else:
  211. return sessions
  212. else:
  213. cls.logger().info("未找到 session,10s后重新获取")
  214. time.sleep(10)
  215. cls.get_session()
  216. except Exception as e:
  217. cls.logger().exception("获取 session 异常,30s后重试:{}", e)
  218. time.sleep(30)
  219. cls.get_session()
  220. @classmethod
  221. def kanyikan_download_count(cls):
  222. videoid_path = r"./txt/kanyikan_videoid.txt"
  223. count = 0
  224. for count, line in enumerate(open(videoid_path, "rb").readlines()):
  225. count += 1
  226. cls.logger().info('累计下载视频数: {}\n', count)
  227. @classmethod
  228. def kanyikan_today_download_count(cls):
  229. """
  230. 统计快手渠道当日下载视频数
  231. :return:
  232. """
  233. # 创建空文件
  234. with open(r"./txt/" + str(cls.today) + "_kanyikan_videoid.txt", "a") as f:
  235. f.write("")
  236. videoid_path = r"./txt/" + str(cls.today) + "_kanyikan_videoid.txt"
  237. count = 0
  238. for count, line in enumerate(open(videoid_path, "rb").readlines()):
  239. count += 1
  240. return count
  241. @classmethod
  242. def del_yesterday_kanyikan_videoid_txt(cls):
  243. """
  244. 删除快手渠道昨日下载视频数的 txt 文件
  245. :return:
  246. """
  247. yesterday_kanyikan_videoid_txt_dir = r"./txt/"
  248. all_files = sorted(os.listdir(yesterday_kanyikan_videoid_txt_dir))
  249. for file in all_files:
  250. name = os.path.splitext(file)[0]
  251. if name == cls.yesterday + "_kanyikan_videoid":
  252. os.remove(yesterday_kanyikan_videoid_txt_dir + file)
  253. Common.logger().info("删除快手昨天下载统计文件成功")
  254. if __name__ == "__main__":
  255. common = Common()
  256. common.del_yesterday_kanyikan_videoid_txt()
  257. print(common.kanyikan_today_download_count())