common.py 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305
  1. # -*- coding: utf-8 -*-
  2. # @Author: wangkun
  3. # @Time: 2023/1/31
  4. """
  5. 公共方法,包含:生成log / 删除log / 下载方法 / 删除 weixinzhishu_chlsfiles / 过滤词库 / 保存视频信息至本地 txt / 翻译 / ffmpeg
  6. """
  7. from datetime import date, timedelta
  8. from loguru import logger
  9. import datetime
  10. import os
  11. import time
  12. import requests
  13. import ffmpeg
  14. import urllib3
  15. import subprocess
  16. proxies = {"http": None, "https": None}
  17. class Common:
  18. # 统一获取当前时间 <class 'datetime.datetime'> 2022-04-14 20:13:51.244472
  19. now = datetime.datetime.now()
  20. # 昨天 <class 'str'> 2022-04-13
  21. yesterday = (date.today() + timedelta(days=-1)).strftime("%Y/%m/%d")
  22. # 今天 <class 'datetime.date'> 2022-04-14
  23. today = date.today()
  24. # 明天 <class 'str'> 2022-04-15
  25. tomorrow = (date.today() + timedelta(days=1)).strftime("%Y/%m/%d")
  26. # 使用 logger 模块生成日志
  27. @staticmethod
  28. def logger(log_type, crawler):
  29. """
  30. 使用 logger 模块生成日志
  31. """
  32. # 日志路径
  33. log_dir = f"./{crawler}/logs/"
  34. log_path = os.getcwd() + os.sep + log_dir
  35. if not os.path.isdir(log_path):
  36. os.makedirs(log_path)
  37. # 日志文件名
  38. log_name = time.strftime("%Y-%m-%d", time.localtime(time.time())) + f'-{crawler}-{log_type}.log'
  39. # 日志不打印到控制台
  40. logger.remove(handler_id=None)
  41. # rotation="500 MB",实现每 500MB 存储一个文件
  42. # rotation="12:00",实现每天 12:00 创建一个文件
  43. # rotation="1 week",每周创建一个文件
  44. # retention="10 days",每隔10天之后就会清理旧的日志
  45. # 初始化日志
  46. logger.add(log_dir + log_name, level="INFO", rotation='00:00')
  47. return logger
  48. # 清除日志,保留最近 10 个文件
  49. @classmethod
  50. def del_logs(cls, log_type, crawler):
  51. """
  52. 清除冗余日志文件
  53. :return: 保留最近 10 个日志
  54. """
  55. log_dir = f"./{crawler}/logs/"
  56. all_files = sorted(os.listdir(log_dir))
  57. all_logs = []
  58. for log in all_files:
  59. name = os.path.splitext(log)[-1]
  60. if name == ".log":
  61. all_logs.append(log)
  62. if len(all_logs) <= 10:
  63. pass
  64. else:
  65. for file in all_logs[:len(all_logs) - 10]:
  66. os.remove(log_dir + file)
  67. cls.logger(log_type, crawler).info("清除日志成功\n")
  68. # 删除 charles 缓存文件,只保留最近的两个文件
  69. @classmethod
  70. def del_charles_files(cls, log_type, crawler):
  71. # 目标文件夹下所有文件
  72. all_file = sorted(os.listdir(f"./{crawler}/{crawler}_chlsfiles/"))
  73. for file in all_file[0:-3]:
  74. os.remove(f"./{crawler}/{crawler}_chlsfiles/{file}")
  75. cls.logger(log_type, crawler).info("删除 charles 缓存文件成功\n")
  76. # 保存视频信息至 "./videos/{video_dict['video_title}/info.txt"
  77. @classmethod
  78. def save_video_info(cls, log_type, crawler, video_dict):
  79. with open(f"./{crawler}/videos/{video_dict['video_title']}/info.txt",
  80. "a", encoding="UTF-8") as f_a:
  81. f_a.write(str(video_dict['video_id']) + "\n" +
  82. str(video_dict['video_title']) + "\n" +
  83. str(video_dict['duration']) + "\n" +
  84. str(video_dict['play_cnt']) + "\n" +
  85. str(video_dict['comment_cnt']) + "\n" +
  86. str(video_dict['like_cnt']) + "\n" +
  87. str(video_dict['share_cnt']) + "\n" +
  88. f"{video_dict['video_width']}*{video_dict['video_height']}" + "\n" +
  89. str(video_dict['publish_time_stamp']) + "\n" +
  90. str(video_dict['user_name']) + "\n" +
  91. str(video_dict['avatar_url']) + "\n" +
  92. str(video_dict['video_url']) + "\n" +
  93. str(video_dict['cover_url']) + "\n" +
  94. str(video_dict['session']))
  95. Common.logger(log_type, crawler).info("==========视频信息已保存至info.txt==========")
  96. # 封装下载视频或封面的方法
  97. @classmethod
  98. def download_method(cls, log_type, crawler, text, title, url):
  99. """
  100. 下载封面:text == "cover" ; 下载视频:text == "video"
  101. 需要下载的视频标题:d_title
  102. 视频封面,或视频播放地址:d_url
  103. 下载保存路径:"./files/{d_title}/"
  104. """
  105. videos_dir = f"./{crawler}/videos/"
  106. if not os.path.exists(videos_dir):
  107. os.mkdir(videos_dir)
  108. # 首先创建一个保存该视频相关信息的文件夹
  109. video_path = f"./{crawler}/videos/{title}/"
  110. if not os.path.exists(video_path):
  111. os.mkdir(video_path)
  112. # 下载视频
  113. if text == "video":
  114. # 需要下载的视频地址
  115. video_url = str(url).replace('http://', 'https://')
  116. # 视频名
  117. video_name = "video.mp4"
  118. # 下载视频
  119. urllib3.disable_warnings()
  120. # response = requests.get(video_url, stream=True, proxies=cls.tunnel_proxies(), verify=False)
  121. response = requests.get(video_url, stream=True, proxies=proxies, verify=False)
  122. try:
  123. with open(video_path + video_name, "wb") as f:
  124. for chunk in response.iter_content(chunk_size=10240):
  125. f.write(chunk)
  126. cls.logger(log_type, crawler).info("==========视频下载完成==========")
  127. except Exception as e:
  128. cls.logger(log_type, crawler).error(f"视频下载失败:{e}\n")
  129. # 下载音频
  130. elif text == "audio":
  131. # 需要下载的视频地址
  132. audio_url = str(url).replace('http://', 'https://')
  133. # 音频名
  134. audio_name = "audio.mp4"
  135. # 下载视频
  136. urllib3.disable_warnings()
  137. # response = requests.get(audio_url, stream=True, proxies=cls.tunnel_proxies(), verify=False)
  138. response = requests.get(audio_url, stream=True, proxies=proxies, verify=False)
  139. try:
  140. with open(video_path + audio_name, "wb") as f:
  141. for chunk in response.iter_content(chunk_size=10240):
  142. f.write(chunk)
  143. cls.logger(log_type, crawler).info("==========音频下载完成==========")
  144. except Exception as e:
  145. cls.logger(log_type, crawler).error(f"音频下载失败:{e}\n")
  146. # 下载封面
  147. elif text == "cover":
  148. # 需要下载的封面地址
  149. cover_url = str(url)
  150. # 封面名
  151. cover_name = "image.jpg"
  152. # 下载封面
  153. urllib3.disable_warnings()
  154. # response = requests.get(cover_url, proxies=cls.tunnel_proxies(), verify=False)
  155. response = requests.get(cover_url, proxies=proxies, verify=False)
  156. try:
  157. with open(video_path + cover_name, "wb") as f:
  158. f.write(response.content)
  159. cls.logger(log_type, crawler).info("==========封面下载完成==========")
  160. except Exception as e:
  161. cls.logger(log_type, crawler).error(f"封面下载失败:{e}\n")
  162. # youtube 视频下载
  163. elif text == "youtube_video":
  164. # 需要下载的视频地址
  165. video_url = url
  166. # 视频名
  167. video_name = "video.mp4"
  168. try:
  169. download_cmd = f'yt-dlp -f "bv[height=720][ext=mp4]+ba[ext=m4a]" --merge-output-format mp4 {video_url}-U -o {video_path}{video_name}'
  170. Common.logger(log_type, crawler).info(f"download_cmd:{download_cmd}")
  171. os.system(download_cmd)
  172. # move_cmd = f"mv {video_name} {video_path}"
  173. # os.system(move_cmd)
  174. cls.logger(log_type, crawler).info("==========视频下载完成==========")
  175. except Exception as e:
  176. Common.logger(log_type, crawler).error(f"视频下载失败:{e}\n")
  177. # 西瓜视频 / 音频下载
  178. elif text == "xigua_video":
  179. # 需要下载的视频地址
  180. video_url = str(url).replace('http://', 'https://')
  181. # 视频名
  182. video_name = "video1.mp4"
  183. # 下载视频
  184. urllib3.disable_warnings()
  185. # response = requests.get(video_url, stream=True, proxies=cls.tunnel_proxies(), verify=False)
  186. response = requests.get(video_url, stream=True, proxies=proxies, verify=False)
  187. try:
  188. with open(video_path + video_name, "wb") as f:
  189. for chunk in response.iter_content(chunk_size=10240):
  190. f.write(chunk)
  191. cls.logger(log_type, crawler).info("==========视频下载完成==========")
  192. except Exception as e:
  193. cls.logger(log_type, crawler).error(f"视频下载失败:{e}\n")
  194. elif text == "xigua_audio":
  195. # 需要下载的视频地址
  196. audio_url = str(url).replace('http://', 'https://')
  197. # 音频名
  198. audio_name = "audio1.mp4"
  199. # 下载视频
  200. urllib3.disable_warnings()
  201. # response = requests.get(audio_url, stream=True, proxies=cls.tunnel_proxies(), verify=False)
  202. response = requests.get(audio_url, stream=True, proxies=proxies, verify=False)
  203. try:
  204. with open(video_path + audio_name, "wb") as f:
  205. for chunk in response.iter_content(chunk_size=10240):
  206. f.write(chunk)
  207. cls.logger(log_type, crawler).info("==========音频下载完成==========")
  208. except Exception as e:
  209. cls.logger(log_type, crawler).error(f"音频下载失败:{e}\n")
  210. @classmethod
  211. def ffmpeg(cls, log_type, crawler, video_path):
  212. probe = ffmpeg.probe(video_path)
  213. video_stream = next((stream for stream in probe['streams'] if stream['codec_type'] == 'video'), None)
  214. if video_stream is None:
  215. Common.logger(log_type, crawler).info('No video Stream found!')
  216. return
  217. format1 = probe['format']
  218. size = int(int(format1['size']) / 1024 / 1024)
  219. width = int(video_stream['width'])
  220. height = int(video_stream['height'])
  221. duration = int(float(video_stream['duration']))
  222. ffmpeg_dict = {
  223. 'width': width,
  224. 'height': height,
  225. 'duration': duration,
  226. 'size': size
  227. }
  228. return ffmpeg_dict
  229. # 合并音视频
  230. @classmethod
  231. def video_compose(cls, log_type, crawler, video_dir):
  232. try:
  233. video_path = f'{video_dir}/video1.mp4'
  234. audio_path = f'{video_dir}/audio1.mp4'
  235. out_path = f'{video_dir}/video.mp4'
  236. cmd = f'ffmpeg -i {video_path} -i {audio_path} -c:v copy -c:a aac -strict experimental -map 0:v:0 -map 1:a:0 {out_path}'
  237. # print(cmd)
  238. subprocess.call(cmd, shell=True)
  239. for file in os.listdir(video_dir):
  240. if file.split('.mp4')[0] == 'video1' or file.split('.mp4')[0] == 'audio1':
  241. os.remove(f'{video_dir}/{file}')
  242. Common.logger(log_type, crawler).info('合成成功\n')
  243. except Exception as e:
  244. Common.logger(log_type, crawler).error(f'video_compose异常:{e}\n')
  245. # 快代理
  246. @classmethod
  247. def tunnel_proxies(cls):
  248. # 隧道域名:端口号
  249. tunnel = "q796.kdltps.com:15818"
  250. # 用户名密码方式
  251. username = "t17772369458618"
  252. password = "5zqcjkmy"
  253. proxies = {
  254. "http": "http://%(user)s:%(pwd)s@%(proxy)s/" % {"user": username, "pwd": password, "proxy": tunnel},
  255. "https": "http://%(user)s:%(pwd)s@%(proxy)s/" % {"user": username, "pwd": password, "proxy": tunnel}
  256. }
  257. # 白名单方式(需提前设置白名单)
  258. # proxies = {
  259. # "http": "http://%(proxy)s/" % {"proxy": tunnel},
  260. # "https": "http://%(proxy)s/" % {"proxy": tunnel}
  261. # }
  262. # 要访问的目标网页
  263. # target_url = "https://www.kuaishou.com/profile/3xk9tkk6kkwkf7g"
  264. # target_url = "https://dev.kdlapi.com/testproxy"
  265. # # 使用隧道域名发送请求
  266. # response = requests.get(target_url, proxies=proxies)
  267. # print(response.text)
  268. return proxies
  269. if __name__ == "__main__":
  270. Common.tunnel_proxies()
  271. pass