common.py 4.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165
  1. """
  2. @author: luojunhui
  3. """
  4. import hashlib
  5. from datetime import datetime, timezone
  6. from requests import RequestException
  7. from urllib.parse import urlparse, parse_qs
  8. from tenacity import (
  9. stop_after_attempt,
  10. wait_exponential,
  11. retry_if_exception_type,
  12. )
  13. def str_to_md5(strings):
  14. """
  15. 字符串转化为 md5 值
  16. :param strings:
  17. :return:
  18. """
  19. # 将字符串转换为字节
  20. original_bytes = strings.encode("utf-8")
  21. # 创建一个md5 hash对象
  22. md5_hash = hashlib.md5()
  23. # 更新hash对象,传入原始字节
  24. md5_hash.update(original_bytes)
  25. # 获取16进制形式的MD5哈希值
  26. md5_value = md5_hash.hexdigest()
  27. return md5_value
  28. def proxy():
  29. """
  30. 快代理
  31. """
  32. # 隧道域名:端口号
  33. tunnel = "j685.kdltps.com:15818"
  34. # 用户名密码方式
  35. username = "t14070979713487"
  36. password = "hqwanfvy"
  37. proxies = {
  38. "http": "http://%(user)s:%(pwd)s@%(proxy)s/"
  39. % {"user": username, "pwd": password, "proxy": tunnel},
  40. "https": "http://%(user)s:%(pwd)s@%(proxy)s/"
  41. % {"user": username, "pwd": password, "proxy": tunnel},
  42. }
  43. return proxies
  44. def request_retry(retry_times, min_retry_delay, max_retry_delay):
  45. """
  46. :param retry_times:
  47. :param min_retry_delay:
  48. :param max_retry_delay:
  49. """
  50. common_retry = dict(
  51. stop=stop_after_attempt(retry_times),
  52. wait=wait_exponential(min=min_retry_delay, max=max_retry_delay),
  53. retry=retry_if_exception_type((RequestException, TimeoutError)),
  54. reraise=True, # 重试耗尽后重新抛出异常
  55. )
  56. return common_retry
  57. def yield_batch(data, batch_size):
  58. """
  59. 生成批次数据
  60. :param data:
  61. :param batch_size:
  62. :return:
  63. """
  64. for i in range(0, len(data), batch_size):
  65. yield data[i : i + batch_size]
  66. def extract_root_source_id(path: str) -> dict:
  67. """
  68. 提取path参数
  69. :param path:
  70. :return:
  71. """
  72. params = parse_qs(urlparse(path).query)
  73. jump_page = params.get("jumpPage", [None])[0]
  74. if jump_page:
  75. params2 = parse_qs(jump_page)
  76. res = {
  77. "video_id": params2["pages/user-videos?id"][0],
  78. "root_source_id": params2["rootSourceId"][0],
  79. }
  80. return res
  81. else:
  82. return {}
  83. def show_desc_to_sta(show_desc):
  84. def decode_show_v(show_v):
  85. """
  86. :param show_v:
  87. :return:
  88. """
  89. foo = show_v.replace('千', 'e3').replace('万', 'e4').replace('亿', 'e8')
  90. foo = eval(foo)
  91. return int(foo)
  92. def decode_show_k(show_k):
  93. """
  94. :param show_k:
  95. :return:
  96. """
  97. this_dict = {
  98. '阅读': 'show_view_count', # 文章
  99. '看过': 'show_view_count', # 图文
  100. '观看': 'show_view_count', # 视频
  101. '赞': 'show_like_count',
  102. '付费': 'show_pay_count',
  103. '赞赏': 'show_zs_count',
  104. }
  105. if show_k not in this_dict:
  106. print(f'error from decode_show_k, show_k not found: {show_k}')
  107. return this_dict.get(show_k, 'show_unknown')
  108. show_desc = show_desc.replace('+', '')
  109. sta = {}
  110. for show_kv in show_desc.split('\u2004\u2005'):
  111. if not show_kv:
  112. continue
  113. show_k, show_v = show_kv.split('\u2006')
  114. k = decode_show_k(show_k)
  115. v = decode_show_v(show_v)
  116. sta[k] = v
  117. res = {
  118. 'show_view_count': sta.get('show_view_count', 0),
  119. 'show_like_count': sta.get('show_like_count', 0),
  120. 'show_pay_count': sta.get('show_pay_count', 0),
  121. 'show_zs_count': sta.get('show_zs_count', 0),
  122. }
  123. return res
  124. def generate_gzh_id(url):
  125. biz = url.split("biz=")[1].split("&")[0]
  126. idx = url.split("&idx=")[1].split("&")[0]
  127. sn = url.split("&sn=")[1].split("&")[0]
  128. url_bit = "{}-{}-{}".format(biz, idx, sn).encode()
  129. md5_hash = hashlib.md5()
  130. md5_hash.update(url_bit)
  131. md5_value = md5_hash.hexdigest()
  132. return md5_value
  133. def timestamp_to_str(timestamp, string_format='%Y-%m-%d %H:%M:%S') -> str:
  134. """
  135. :param string_format:
  136. :param timestamp:
  137. """
  138. dt_object = datetime.utcfromtimestamp(timestamp).replace(tzinfo=timezone.utc).astimezone()
  139. date_string = dt_object.strftime(string_format)
  140. return date_string