get_signature.py 5.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122
  1. # -*- coding: utf-8 -*-
  2. # @Author: wangkun
  3. # @Time: 2022/9/21
  4. import json
  5. import os
  6. import sys
  7. import time
  8. from selenium import webdriver
  9. from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
  10. sys.path.append(os.getcwd())
  11. from main.common import Common
  12. from main.feishu import Feishu
  13. proxies = {'http': None, 'https': None}
  14. class GetSignature:
  15. @classmethod
  16. def get_signature(cls, log_type):
  17. try:
  18. # 打印请求配置
  19. ca = DesiredCapabilities.CHROME
  20. ca["goog:loggingPrefs"] = {"performance": "ALL"}
  21. # 不打开浏览器运行
  22. chrome_options = webdriver.ChromeOptions()
  23. chrome_options.add_argument("headless")
  24. chrome_options.add_argument(f'user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.79 Safari/537.36')
  25. chrome_options.add_argument("--no-sandbox")
  26. # driver初始化
  27. Common.logger(log_type).info('初始化 webdriver')
  28. # driver = webdriver.Chrome(desired_capabilities=ca)
  29. driver = webdriver.Chrome(desired_capabilities=ca, options=chrome_options)
  30. driver.implicitly_wait(10)
  31. Common.logger(log_type).info('打开『北京旅游等着瞧』个人主页')
  32. driver.get('https://www.ixigua.com/home/2176672314697038')
  33. time.sleep(5)
  34. # 向上滑动 1000 个像素
  35. Common.logger(log_type).info('向上滑动页面')
  36. driver.execute_script('window.scrollBy(0, 2000)')
  37. Common.logger(log_type).info('休眠 5s')
  38. time.sleep(5)
  39. Common.logger(log_type).info('刷新页面')
  40. driver.refresh()
  41. Common.logger(log_type).info('休眠 1s')
  42. time.sleep(1)
  43. Common.logger(log_type).info('解析signature')
  44. logs = driver.get_log("performance")
  45. # Common.logger(log_type).info('已获取logs:{}\n', logs)
  46. Common.logger(log_type).info('退出浏览器')
  47. driver.quit()
  48. for line in logs:
  49. msg = json.loads(line['message'])
  50. if 'params' not in msg['message']:
  51. pass
  52. elif 'documentURL' not in msg['message']['params']:
  53. pass
  54. elif 'www.ixigua.com/home/2176672314697038' not in msg['message']['params']['documentURL']:
  55. pass
  56. elif 'url' not in msg['message']['params']['request']:
  57. pass
  58. elif '_signature' not in msg['message']['params']['request']['url']:
  59. pass
  60. elif 'web/user/settings' not in msg['message']['params']['request']['url']:
  61. pass
  62. else:
  63. url = msg['message']['params']['request']['url']
  64. signature = url.split('_signature=')[-1].split('&')[0]
  65. Common.logger(log_type).info('url:{}\n', url)
  66. Common.logger(log_type).info('signature:{}\n', signature)
  67. get_signature_time = time.strftime('%Y/%m/%d %H:%M:%S', time.localtime(time.time()))
  68. values = [[get_signature_time, signature]]
  69. Feishu.insert_columns(log_type, 'xigua', '6tZHhs', 'ROWS', 1, 2)
  70. time.sleep(0.5)
  71. Feishu.update_values(log_type, 'xigua', '6tZHhs', 'A2:B2', values)
  72. Common.logger(log_type).info('signature写入飞书成功:{}\n', signature)
  73. cls.del_signature(log_type)
  74. # if 'message' in line \
  75. # and 'message' in json.loads(line['message'])\
  76. # and 'params' in json.loads(line['message'])['message']\
  77. # and 'response' in json.loads(line['message'])['message']['params']\
  78. # and 'url' in json.loads(line['message'])['message']['params']['response']\
  79. # and '_signature' in json.loads(line['message'])['message']['params']['response']['url']\
  80. # and 'web/user/settings' in json.loads(line['message'])['message']['params']['response']['url']:
  81. # url = json.loads(line['message'])['message']['params']['response']['url']
  82. # signature = url.split('_signature=')[-1]
  83. # if len(signature) < 50:
  84. # pass
  85. # else:
  86. # get_signature_time = time.strftime('%Y/%m/%d %H:%M:%S', time.localtime(time.time()))
  87. # values = [[get_signature_time, signature]]
  88. # Feishu.update_values(log_type, 'xigua', '6tZHhs', 'A2:B2', values)
  89. # Common.logger(log_type).info('signature写入飞书成功:{}\n', signature)
  90. # cls.del_signature(log_type)
  91. # else:
  92. # Common.logger(log_type).info('未获取到signature:{}\n', line)
  93. except Exception as e:
  94. Common.logger(log_type).error('get_signature异常:{}', e)
  95. @classmethod
  96. def del_signature(cls, log_type):
  97. try:
  98. while True:
  99. signature_sht = Feishu.get_values_batch(log_type, 'xigua', '6tZHhs')
  100. if len(signature_sht) <= 25:
  101. break
  102. else:
  103. Feishu.dimension_range(log_type, 'xigua', '6tZHhs', 'ROWS', 26, 26)
  104. Common.logger(log_type).info('删除signature完成\n')
  105. except Exception as e:
  106. Common.logger(log_type).error('del_signature异常:{}', e)
  107. if __name__ == '__main__':
  108. GetSignature.get_signature('follow')
  109. # GetSignature.del_signature('follow')
  110. pass