|
@@ -76,42 +76,45 @@ class SearchKey:
|
|
|
# 获取微信指数小程序 search_key
|
|
|
@classmethod
|
|
|
def get_wechat_key(cls, log_type, crawler):
|
|
|
- while True:
|
|
|
- chlsfile_path = f"./{crawler}/{crawler}_chlsfiles/"
|
|
|
- if len(os.listdir(chlsfile_path)) == 0:
|
|
|
- Common.logger(log_type, crawler).info("chlsfile文件夹空,等待 3 秒")
|
|
|
- time.sleep(3)
|
|
|
- cls.start_wechat(log_type, crawler)
|
|
|
- continue
|
|
|
- Common.logger(log_type, crawler).info(f"chlsfile_list:{sorted(os.listdir(chlsfile_path))}")
|
|
|
- # 获取最新的 chlsfile
|
|
|
- chlsfile = sorted(os.listdir(chlsfile_path))[-1]
|
|
|
- # 分离文件名与扩展名
|
|
|
- new_file = os.path.splitext(chlsfile)
|
|
|
-
|
|
|
- # 重命名文件后缀
|
|
|
- os.rename(os.path.join(chlsfile_path, chlsfile),
|
|
|
- os.path.join(chlsfile_path, new_file[0] + ".txt"))
|
|
|
-
|
|
|
- with open(f"{chlsfile_path}{new_file[0]}.txt", encoding='utf-8-sig', errors='ignore') as f:
|
|
|
- contents = json.load(f, strict=False)
|
|
|
-
|
|
|
- if "search.weixin.qq.com" not in [text['host'] for text in contents]:
|
|
|
- return "未找到wechat_key"
|
|
|
- else:
|
|
|
- for content in contents:
|
|
|
- if content["host"] == "search.weixin.qq.com" and content["path"] == "/cgi-bin/wxaweb/wxindexgetusergroup":
|
|
|
- # if content["host"] == "search.weixin.qq.com" and content["path"] == "/cgi-bin/wxaweb/wxawebreport":
|
|
|
- # print(f"content:{content}")
|
|
|
- text = content['request']['body']['text']
|
|
|
- search_key = json.loads(text)['search_key']
|
|
|
- openid = json.loads(text)['openid']
|
|
|
- wechat_key_dict = {
|
|
|
- "search_key": search_key,
|
|
|
- "openid": openid,
|
|
|
- }
|
|
|
- return wechat_key_dict
|
|
|
- return "未找到wechat_key"
|
|
|
+ try:
|
|
|
+ while True:
|
|
|
+ chlsfile_path = f"./{crawler}/{crawler}_chlsfiles/"
|
|
|
+ if len(os.listdir(chlsfile_path)) == 0:
|
|
|
+ Common.logger(log_type, crawler).info("chlsfile文件夹空,等待 3 秒")
|
|
|
+ time.sleep(3)
|
|
|
+ cls.start_wechat(log_type, crawler)
|
|
|
+ continue
|
|
|
+ Common.logger(log_type, crawler).info(f"chlsfile_list:{sorted(os.listdir(chlsfile_path))}")
|
|
|
+ # 获取最新的 chlsfile
|
|
|
+ chlsfile = sorted(os.listdir(chlsfile_path))[-1]
|
|
|
+ # 分离文件名与扩展名
|
|
|
+ new_file = os.path.splitext(chlsfile)
|
|
|
+
|
|
|
+ # 重命名文件后缀
|
|
|
+ os.rename(os.path.join(chlsfile_path, chlsfile),
|
|
|
+ os.path.join(chlsfile_path, new_file[0] + ".txt"))
|
|
|
+
|
|
|
+ with open(f"{chlsfile_path}{new_file[0]}.txt", encoding='utf-8-sig', errors='ignore') as f:
|
|
|
+ contents = json.load(f, strict=False)
|
|
|
+
|
|
|
+ if "search.weixin.qq.com" not in [text['host'] for text in contents]:
|
|
|
+ return "未找到wechat_key"
|
|
|
+ else:
|
|
|
+ for content in contents:
|
|
|
+ if content["host"] == "search.weixin.qq.com" and content["path"] == "/cgi-bin/wxaweb/wxindexgetusergroup":
|
|
|
+ # if content["host"] == "search.weixin.qq.com" and content["path"] == "/cgi-bin/wxaweb/wxawebreport":
|
|
|
+ # print(f"content:{content}")
|
|
|
+ text = content['request']['body']['text']
|
|
|
+ search_key = json.loads(text)['search_key']
|
|
|
+ openid = json.loads(text)['openid']
|
|
|
+ wechat_key_dict = {
|
|
|
+ "search_key": search_key,
|
|
|
+ "openid": openid,
|
|
|
+ }
|
|
|
+ return wechat_key_dict
|
|
|
+ return "未找到wechat_key"
|
|
|
+ except Exception as e:
|
|
|
+ Common.logger(log_type, crawler).error(f"get_wechat_key:{e}\n")
|
|
|
|
|
|
# 删除 chlsfile 文件
|
|
|
@classmethod
|
|
@@ -146,29 +149,32 @@ class SearchKey:
|
|
|
# 微信指数小程序 search_key 写入飞书
|
|
|
@classmethod
|
|
|
def write_wechat_key(cls, log_type, crawler):
|
|
|
- Common.logger(log_type, crawler).info(f"清空 chlsfiles 文件夹")
|
|
|
- cls.remove_chlsfile(log_type, crawler)
|
|
|
- # Common.logger(log_type, crawler).info('启动微信指数小程序')
|
|
|
- # cls.start_wechat(log_type, crawler)
|
|
|
- Common.logger(log_type, crawler).info('获取 wechat_key')
|
|
|
- while True:
|
|
|
- cls.start_wechat(log_type, crawler)
|
|
|
- wechat_key_dict = cls.get_wechat_key(log_type, crawler)
|
|
|
- if wechat_key_dict is None or wechat_key_dict == "未找到wechat_key":
|
|
|
- Common.logger(log_type, crawler).info("未找到wechat_key,重新获取")
|
|
|
+ try:
|
|
|
+ Common.logger(log_type, crawler).info(f"清空 chlsfiles 文件夹")
|
|
|
+ cls.remove_chlsfile(log_type, crawler)
|
|
|
+ # Common.logger(log_type, crawler).info('启动微信指数小程序')
|
|
|
+ # cls.start_wechat(log_type, crawler)
|
|
|
+ Common.logger(log_type, crawler).info('获取 wechat_key')
|
|
|
+ while True:
|
|
|
+ cls.start_wechat(log_type, crawler)
|
|
|
+ wechat_key_dict = cls.get_wechat_key(log_type, crawler)
|
|
|
+ if wechat_key_dict is None or wechat_key_dict == "未找到wechat_key":
|
|
|
+ Common.logger(log_type, crawler).info("未找到wechat_key,重新获取")
|
|
|
+ time.sleep(1)
|
|
|
+ continue
|
|
|
+ for k, v in wechat_key_dict.items():
|
|
|
+ Common.logger(log_type, crawler).info(f"{k}:{v}")
|
|
|
+ Feishu.insert_columns(log_type, crawler, 'sVL74k', 'ROWS', 1, 2)
|
|
|
+ time_str = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(int(time.time())))
|
|
|
+ values = [[time_str, wechat_key_dict['search_key'], wechat_key_dict['openid']]]
|
|
|
time.sleep(1)
|
|
|
- continue
|
|
|
- for k, v in wechat_key_dict.items():
|
|
|
- Common.logger(log_type, crawler).info(f"{k}:{v}")
|
|
|
- Feishu.insert_columns(log_type, crawler, 'sVL74k', 'ROWS', 1, 2)
|
|
|
- time_str = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(int(time.time())))
|
|
|
- values = [[time_str, wechat_key_dict['search_key'], wechat_key_dict['openid']]]
|
|
|
- time.sleep(1)
|
|
|
- Feishu.update_values(log_type, crawler, 'sVL74k', 'A2:Z2', values)
|
|
|
- cls.del_wechat_key(log_type, crawler)
|
|
|
- Common.logger(log_type, crawler).info("wechat_key写入飞书成功")
|
|
|
- Common.del_logs(log_type, crawler)
|
|
|
- return
|
|
|
+ Feishu.update_values(log_type, crawler, 'sVL74k', 'A2:Z2', values)
|
|
|
+ cls.del_wechat_key(log_type, crawler)
|
|
|
+ Common.logger(log_type, crawler).info("wechat_key写入飞书成功")
|
|
|
+ Common.del_logs(log_type, crawler)
|
|
|
+ return
|
|
|
+ except Exception as e:
|
|
|
+ Common.logger(log_type, crawler).error(f"write_wechat_key:{e}\n")
|
|
|
|
|
|
@classmethod
|
|
|
def main(cls, log_type, crawler):
|