|
@@ -10,6 +10,7 @@ import requests
|
|
sys.path.append(os.getcwd())
|
|
sys.path.append(os.getcwd())
|
|
from common.common import Common
|
|
from common.common import Common
|
|
from common.feishu import Feishu
|
|
from common.feishu import Feishu
|
|
|
|
+proxies = {"http": None, "https": None}
|
|
|
|
|
|
|
|
|
|
class Test:
|
|
class Test:
|
|
@@ -42,19 +43,19 @@ class Test:
|
|
def get_words(cls, log_type, crawler):
|
|
def get_words(cls, log_type, crawler):
|
|
try:
|
|
try:
|
|
while True:
|
|
while True:
|
|
- sheet = Feishu.get_values_batch(log_type, crawler, '6dsgUk')
|
|
|
|
|
|
+ sheet = Feishu.get_values_batch(log_type, crawler, 'MvFi8s')
|
|
if sheet is None:
|
|
if sheet is None:
|
|
Common.logger(log_type, crawler).warning(f"获取热词sheet:{sheet} ,10秒钟后重试")
|
|
Common.logger(log_type, crawler).warning(f"获取热词sheet:{sheet} ,10秒钟后重试")
|
|
time.sleep(10)
|
|
time.sleep(10)
|
|
else:
|
|
else:
|
|
break
|
|
break
|
|
word_list = []
|
|
word_list = []
|
|
- for x in sheet:
|
|
|
|
- for y in x:
|
|
|
|
- if y is None:
|
|
|
|
- pass
|
|
|
|
- else:
|
|
|
|
- word_list.append(y)
|
|
|
|
|
|
+ for i in range(len(sheet)):
|
|
|
|
+ word_dict = {
|
|
|
|
+ "title": sheet[i][0],
|
|
|
|
+ "word": sheet[i][1]
|
|
|
|
+ }
|
|
|
|
+ word_list.append(word_dict)
|
|
return word_list
|
|
return word_list
|
|
except Exception as e:
|
|
except Exception as e:
|
|
Common.logger(log_type, crawler).error(f"get_words:{e}\n")
|
|
Common.logger(log_type, crawler).error(f"get_words:{e}\n")
|
|
@@ -67,7 +68,7 @@ class Test:
|
|
|
|
|
|
word_list = cls.get_words(log_type, crawler)
|
|
word_list = cls.get_words(log_type, crawler)
|
|
for i in range(len(word_list)):
|
|
for i in range(len(word_list)):
|
|
- Common.logger(log_type, crawler).info(f"热词: {word_list[i]}")
|
|
|
|
|
|
+ Common.logger(log_type, crawler).info(f"热词: {word_list[i]['word']}")
|
|
while True:
|
|
while True:
|
|
wechat_key = cls.get_wechat_key(log_type, crawler)
|
|
wechat_key = cls.get_wechat_key(log_type, crawler)
|
|
if wechat_key is None:
|
|
if wechat_key is None:
|
|
@@ -85,7 +86,7 @@ class Test:
|
|
"cgi_name": "GetDefaultIndex",
|
|
"cgi_name": "GetDefaultIndex",
|
|
"start_ymd": start_ymd,
|
|
"start_ymd": start_ymd,
|
|
"end_ymd": end_ymd,
|
|
"end_ymd": end_ymd,
|
|
- "query": word_list[i]
|
|
|
|
|
|
+ "query": word_list[i]['word']
|
|
})
|
|
})
|
|
headers = {
|
|
headers = {
|
|
'Host': 'search.weixin.qq.com',
|
|
'Host': 'search.weixin.qq.com',
|
|
@@ -93,7 +94,7 @@ class Test:
|
|
'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 13_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 MicroMessenger/8.0.32(0x1800202a) NetType/WIFI Language/zh_CN',
|
|
'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 13_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 MicroMessenger/8.0.32(0x1800202a) NetType/WIFI Language/zh_CN',
|
|
'Referer': 'https://servicewechat.com/wxc026e7662ec26a3a/42/page-frame.html'
|
|
'Referer': 'https://servicewechat.com/wxc026e7662ec26a3a/42/page-frame.html'
|
|
}
|
|
}
|
|
- response = requests.request("POST", url, headers=headers, data=payload)
|
|
|
|
|
|
+ response = requests.request("POST", url, headers=headers, data=payload, proxies=proxies)
|
|
if response.json()['code'] == -10000:
|
|
if response.json()['code'] == -10000:
|
|
Common.logger(log_type, crawler).info(f"{time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(int(time.time())))} response:{response.json()['msg']} 休眠 10 秒,重新获取")
|
|
Common.logger(log_type, crawler).info(f"{time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(int(time.time())))} response:{response.json()['msg']} 休眠 10 秒,重新获取")
|
|
time.sleep(10)
|
|
time.sleep(10)
|
|
@@ -102,20 +103,21 @@ class Test:
|
|
wechat_score_list = []
|
|
wechat_score_list = []
|
|
word_wechat_score_dict = {
|
|
word_wechat_score_dict = {
|
|
"id": i+1,
|
|
"id": i+1,
|
|
- "word": word_list[i],
|
|
|
|
|
|
+ "word": word_list[i]['word'],
|
|
"wechatScores": wechat_score_list,
|
|
"wechatScores": wechat_score_list,
|
|
}
|
|
}
|
|
if response.json()['code'] == -10002:
|
|
if response.json()['code'] == -10002:
|
|
Common.logger(log_type, crawler).info("该词暂未收录")
|
|
Common.logger(log_type, crawler).info("该词暂未收录")
|
|
# 写飞书
|
|
# 写飞书
|
|
- if word_list[i] in [x for y in Feishu.get_values_batch(log_type, crawler, "5011a2") for x in y]:
|
|
|
|
|
|
+ if word_list[i]['word'] in [x for y in Feishu.get_values_batch(log_type, crawler, "YVuVgQ") for x in y]:
|
|
Common.logger(log_type, crawler).info("该词已存在")
|
|
Common.logger(log_type, crawler).info("该词已存在")
|
|
continue
|
|
continue
|
|
- Feishu.insert_columns(log_type, crawler, "5011a2", "ROWS", 1, 2)
|
|
|
|
|
|
+ Feishu.insert_columns(log_type, crawler, "YVuVgQ", "ROWS", 1, 2)
|
|
time.sleep(0.5)
|
|
time.sleep(0.5)
|
|
- Feishu.update_values(log_type, crawler, "5011a2", "F2:Z2",
|
|
|
|
|
|
+ Feishu.update_values(log_type, crawler, "YVuVgQ", "F2:Z2",
|
|
[[time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(int(time.time()))),
|
|
[[time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(int(time.time()))),
|
|
- word_list[i],
|
|
|
|
|
|
+ word_list[i]['title'],
|
|
|
|
+ word_list[i]['word'],
|
|
"",
|
|
"",
|
|
"该词暂未收录"]])
|
|
"该词暂未收录"]])
|
|
Common.logger(log_type, crawler).info("写入飞书成功\n")
|
|
Common.logger(log_type, crawler).info("写入飞书成功\n")
|
|
@@ -125,23 +127,25 @@ class Test:
|
|
else:
|
|
else:
|
|
time_index = response.json()['content']['resp_list'][0]['indexes'][0]['time_indexes']
|
|
time_index = response.json()['content']['resp_list'][0]['indexes'][0]['time_indexes']
|
|
for x in range(len(time_index)):
|
|
for x in range(len(time_index)):
|
|
- Common.logger(log_type, crawler).info(f"正在更新 {word_list[i]}")
|
|
|
|
|
|
+ Common.logger(log_type, crawler).info(f"正在更新 {word_list[i]['word']}")
|
|
score_time = time_index[x]['time']
|
|
score_time = time_index[x]['time']
|
|
score_time_str = f"{str(score_time)[:4]}-{str(score_time)[4:6]}-{str(score_time)[6:]}"
|
|
score_time_str = f"{str(score_time)[:4]}-{str(score_time)[4:6]}-{str(score_time)[6:]}"
|
|
score = time_index[x]['score']
|
|
score = time_index[x]['score']
|
|
wechat_score_dict = {"score": score, "scoreDate": score_time_str}
|
|
wechat_score_dict = {"score": score, "scoreDate": score_time_str}
|
|
wechat_score_list.append(wechat_score_dict)
|
|
wechat_score_list.append(wechat_score_dict)
|
|
Common.logger(log_type, crawler).info(f"wechat_score_dict:{wechat_score_dict}")
|
|
Common.logger(log_type, crawler).info(f"wechat_score_dict:{wechat_score_dict}")
|
|
- Feishu.insert_columns(log_type, crawler, "5011a2", "ROWS", 1, 2)
|
|
|
|
|
|
+ Feishu.insert_columns(log_type, crawler, "YVuVgQ", "ROWS", 1, 2)
|
|
time.sleep(1)
|
|
time.sleep(1)
|
|
- Feishu.update_values(log_type, crawler, "5011a2", "F2:Z2", [[time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(int(time.time()))),
|
|
|
|
- word_list[i],
|
|
|
|
- score_time_str,
|
|
|
|
- score]])
|
|
|
|
|
|
+ Feishu.update_values(log_type, crawler, "YVuVgQ", "F2:Z2", [[time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(int(time.time()))),
|
|
|
|
+ word_list[i]['title'],
|
|
|
|
+ word_list[i]['word'],
|
|
|
|
+ score_time_str,
|
|
|
|
+ score]])
|
|
Common.logger(log_type, crawler).info("写入飞书成功\n")
|
|
Common.logger(log_type, crawler).info("写入飞书成功\n")
|
|
break
|
|
break
|
|
|
|
|
|
|
|
|
|
if __name__ == "__main__":
|
|
if __name__ == "__main__":
|
|
- Test.get_score_test("out", "weixinzhishu")
|
|
|
|
|
|
+ # print(Test.get_words("test", "weixinzhishu"))
|
|
|
|
+ Test.get_score_test("out-1", "weixinzhishu")
|
|
pass
|
|
pass
|