|
|
@@ -564,8 +564,8 @@ class XiguasearchScheduling:
|
|
|
# driver初始化
|
|
|
driver = webdriver.Chrome(desired_capabilities=ca, options=chrome_options, service=Service(chromedriver))
|
|
|
driver.implicitly_wait(10)
|
|
|
- Common.logger(log_type, crawler).info(f"打开搜索页:{user_dict['nick_name']}")
|
|
|
- driver.get(f"https://www.ixigua.com/search/{user_dict['nick_name']}/")
|
|
|
+ Common.logger(log_type, crawler).info(f"打开搜索页:{user_dict['link']}")
|
|
|
+ driver.get(f"https://www.ixigua.com/search/{user_dict['link']}/")
|
|
|
time.sleep(3)
|
|
|
# driver.get_screenshot_as_file(f"./{crawler}/logs/打开搜索页.jpg")
|
|
|
if len(driver.find_elements(By.XPATH, '//*[@class="xg-notification-close"]')) != 0:
|
|
|
@@ -591,7 +591,7 @@ class XiguasearchScheduling:
|
|
|
for i, video_element in enumerate(video_element_temp):
|
|
|
try:
|
|
|
if cls.download_cnt >= int(rule_dict.get("videos_cnt", {}).get("min", 30)):
|
|
|
- Common.logger(log_type, crawler).info(f"搜索词: {user_dict['nick_name']}，已下载视频数: {cls.download_cnt}\n")
|
|
|
+ Common.logger(log_type, crawler).info(f"搜索词: {user_dict['link']}，已下载视频数: {cls.download_cnt}\n")
|
|
|
driver.quit()
|
|
|
return
|
|
|
if video_element is None:
|
|
|
@@ -743,7 +743,7 @@ class XiguasearchScheduling:
|
|
|
|
|
|
# 视频信息写入飞书
|
|
|
Feishu.insert_columns(log_type, crawler, "BUNvGC", "ROWS", 1, 2)
|
|
|
- values = [[user_dict["nick_name"],
|
|
|
+ values = [[user_dict["link"],
|
|
|
time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(int(time.time()))),
|
|
|
"关键词搜索",
|
|
|
video_dict['video_title'],
|
|
|
@@ -772,14 +772,14 @@ class XiguasearchScheduling:
|
|
|
for user_dict in user_list:
|
|
|
try:
|
|
|
cls.download_cnt = 0
|
|
|
- Common.logger(log_type, crawler).info(f"开始抓取 {user_dict['nick_name']} 视频\n")
|
|
|
+ Common.logger(log_type, crawler).info(f"开始抓取 {user_dict['link']} 视频\n")
|
|
|
cls.get_videoList(log_type=log_type,
|
|
|
crawler=crawler,
|
|
|
user_dict=user_dict,
|
|
|
rule_dict=rule_dict,
|
|
|
env=env)
|
|
|
except Exception as e:
|
|
|
- Common.logger(log_type, crawler).error(f"抓取{user_dict['nick_name']}视频时异常:{e}\n")
|
|
|
+ Common.logger(log_type, crawler).error(f"抓取{user_dict['link']}视频时异常:{e}\n")
|
|
|
|
|
|
|
|
|
if __name__ == '__main__':
|
