|
@@ -85,12 +85,14 @@ class CrawlerToutiao(CrawlerPipeline, CrawlerToutiaoConst):
|
|
|
key="toutiao_blogger_cookie", output_type="string"
|
|
|
)
|
|
|
while has_more:
|
|
|
+ print(account_id, max_cursor)
|
|
|
response = await get_toutiao_account_info_list(
|
|
|
account_id=account_id,
|
|
|
cookie=cookie,
|
|
|
media_type=media_type,
|
|
|
max_behot_time=current_cursor,
|
|
|
)
|
|
|
+ print(response)
|
|
|
if not response:
|
|
|
break
|
|
|
|
|
@@ -118,6 +120,7 @@ class CrawlerToutiao(CrawlerPipeline, CrawlerToutiaoConst):
|
|
|
raise Exception(f"unknown media type: {media_type}")
|
|
|
|
|
|
crawler_info_list_bar = tqdm(info_list, desc=bar_description)
|
|
|
+ print(json.dumps(info_list, ensure_ascii=False, indent=4))
|
|
|
for info in crawler_info_list_bar:
|
|
|
try:
|
|
|
crawler_info_list_bar.set_postfix({"id": info["id"]})
|