|
@@ -98,16 +98,22 @@ if __name__ == "__main__":
|
|
|
long_articles_db_client = DatabaseConnector(db_config=long_articles_config)
|
|
|
long_articles_db_client.connect()
|
|
|
|
|
|
- # Number of concurrent threads
|
|
|
- MAX_WORKERS = 4
|
|
|
+ # # Number of concurrent threads
|
|
|
+ # MAX_WORKERS = 4
|
|
|
|
|
|
article_list = get_article_list(run_date=None)
|
|
|
|
|
|
- with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
|
|
|
- list(
|
|
|
- tqdm(
|
|
|
- executor.map(monitor_article, article_list),
|
|
|
- total=len(article_list),
|
|
|
- desc="Monitor Article List",
|
|
|
- )
|
|
|
- )
|
|
|
+ for article in tqdm(article_list):
|
|
|
+ try:
|
|
|
+ monitor_article(article)
|
|
|
+ except Exception as e:
|
|
|
+ print(e)
|
|
|
+
|
|
|
+ # with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
|
|
|
+ # list(
|
|
|
+ # tqdm(
|
|
|
+ # executor.map(monitor_article, article_list),
|
|
|
+ # total=len(article_list),
|
|
|
+ # desc="Monitor Article List",
|
|
|
+ # )
|
|
|
+ # )
|