|
@@ -1,5 +1,6 @@
|
|
import asyncio
|
|
import asyncio
|
|
from typing import List
|
|
from typing import List
|
|
|
|
+from tqdm import tqdm
|
|
|
|
|
|
from applications.api import get_basic_embedding
|
|
from applications.api import get_basic_embedding
|
|
from applications.utils.async_utils import run_tasks_with_asyncio_task_group
|
|
from applications.utils.async_utils import run_tasks_with_asyncio_task_group
|
|
@@ -233,7 +234,7 @@ class ChunkEmbeddingTask(TopicAwarePackerV2):
|
|
return
|
|
return
|
|
|
|
|
|
# dev
|
|
# dev
|
|
- for chunk in chunks:
|
|
|
|
|
|
+ for chunk in tqdm(chunks):
|
|
await self.save_each_chunk(chunk)
|
|
await self.save_each_chunk(chunk)
|
|
|
|
|
|
# await run_tasks_with_asyncio_task_group(
|
|
# await run_tasks_with_asyncio_task_group(
|