yujunjun/olmocr
1
0
Fork 0
mirror of https://github.com/allenai/olmocr.git synced 2025-10-12 08:43:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

Convert fix

Browse Source
This commit is contained in:
Jake Poznanski 2025-09-19 17:36:34 +00:00
parent b1242db8e2
commit a8ad6c12b5
1 changed files with 23 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
olmocr/bench/convert.py
View File

@ -192,15 +192,29 @@ async def process_pdfs(config, pdf_directory, data_directory, repeats, remove_te
if limited_tasks: if limited_tasks:
completed = 0 completed = 0
with tqdm(total=len(limited_tasks), desc=f"Processing {candidate}") as pbar: with tqdm(total=len(limited_tasks), desc=f"Processing {candidate}") as pbar:
for task in asyncio.as_completed(limited_tasks): # When parallel=0, tasks complete synchronously and we need to handle them differently
try: if max_parallel == 0:
result = await task # Process tasks sequentially with immediate progress updates
if result: for task in limited_tasks:
completed += 1 try:
except Exception as e: result = await task
print(f"Task failed: {e}") if result:
finally: completed += 1
pbar.update(1) except Exception as e:
print(f"Task failed: {e}")
finally:
pbar.update(1)
else:
# Use as_completed for parallel processing
for task in asyncio.as_completed(limited_tasks):
try:
result = await task
if result:
completed += 1
except Exception as e:
print(f"Task failed: {e}")
finally:
pbar.update(1)
print(f"Completed {completed} out of {len(limited_tasks)} tasks for {candidate}") print(f"Completed {completed} out of {len(limited_tasks)} tasks for {candidate}")
finally: finally: