yujunjun/olmocr
1
0
Fork 0
mirror of https://github.com/allenai/olmocr.git synced 2025-10-11 00:02:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

Convert fix

Browse Source
This commit is contained in:
Jake Poznanski 2025-09-19 17:36:34 +00:00
parent b1242db8e2
commit a8ad6c12b5
1 changed files with 23 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
olmocr/bench/convert.py
View File

@ -192,15 +192,29 @@ async def process_pdfs(config, pdf_directory, data_directory, repeats, remove_te
if limited_tasks:
completed = 0
with tqdm(total=len(limited_tasks), desc=f"Processing {candidate}") as pbar:
for task in asyncio.as_completed(limited_tasks):
try:
result = await task
if result:
completed += 1
except Exception as e:
print(f"Task failed: {e}")
finally:
pbar.update(1)
# When parallel=0, tasks complete synchronously and we need to handle them differently
if max_parallel == 0:
# Process tasks sequentially with immediate progress updates
for task in limited_tasks:
try:
result = await task
if result:
completed += 1
except Exception as e:
print(f"Task failed: {e}")
finally:
pbar.update(1)
else:
# Use as_completed for parallel processing
for task in asyncio.as_completed(limited_tasks):
try:
result = await task
if result:
completed += 1
except Exception as e:
print(f"Task failed: {e}")
finally:
pbar.update(1)
print(f"Completed {completed} out of {len(limited_tasks)} tasks for {candidate}")
finally: