yujunjun/olmocr
1
0
Fork 0
mirror of https://github.com/allenai/olmocr.git synced 2025-08-19 14:22:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixing work queue population

Browse Source
This commit is contained in:
Jake Poznanski 2024-11-14 12:48:46 -08:00
parent 827b77e8df
commit b67d8e7555
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
pdelfin/beakerpipeline.py
View File

@ -194,7 +194,7 @@ async def populate_pdf_work_queue(args):
async def load_pdf_work_queue(args) -> asyncio.Queue: async def load_pdf_work_queue(args) -> asyncio.Queue:
index_file_s3_path = os.path.join(args.workspace, "pdf_index_list.csv.zstd") index_file_s3_path = os.path.join(args.workspace, "pdf_index_list.csv.zstd")
output_glob = f"{args.workspace}/dolma_documents/output_*.jsonl" output_glob = os.path.join(args.workspace, "dolma_documents", "*.jsonl")
# Define the two blocking I/O operations # Define the two blocking I/O operations
download_task = asyncio.to_thread(download_zstd_csv, workspace_s3, index_file_s3_path) download_task = asyncio.to_thread(download_zstd_csv, workspace_s3, index_file_s3_path)