yujunjun/olmocr
1
0
Fork 0
mirror of https://github.com/allenai/olmocr.git synced 2025-11-12 00:20:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into jakep/olmocr_v03

Browse Source
This commit is contained in:
Jake Poznanski 2025-08-04 22:28:29 +00:00
commit 3ae173bd72
1 changed files with 7 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
olmocr/train/prepare_olmocrmix.py
View File

@ -155,10 +155,14 @@ def prepare_olmocr_mix(dataset_path: str, subset: str, split: str, destination:
f.write("---\n") f.write("---\n")
for k, v in front_matter.items(): for k, v in front_matter.items():
f.write(f"{k}: {v}\n") f.write(f"{k}: {v}\n")
if natural_text is not None and len(natural_text.strip()) > 0:
f.write("---\n") f.write("---\n")
# Write natural text # Write natural text
f.write(natural_text) f.write(natural_text)
else:
f.write("---")
# Look for matching PDF in extracted directory and create symlinks # Look for matching PDF in extracted directory and create symlinks
extracted_pdfs_dir = dest_path / "hugging_face" / "pdf_tarballs" / "extracted" extracted_pdfs_dir = dest_path / "hugging_face" / "pdf_tarballs" / "extracted"