yujunjun/olmocr
1
0
Fork 0
mirror of https://github.com/allenai/olmocr.git synced 2025-10-13 01:02:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

Better error messages

Browse Source
This commit is contained in:
Jake Poznanski 2025-08-04 15:33:29 +00:00
parent c1061146e5
commit 1255f64d59
1 changed files with 10 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
olmocr/s3_utils.py
View File

@ -85,11 +85,17 @@ def expand_s3_glob(s3_client, s3_glob: str) -> dict[str, str]:
def get_s3_bytes(s3_client, s3_path: str, start_index: Optional[int] = None, end_index: Optional[int] = None) -> bytes: def get_s3_bytes(s3_client, s3_path: str, start_index: Optional[int] = None, end_index: Optional[int] = None) -> bytes:
is_cloud_path = s3_path.startswith("s3://") or s3_path.startswith("gs://") or s3_path.startswith("weka://")
# Fall back for local files # Fall back for local files
if os.path.exists(s3_path): if not is_cloud_path:
assert start_index is None and end_index is None, "Range query not supported yet" if os.path.exists(s3_path):
with open(s3_path, "rb") as f: assert start_index is None and end_index is None, "Range query not supported yet"
return f.read() with open(s3_path, "rb") as f:
return f.read()
else:
logger.error(f"Could not find local file {s3_path}")
raise Exception(f"Could not find local file {s3_path}")
bucket, key = parse_s3_path(s3_path) bucket, key = parse_s3_path(s3_path)