yujunjun/olmocr
1
0
Fork 0
mirror of https://github.com/allenai/olmocr.git synced 2025-11-04 12:07:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixing one old bug to make update_static atomic

Browse Source
This commit is contained in:
Jake Poznanski 2024-10-23 17:51:22 +00:00
parent 38dc5a2a0f
commit 4047258277
1 changed files with 13 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
pdelfin/data/runopenaibatch.py
View File

@ -120,8 +120,19 @@ def update_state(folder_path: str, filename: str, **kwargs):
all_state[filename]["last_checked"] = datetime.datetime.now() all_state[filename]["last_checked"] = datetime.datetime.now()
state_file = os.path.join(folder_path, UPLOAD_STATE_FILENAME) state_file = os.path.join(folder_path, UPLOAD_STATE_FILENAME)
with open(state_file, "w") as f: temp_file = state_file + '.tmp'
return json.dump(all_state, f, default=_json_datetime_encoder)
# Write to temporary file first
with open(temp_file, "w") as f:
json.dump(all_state, f, default=_json_datetime_encoder)
f.flush()
os.fsync(f.fileno())
# Atomic rename of temporary file to target file
os.replace(temp_file, state_file)
return all_state
def get_total_space_usage(): def get_total_space_usage():
return sum(file.bytes for file in client.files.list()) return sum(file.bytes for file in client.files.list())