yujunjun/olmocr
1
0
Fork 0
mirror of https://github.com/allenai/olmocr.git synced 2025-10-28 16:41:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix pipeline bug with indexing

Browse Source
This commit is contained in:
Jake Poznanski 2024-10-22 15:47:11 +00:00
parent 31becaf7e4
commit 7c7867626f
1 changed files with 5 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
pdelfin/birrpipeline.py
View File

@ -162,6 +162,10 @@ class DatabaseManager:
) )
for row in rows for row in rows
] ]
def delete_index_entries_by_inference_s3_path(self, inference_s3_path: str):
self.cursor.execute("DELETE FROM page_results WHERE inference_s3_path = ?", (inference_s3_path,))
self.conn.commit()
def get_last_indexed_round(self) -> int: def get_last_indexed_round(self) -> int:
self.cursor.execute(""" self.cursor.execute("""
@ -617,6 +621,7 @@ if __name__ == '__main__':
try: try:
inference_records = future.result() inference_records = future.result()
db.delete_index_entries_by_inference_s3_path(s3_path)
db.add_index_entries(inference_records) db.add_index_entries(inference_records)
db.update_processed_file(s3_path, etag=etag) db.update_processed_file(s3_path, etag=etag)
except urllib3.exceptions.SSLError: except urllib3.exceptions.SSLError: