yujunjun/olmocr
1
0
Fork 0
mirror of https://github.com/allenai/olmocr.git synced 2025-09-26 00:40:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

Better markdown table parsing

Browse Source
This commit is contained in:
Jake Poznanski 2025-03-10 16:40:30 +00:00
parent 3fef3f914f
commit a2b5ca8d41
1 changed files with 28 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
olmocr/bench/tests.py
View File

@ -193,8 +193,15 @@ class TableTest(BasePDFTest):
Returns:
A list of numpy arrays, each representing a parsed table
"""
# Extract all tables from markdown
table_pattern = r'(\|(?:[^|]*\|)+)\s*\n\|(?:[ :-]+\|)+\s*\n((?:\|(?:[^|]*\|)+\s*\n)+)'
import re
import numpy as np
# Updated regex to allow optional leading and trailing pipes
table_pattern = (
r'(\|?(?:[^|\n]*\|)+[^|\n]*\|?)\s*\n'
r'\|?(?:[ :-]+\|)+[ :-]+\|?\s*\n'
r'((?:\|?(?:[^|\n]*\|)+[^|\n]*\|?\s*\n)+)'
)
table_matches = re.finditer(table_pattern, md_content)
parsed_tables = []
@ -204,11 +211,11 @@ class TableTest(BasePDFTest):
header_row = table_match.group(1).strip()
body_rows = table_match.group(2).strip().split('\n')
# Process header and rows to remove leading/trailing |
# Process header and rows to remove leading/trailing pipes
header_cells = [cell.strip() for cell in header_row.split('|')]
if header_cells[0] == '':
if header_cells and header_cells[0] == '':
header_cells = header_cells[1:]
if header_cells[-1] == '':
if header_cells and header_cells[-1] == '':
header_cells = header_cells[:-1]
# Process table body rows
@ -218,9 +225,9 @@ class TableTest(BasePDFTest):
continue
cells = [cell.strip() for cell in row.split('|')]
if cells[0] == '':
if cells and cells[0] == '':
cells = cells[1:]
if cells[-1] == '':
if cells and cells[-1] == '':
cells = cells[:-1]
table_data.append(cells)
@ -230,7 +237,7 @@ class TableTest(BasePDFTest):
table_data = [table_data[0]] + table_data[2:]
# Convert to numpy array for easier manipulation
# First ensure all rows have the same number of columns by padding if necessary
# Ensure all rows have the same number of columns by padding if necessary
max_cols = max(len(row) for row in table_data)
padded_data = [row + [''] * (max_cols - len(row)) for row in table_data]
table_array = np.array(padded_data)
@ -239,7 +246,6 @@ class TableTest(BasePDFTest):
return parsed_tables
def parse_html_tables(self, html_content: str) -> List[np.ndarray]:
"""
Extract and parse all HTML tables from the provided content.