yujunjun/olmocr
1
0
Fork 0
mirror of https://github.com/allenai/olmocr.git synced 2025-08-18 05:42:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' of https://github.com/allenai/pdelfin

Browse Source
This commit is contained in:
Jake Poznanski 2025-01-23 23:32:56 +00:00
commit aa59d38a5b
2 changed files with 8 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
pdelfin/train/loaddataset.py
View File

@ -20,7 +20,13 @@ def main():
print("Training dataset........") print("Training dataset........")
print(train_dataset) print(train_dataset)
print(train_dataset[0])
train_example = train_dataset[0]
print(train_example)
print({(x, y.shape) for x,y in train_example.items()})
print("\nTokens")
print(processor.tokenizer.batch_decode(train_example["input_ids"]))
print("\n\n") print("\n\n")
print("Validation dataset........") print("Validation dataset........")

1
tests/test_molmo.py
View File

@ -37,6 +37,7 @@ class MolmoProcessorTest(unittest.TestCase):
print(inputs) print(inputs)
print("\nShapes") print("\nShapes")
# {('input_ids', torch.Size([1, 589])), ('images', torch.Size([1, 5, 576, 588])), ('image_masks', torch.Size([1, 5, 576])), ('image_input_idx', torch.Size([1, 5, 144]))}
print({(x, y.shape) for x,y in inputs.items()}) print({(x, y.shape) for x,y in inputs.items()})
print("\nTokens") print("\nTokens")