mirror of
https://github.com/Unstructured-IO/unstructured.git
synced 2025-07-03 23:20:35 +00:00
28a4ae985d
* Implement save_as_jsonl and read_from_jsonl utility functions * Add unit tests for save_as_jsonl and read_from_jsonl utility functions * Add example of using save_as_jsonl with prodigy staging brick * Bump version and update changelog * remove accidentally added prodigy json file * added "the" in jsonl description Co-authored-by: Matt Robinson <mrobinson@unstructuredai.io>
39 lines
1012 B
Python
39 lines
1012 B
Python
import os
|
|
import json
|
|
import pytest
|
|
|
|
import unstructured.utils as utils
|
|
|
|
|
|
@pytest.fixture
|
|
def input_data():
|
|
return [
|
|
{"text": "This is a sentence."},
|
|
{"text": "This is another sentence.", "meta": {"score": 0.1}},
|
|
]
|
|
|
|
|
|
@pytest.fixture
|
|
def output_jsonl_file(tmp_path):
|
|
return os.path.join(tmp_path, "output.jsonl")
|
|
|
|
|
|
@pytest.fixture
|
|
def input_jsonl_file(tmp_path, input_data):
|
|
file_path = os.path.join(tmp_path, "input.jsonl")
|
|
with open(file_path, "w+") as input_file:
|
|
input_file.writelines([json.dumps(obj) + "\n" for obj in input_data])
|
|
return file_path
|
|
|
|
|
|
def test_save_as_jsonl(input_data, output_jsonl_file):
|
|
utils.save_as_jsonl(input_data, output_jsonl_file)
|
|
with open(output_jsonl_file, "r") as output_file:
|
|
file_data = [json.loads(line) for line in output_file]
|
|
assert file_data == input_data
|
|
|
|
|
|
def test_read_as_jsonl(input_jsonl_file, input_data):
|
|
file_data = utils.read_from_jsonl(input_jsonl_file)
|
|
assert file_data == input_data
|