mirror of
https://github.com/Unstructured-IO/unstructured.git
synced 2025-07-04 07:27:34 +00:00
39 lines
1012 B
Python
39 lines
1012 B
Python
|
import os
|
||
|
|
import json
|
||
|
|
import pytest
|
||
|
|
|
||
|
|
import unstructured.utils as utils
|
||
|
|
|
||
|
|
|
||
|
|
@pytest.fixture
|
||
|
|
def input_data():
|
||
|
|
return [
|
||
|
|
{"text": "This is a sentence."},
|
||
|
|
{"text": "This is another sentence.", "meta": {"score": 0.1}},
|
||
|
|
]
|
||
|
|
|
||
|
|
|
||
|
|
@pytest.fixture
|
||
|
|
def output_jsonl_file(tmp_path):
|
||
|
|
return os.path.join(tmp_path, "output.jsonl")
|
||
|
|
|
||
|
|
|
||
|
|
@pytest.fixture
|
||
|
|
def input_jsonl_file(tmp_path, input_data):
|
||
|
|
file_path = os.path.join(tmp_path, "input.jsonl")
|
||
|
|
with open(file_path, "w+") as input_file:
|
||
|
|
input_file.writelines([json.dumps(obj) + "\n" for obj in input_data])
|
||
|
|
return file_path
|
||
|
|
|
||
|
|
|
||
|
|
def test_save_as_jsonl(input_data, output_jsonl_file):
|
||
|
|
utils.save_as_jsonl(input_data, output_jsonl_file)
|
||
|
|
with open(output_jsonl_file, "r") as output_file:
|
||
|
|
file_data = [json.loads(line) for line in output_file]
|
||
|
|
assert file_data == input_data
|
||
|
|
|
||
|
|
|
||
|
|
def test_read_as_jsonl(input_jsonl_file, input_data):
|
||
|
|
file_data = utils.read_from_jsonl(input_jsonl_file)
|
||
|
|
assert file_data == input_data
|