mirror of
https://github.com/microsoft/graphrag.git
synced 2025-06-26 23:19:58 +00:00
1df89727c3
* Move covariate run conditional * All pipeline registration * Fix method name construction * Rename context storage -> output_storage * Rename OutputConfig as generic StorageConfig * Reuse Storage model under InputConfig * Move input storage creation out of document loading * Move document loading into workflows * Semver * Fix smoke test config for new workflows * Fix unit tests --------- Co-authored-by: Alonso Guevara <alonsog@microsoft.com>
82 lines
2.8 KiB
Python
82 lines
2.8 KiB
Python
# Copyright (c) 2024 Microsoft Corporation.
|
|
# Licensed under the MIT License
|
|
|
|
from graphrag.config.enums import InputFileType
|
|
from graphrag.config.models.input_config import InputConfig
|
|
from graphrag.config.models.storage_config import StorageConfig
|
|
from graphrag.index.input.factory import create_input
|
|
from graphrag.utils.api import create_storage_from_config
|
|
|
|
|
|
async def test_json_loader_one_file_one_object():
|
|
config = InputConfig(
|
|
storage=StorageConfig(
|
|
base_dir="tests/unit/indexing/input/data/one-json-one-object",
|
|
),
|
|
file_type=InputFileType.json,
|
|
file_pattern=".*\\.json$",
|
|
)
|
|
storage = create_storage_from_config(config.storage)
|
|
documents = await create_input(config=config, storage=storage)
|
|
assert documents.shape == (1, 4)
|
|
assert documents["title"].iloc[0] == "input.json"
|
|
|
|
|
|
async def test_json_loader_one_file_multiple_objects():
|
|
config = InputConfig(
|
|
storage=StorageConfig(
|
|
base_dir="tests/unit/indexing/input/data/one-json-multiple-objects",
|
|
),
|
|
file_type=InputFileType.json,
|
|
file_pattern=".*\\.json$",
|
|
)
|
|
storage = create_storage_from_config(config.storage)
|
|
documents = await create_input(config=config, storage=storage)
|
|
print(documents)
|
|
assert documents.shape == (3, 4)
|
|
assert documents["title"].iloc[0] == "input.json"
|
|
|
|
|
|
async def test_json_loader_one_file_with_title():
|
|
config = InputConfig(
|
|
storage=StorageConfig(
|
|
base_dir="tests/unit/indexing/input/data/one-json-one-object",
|
|
),
|
|
file_type=InputFileType.json,
|
|
file_pattern=".*\\.json$",
|
|
title_column="title",
|
|
)
|
|
storage = create_storage_from_config(config.storage)
|
|
documents = await create_input(config=config, storage=storage)
|
|
assert documents.shape == (1, 4)
|
|
assert documents["title"].iloc[0] == "Hello"
|
|
|
|
|
|
async def test_json_loader_one_file_with_metadata():
|
|
config = InputConfig(
|
|
storage=StorageConfig(
|
|
base_dir="tests/unit/indexing/input/data/one-json-one-object",
|
|
),
|
|
file_type=InputFileType.json,
|
|
file_pattern=".*\\.json$",
|
|
title_column="title",
|
|
metadata=["title"],
|
|
)
|
|
storage = create_storage_from_config(config.storage)
|
|
documents = await create_input(config=config, storage=storage)
|
|
assert documents.shape == (1, 5)
|
|
assert documents["metadata"][0] == {"title": "Hello"}
|
|
|
|
|
|
async def test_json_loader_multiple_files():
|
|
config = InputConfig(
|
|
storage=StorageConfig(
|
|
base_dir="tests/unit/indexing/input/data/multiple-jsons",
|
|
),
|
|
file_type=InputFileType.json,
|
|
file_pattern=".*\\.json$",
|
|
)
|
|
storage = create_storage_from_config(config.storage)
|
|
documents = await create_input(config=config, storage=storage)
|
|
assert documents.shape == (4, 4)
|