mirror of
https://github.com/deepset-ai/haystack.git
synced 2025-07-27 10:49:52 +00:00
d470b9d0bd
* Fist attempt at using setup.cfg for dependency management * Trying the new package on the CI and in Docker too * Add composite extras_require * Add the safe_import function for document store imports and add some try-catch statements on rest_api and ui imports * Fix bug on class import and rephrase error message * Introduce typing for optional modules and add type: ignore in sparse.py * Include importlib_metadata backport for py3.7 * Add colab group to extra_requires * Fix pillow version * Fix grpcio * Separate out the crawler as another extra * Make paths relative in rest_api and ui * Update the test matrix in the CI * Add try catch statements around the optional imports too to account for direct imports * Never mix direct deps with self-references and add ES deps to the base install * Refactor several paths in tests to make them insensitive to the execution path * Include tstadel review and re-introduce Milvus1 in the tests suite, to fix * Wrap pdf conversion utils into safe_import * Update some tutorials and rever Milvus1 as default for now, see #2067 * Fix mypy config Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
50 lines
1.5 KiB
Python
50 lines
1.5 KiB
Python
import logging
|
|
from pathlib import Path
|
|
|
|
from haystack.modeling.data_handler.processor import SquadProcessor
|
|
from haystack.modeling.model.tokenization import Tokenizer
|
|
from haystack.modeling.utils import set_all_seeds
|
|
import torch
|
|
|
|
from conftest import SAMPLES_PATH
|
|
|
|
|
|
def test_processor_saving_loading(caplog):
|
|
if caplog is not None:
|
|
caplog.set_level(logging.CRITICAL)
|
|
|
|
set_all_seeds(seed=42)
|
|
lang_model = "roberta-base"
|
|
|
|
tokenizer = Tokenizer.load(
|
|
pretrained_model_name_or_path=lang_model, do_lower_case=False
|
|
)
|
|
|
|
processor = SquadProcessor(
|
|
tokenizer=tokenizer,
|
|
max_seq_len=256,
|
|
label_list=["start_token", "end_token"],
|
|
train_filename="train-sample.json",
|
|
dev_filename="dev-sample.json",
|
|
test_filename=None,
|
|
data_dir=SAMPLES_PATH/"qa",
|
|
)
|
|
|
|
dicts = processor.file_to_dicts(file=SAMPLES_PATH/"qa"/"dev-sample.json")
|
|
data, tensor_names, _ = processor.dataset_from_dicts(dicts=dicts, indices=[1])
|
|
|
|
save_dir = Path("testsave/processor")
|
|
processor.save(save_dir)
|
|
|
|
processor = processor.load_from_dir(save_dir)
|
|
dicts = processor.file_to_dicts(file=SAMPLES_PATH/"qa"/"dev-sample.json")
|
|
data_loaded, tensor_names_loaded, _ = processor.dataset_from_dicts(dicts, indices=[1])
|
|
|
|
assert tensor_names == tensor_names_loaded
|
|
for i in range(len(data.tensors)):
|
|
assert torch.all(torch.eq(data.tensors[i], data_loaded.tensors[i]))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
test_processor_saving_loading(None)
|