mirror of
https://github.com/deepset-ai/haystack.git
synced 2025-07-27 10:49:52 +00:00
0b0b9689a4
* add tinybert data augmentation * don't reload glove in tinybert data augmentation * fix unnecessary load_glove call * fix type hints * add comments and type hints * add batch_size argument * don't predict subwords as alternative for words * fix subword predictions * limit sequence length * actually limit sequence length * improve performance by calculating nearest glove vector on gpu * add model and tokenizer parameter * fix type hints * improve data augmentation performance * explained limits of script * corrected comment * added data augmentation test * don't label every question in augmented dataset as impossible * add sample glove * better handling of downloading of glove * fix typo of last commit
28 lines
1.3 KiB
Python
28 lines
1.3 KiB
Python
import pytest
|
|
from pathlib import Path
|
|
|
|
from haystack.utils.preprocessing import convert_files_to_dicts, tika_convert_files_to_dicts
|
|
from haystack.utils.cleaning import clean_wiki_text
|
|
from haystack.utils.augment_squad import augment_squad
|
|
from haystack.utils.squad_data import SquadData
|
|
|
|
def test_convert_files_to_dicts():
|
|
documents = convert_files_to_dicts(dir_path="samples", clean_func=clean_wiki_text, split_paragraphs=True)
|
|
assert documents and len(documents) > 0
|
|
|
|
|
|
@pytest.mark.tika
|
|
def test_tika_convert_files_to_dicts():
|
|
documents = tika_convert_files_to_dicts(dir_path="samples", clean_func=clean_wiki_text, split_paragraphs=True)
|
|
assert documents and len(documents) > 0
|
|
|
|
def test_squad_augmentation():
|
|
input_ = Path("samples/squad/tiny.json")
|
|
output = Path("samples/squad/tiny_augmented.json")
|
|
glove_path = Path("samples/glove/tiny.txt") # dummy glove file, will not even be use when augmenting tiny.json
|
|
multiplication_factor = 5
|
|
augment_squad("distilbert-base-uncased", "distilbert-base-uncased", input_, output,
|
|
glove_path, multiplication_factor=multiplication_factor)
|
|
original_squad = SquadData.from_file(input_)
|
|
augmented_squad = SquadData.from_file(output)
|
|
assert original_squad.count(unit="paragraph") == augmented_squad.count(unit="paragraph") * multiplication_factor |