mirror of
https://github.com/datahub-project/datahub.git
synced 2025-07-23 09:32:04 +00:00
84 lines
2.6 KiB
Python
84 lines
2.6 KiB
Python
|
import random
|
||
|
|
import re
|
||
|
|
import time
|
||
|
|
|
||
|
|
import pytest
|
||
|
|
|
||
|
|
from datahub.utilities.lossy_collections import LossyDict, LossyList, LossySet
|
||
|
|
|
||
|
|
|
||
|
|
@pytest.mark.parametrize("length, sampling", [(10, False), (100, True)])
|
||
|
|
def test_lossylist_sampling(length, sampling):
|
||
|
|
l: LossyList[str] = LossyList()
|
||
|
|
for i in range(0, length):
|
||
|
|
l.append(f"{i} Hello World")
|
||
|
|
|
||
|
|
assert len(l) == length
|
||
|
|
assert l.sampled is sampling
|
||
|
|
if sampling:
|
||
|
|
assert f"... sampled of {length} total elements" in str(l)
|
||
|
|
else:
|
||
|
|
assert "sampled" not in str(l)
|
||
|
|
|
||
|
|
list_version = [int(i.split(" ")[0]) for i in l]
|
||
|
|
print(list_version)
|
||
|
|
assert sorted(list_version) == list_version
|
||
|
|
|
||
|
|
|
||
|
|
@pytest.mark.parametrize("length, sampling", [(10, False), (100, True)])
|
||
|
|
def test_lossyset_sampling(length, sampling):
|
||
|
|
l: LossySet[str] = LossySet()
|
||
|
|
for i in range(0, length):
|
||
|
|
l.add(f"{i} Hello World")
|
||
|
|
|
||
|
|
assert len(l) == min(10, length)
|
||
|
|
assert l.sampled is sampling
|
||
|
|
if sampling:
|
||
|
|
assert f"... sampled with at most {length-10} elements missing" in str(l)
|
||
|
|
else:
|
||
|
|
assert "sampled" not in str(l)
|
||
|
|
|
||
|
|
list_version = [int(i.split(" ")[0]) for i in l]
|
||
|
|
set_version = set(list_version)
|
||
|
|
|
||
|
|
assert len(list_version) == len(set_version)
|
||
|
|
assert len(list_version) == min(10, length)
|
||
|
|
|
||
|
|
|
||
|
|
@pytest.mark.parametrize(
|
||
|
|
"length, sampling, sub_length", [(4, False, 4), (10, False, 14), (100, True, 1000)]
|
||
|
|
)
|
||
|
|
def test_lossydict_sampling(length, sampling, sub_length):
|
||
|
|
l: LossyDict[int, LossyList[str]] = LossyDict()
|
||
|
|
elements_added = 0
|
||
|
|
element_length_map = {}
|
||
|
|
for i in range(0, length):
|
||
|
|
list_length = random.choice(range(1, sub_length))
|
||
|
|
element_length_map[i] = 0
|
||
|
|
for num_elements in range(0, list_length):
|
||
|
|
if not l.get(i):
|
||
|
|
elements_added += 1
|
||
|
|
# reset to 0 until we get it back
|
||
|
|
element_length_map[i] = 0
|
||
|
|
else:
|
||
|
|
element_length_map[i] = len(l[i])
|
||
|
|
|
||
|
|
current_list = l.get(i, LossyList())
|
||
|
|
current_list.append(f"{i}:{round(time.time(),2)} Hello World")
|
||
|
|
l[i] = current_list
|
||
|
|
element_length_map[i] += 1
|
||
|
|
|
||
|
|
assert len(l) == min(l.max_elements, length)
|
||
|
|
assert l.sampled is sampling
|
||
|
|
if sampling:
|
||
|
|
assert re.search("sampled of at most .* entries.", str(l))
|
||
|
|
assert f"{l.max_elements} sampled of at most {elements_added} entries." in str(
|
||
|
|
l
|
||
|
|
)
|
||
|
|
else:
|
||
|
|
# cheap way to determine that the dict isn't reporting sampled keys
|
||
|
|
assert not re.search("sampled of at most .* entries.", str(l))
|
||
|
|
|
||
|
|
for k, v in l.items():
|
||
|
|
assert len(v) == element_length_map[k]
|