mirror of
https://github.com/Unstructured-IO/unstructured.git
synced 2025-07-04 15:42:16 +00:00
ce905dd098
Closes #1781. - Adds a Weaviate destination connector - The connector receives a host for the weaviate instance and a weaviate class name. - Defines a weaviate schema for json elements. - Defines the pre-processing to conform unstructured's schema to the proposed weaviate schema.
26 lines
751 B
Python
Executable File
26 lines
751 B
Python
Executable File
#!/usr/bin/env python3
|
|
|
|
import os
|
|
import sys
|
|
|
|
import weaviate
|
|
|
|
weaviate_host_url = os.getenv("WEAVIATE_HOST_URL", "http://localhost:8080")
|
|
class_name = os.getenv("WEAVIATE_CLASS_NAME", "Elements")
|
|
N_ELEMENTS = 5
|
|
|
|
if __name__ == "__main__":
|
|
print(f"Checking contents of class collection " f"{class_name} at {weaviate_host_url}")
|
|
|
|
client = weaviate.Client(
|
|
url=weaviate_host_url,
|
|
)
|
|
|
|
response = client.query.aggregate(class_name).with_meta_count().do()
|
|
count = response["data"]["Aggregate"][class_name][0]["meta"]["count"]
|
|
try:
|
|
assert count == N_ELEMENTS
|
|
except AssertionError:
|
|
sys.exit(f"FAIL: weaviate dest check failed: got {count}, expected {N_ELEMENTS}")
|
|
print("SUCCESS: weaviate dest check")
|