mirror of
https://github.com/Unstructured-IO/unstructured.git
synced 2025-07-03 15:11:30 +00:00
f193d3d43b
### Description Building off of PR https://github.com/Unstructured-IO/unstructured/pull/2179, updating fsspec based connectors to use better authentication field handling. This PR adds in the following changes: * Update the base classes to inherit from the enhanced json mixin * Add in a new access config dataclass that should be used as a nest dataclass in the connector configs * Update the code extracting configs out of the cli options dictionary to support the nested access config if it exists on the parent config * Update all fsspec connectors with explicit access configs given what each one's SDKs support * Update the json mixin and enhanced field to support a name override when serializing/deserializing from json/dicts. This allows a different name to be used for the CLI option than what the name of the field is on the dataclass. * Update all the writes to use class-based approach and share the same structure of the runner classes * Above update allowed for better code to be used in the base source and destination CLI commands * Add in utility code around paring a flat dictionary (coming from the click based options) into dataclass-based configs with potentially nested dataclasses. **Slightly unrelated changes:** * session handle removed from pinecone connector as this was breaking the serialization of the write config and didn't have any benefit as a connection was never being shared, the index used simply makes a new http call each time it's invoked. * Dedicated write configs were created for all destination connectors to better support serialization * Refactor of Elasticsearch connector included, with update to ingest test to use auth **TODOs** * Left a `#TODO` in the code but the way session handler is implemented right now, it breaks serialization since it adds a generic variable based on the library being used for a connector (i.e. `googleapiclient.discovery.Resource`) which is not serializable. This will need to be updated to omit that from serialization but still support the current workflow. --------- Co-authored-by: ryannikolaidis <1208590+ryannikolaidis@users.noreply.github.com> Co-authored-by: rbiseck3 <rbiseck3@users.noreply.github.com>
102 lines
2.7 KiB
Python
102 lines
2.7 KiB
Python
import json
|
|
import typing as t
|
|
from dataclasses import dataclass, field
|
|
|
|
from unstructured.ingest.cli.utils import extract_config
|
|
from unstructured.ingest.interfaces import BaseConfig
|
|
|
|
|
|
@dataclass
|
|
class A(BaseConfig):
|
|
a: str
|
|
|
|
|
|
@dataclass
|
|
class B(BaseConfig):
|
|
a: A
|
|
b: int
|
|
|
|
|
|
flat_data = {"a": "test", "b": 4, "c": True}
|
|
|
|
|
|
def test_extract_config_concrete():
|
|
@dataclass
|
|
class C(BaseConfig):
|
|
b: B
|
|
c: bool
|
|
|
|
c = extract_config(flat_data=flat_data, config=C)
|
|
expected_result = {"b": {"a": {"a": "test"}, "b": 4}, "c": True}
|
|
assert c.to_json(sort_keys=True) == json.dumps(expected_result, sort_keys=True)
|
|
|
|
|
|
def test_extract_config_optional():
|
|
@dataclass
|
|
class C(BaseConfig):
|
|
c: bool
|
|
b: t.Optional[B] = None
|
|
|
|
c = extract_config(flat_data=flat_data, config=C)
|
|
expected_result = {"b": {"a": {"a": "test"}, "b": 4}, "c": True}
|
|
assert c.to_json(sort_keys=True) == json.dumps(expected_result, sort_keys=True)
|
|
|
|
|
|
def test_extract_config_union():
|
|
@dataclass
|
|
class C(BaseConfig):
|
|
c: bool
|
|
b: t.Optional[t.Union[B, int]] = None
|
|
|
|
c = extract_config(flat_data=flat_data, config=C)
|
|
expected_result = {"b": 4, "c": True}
|
|
assert c.to_json(sort_keys=True) == json.dumps(expected_result, sort_keys=True)
|
|
|
|
|
|
def test_extract_config_list():
|
|
@dataclass
|
|
class C(BaseConfig):
|
|
c: t.List[int]
|
|
b: B
|
|
|
|
flat_data = {"a": "test", "b": 4, "c": [1, 2, 3]}
|
|
c = extract_config(flat_data=flat_data, config=C)
|
|
expected_result = {"b": {"a": {"a": "test"}, "b": 4}, "c": [1, 2, 3]}
|
|
assert c.to_json(sort_keys=True) == json.dumps(expected_result, sort_keys=True)
|
|
|
|
|
|
def test_extract_config_optional_list():
|
|
@dataclass
|
|
class C(BaseConfig):
|
|
b: B
|
|
c: t.Optional[t.List[int]] = None
|
|
|
|
flat_data = {"a": "test", "b": 4, "c": [1, 2, 3]}
|
|
c = extract_config(flat_data=flat_data, config=C)
|
|
expected_result = {"b": {"a": {"a": "test"}, "b": 4}, "c": [1, 2, 3]}
|
|
assert c.to_json(sort_keys=True) == json.dumps(expected_result, sort_keys=True)
|
|
|
|
|
|
def test_extract_config_dataclass_list():
|
|
@dataclass
|
|
class C(BaseConfig):
|
|
c: bool
|
|
b: t.List[B] = field(default_factory=list)
|
|
|
|
flat_data = {"a": "test", "c": True}
|
|
c = extract_config(flat_data=flat_data, config=C)
|
|
expected_result = {"b": [], "c": True}
|
|
assert c.to_json(sort_keys=True) == json.dumps(expected_result, sort_keys=True)
|
|
|
|
|
|
def test_extract_config_dict():
|
|
@dataclass
|
|
class C(BaseConfig):
|
|
c: bool
|
|
b: t.Dict[str, B] = field(default_factory=dict)
|
|
|
|
flat_data = {"c": True}
|
|
c = extract_config(flat_data=flat_data, config=C)
|
|
expected_result = {"c": True, "b": {}}
|
|
assert c.to_json(sort_keys=True) == json.dumps(expected_result, sort_keys=True)
|