bugfix: conform all connectors to be added to registry (#3408)

### Description

Looks like some connectors were never added to the registry explicitly
since that change was introduced. All of them are now updated.
This commit is contained in:
Roman Isecke 2024-07-17 11:53:18 -04:00 committed by GitHub
parent a5c9a3695c
commit 35ee6bf8e4
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 39 additions and 40 deletions

View File

@ -18,12 +18,20 @@ from .google_drive import CONNECTOR_TYPE as GOOGLE_DRIVE_CONNECTOR_TYPE
from .google_drive import google_drive_source_entry
from .local import CONNECTOR_TYPE as LOCAL_CONNECTOR_TYPE
from .local import local_destination_entry, local_source_entry
from .mongodb import CONNECTOR_TYPE as MONGODB_CONNECTOR_TYPE
from .mongodb import mongodb_destination_entry
from .onedrive import CONNECTOR_TYPE as ONEDRIVE_CONNECTOR_TYPE
from .onedrive import onedrive_source_entry
from .opensearch import CONNECTOR_TYPE as OPENSEARCH_CONNECTOR_TYPE
from .opensearch import opensearch_destination_entry, opensearch_source_entry
from .pinecone import CONNECTOR_TYPE as PINECONE_CONNECTOR_TYPE
from .pinecone import pinecone_destination_entry
from .salesforce import CONNECTOR_TYPE as SALESFORCE_CONNECTOR_TYPE
from .salesforce import salesforce_source_entry
from .sharepoint import CONNECTOR_TYPE as SHAREPOINT_CONNECTOR_TYPE
from .sharepoint import sharepoint_source_entry
from .singlestore import CONNECTOR_TYPE as SINGLESTORE_CONNECTOR_TYPE
from .singlestore import singlestore_destination_entry
from .sql import CONNECTOR_TYPE as SQL_CONNECTOR_TYPE
from .sql import sql_destination_entry
from .weaviate import CONNECTOR_TYPE as WEAVIATE_CONNECTOR_TYPE
@ -59,3 +67,10 @@ add_destination_entry(
)
add_destination_entry(destination_type=SQL_CONNECTOR_TYPE, entry=sql_destination_entry)
add_destination_entry(destination_type=MONGODB_CONNECTOR_TYPE, entry=mongodb_destination_entry)
add_destination_entry(destination_type=PINECONE_CONNECTOR_TYPE, entry=pinecone_destination_entry)
add_source_entry(source_type=SHAREPOINT_CONNECTOR_TYPE, entry=sharepoint_source_entry)
add_destination_entry(
destination_type=SINGLESTORE_CONNECTOR_TYPE, entry=singlestore_destination_entry
)

View File

@ -19,7 +19,6 @@ from unstructured.ingest.v2.interfaces import (
from unstructured.ingest.v2.logger import logger
from unstructured.ingest.v2.processes.connector_registry import (
DestinationRegistryEntry,
add_destination_entry,
)
from unstructured.utils import requires_dependencies
@ -129,13 +128,10 @@ class MongoDBUploader(Uploader):
collection.insert_many(chunk)
add_destination_entry(
destination_type=CONNECTOR_TYPE,
entry=DestinationRegistryEntry(
connection_config=MongoDBConnectionConfig,
uploader=MongoDBUploader,
uploader_config=MongoDBUploaderConfig,
upload_stager=MongoDBUploadStager,
upload_stager_config=MongoDBUploadStagerConfig,
),
mongodb_destination_entry = DestinationRegistryEntry(
connection_config=MongoDBConnectionConfig,
uploader=MongoDBUploader,
uploader_config=MongoDBUploaderConfig,
upload_stager=MongoDBUploadStager,
upload_stager_config=MongoDBUploadStagerConfig,
)

View File

@ -20,7 +20,6 @@ from unstructured.ingest.v2.interfaces import (
from unstructured.ingest.v2.logger import logger
from unstructured.ingest.v2.processes.connector_registry import (
DestinationRegistryEntry,
add_destination_entry,
)
from unstructured.staging.base import flatten_dict
from unstructured.utils import requires_dependencies
@ -170,13 +169,10 @@ class PineconeUploader(Uploader):
)
add_destination_entry(
destination_type=CONNECTOR_TYPE,
entry=DestinationRegistryEntry(
connection_config=PineconeConnectionConfig,
uploader=PineconeUploader,
uploader_config=PineconeUploaderConfig,
upload_stager=PineconeUploadStager,
upload_stager_config=PineconeUploadStagerConfig,
),
pinecone_destination_entry = DestinationRegistryEntry(
connection_config=PineconeConnectionConfig,
uploader=PineconeUploader,
uploader_config=PineconeUploaderConfig,
upload_stager=PineconeUploadStager,
upload_stager_config=PineconeUploadStagerConfig,
)

View File

@ -24,7 +24,6 @@ from unstructured.ingest.v2.interfaces import (
from unstructured.ingest.v2.logger import logger
from unstructured.ingest.v2.processes.connector_registry import (
SourceRegistryEntry,
add_source_entry,
)
from unstructured.utils import requires_dependencies
@ -403,13 +402,10 @@ class SharepointDownloader(Downloader):
return self.get_site_page(file_data=file_data)
add_source_entry(
source_type=CONNECTOR_TYPE,
entry=SourceRegistryEntry(
connection_config=SharepointConnectionConfig,
indexer_config=SharepointIndexerConfig,
indexer=SharepointIndexer,
downloader_config=SharepointDownloaderConfig,
downloader=SharepointDownloader,
),
sharepoint_source_entry = SourceRegistryEntry(
connection_config=SharepointConnectionConfig,
indexer_config=SharepointIndexerConfig,
indexer=SharepointIndexer,
downloader_config=SharepointDownloaderConfig,
downloader=SharepointDownloader,
)

View File

@ -24,7 +24,6 @@ from unstructured.ingest.v2.interfaces import (
from unstructured.ingest.v2.logger import logger
from unstructured.ingest.v2.processes.connector_registry import (
DestinationRegistryEntry,
add_destination_entry,
)
from unstructured.utils import requires_dependencies
@ -152,13 +151,10 @@ class SingleStoreUploader(Uploader):
self.upload_csv(content=content)
add_destination_entry(
destination_type=CONNECTOR_TYPE,
entry=DestinationRegistryEntry(
connection_config=SingleStoreConnectionConfig,
uploader=SingleStoreUploader,
uploader_config=SingleStoreUploaderConfig,
upload_stager=SingleStoreUploadStager,
upload_stager_config=SingleStoreUploadStagerConfig,
),
singlestore_destination_entry = DestinationRegistryEntry(
connection_config=SingleStoreConnectionConfig,
uploader=SingleStoreUploader,
uploader_config=SingleStoreUploaderConfig,
upload_stager=SingleStoreUploadStager,
upload_stager_config=SingleStoreUploadStagerConfig,
)