From 4dad80e2feac0bc859480aecde32faf00fca276e Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Tue, 28 Mar 2023 17:52:47 +0200 Subject: [PATCH] S3 Improvements for test connection and connector status (#10795) * Handle test connection for S3 and new status * Remove exception * Update endpoint * Update endpoint * Update endpoint * chore: remove toast messages for test connection * chore: change objectstoreService to objectStoreService * minor fix * Use the default service param for listing filters * Format * Format * address comments --------- Co-authored-by: Sachin Chaurasiya --- .../src/metadata/ingestion/ometa/ometa_api.py | 2 +- .../source/objectstore/objectstore_service.py | 21 +++------------ .../source/objectstore/s3/metadata.py | 27 ++++++++++--------- .../objectstores/ContainerResource.java | 6 ++--- .../ObjectStoreServiceResource.java | 4 +-- .../converter/ClassConverterFactory.java | 12 ++++----- ...erviceConnectionRequestClassConverter.java | 4 ++- .../ObjectStoreServiceResourceTest.java | 2 +- .../automations/testServiceConnection.json | 3 +++ .../entity/services/objectstoreService.json | 18 ++----------- .../ServiceConfig/ConnectionConfigForm.tsx | 2 +- .../common/TestConnection/TestConnection.tsx | 10 ------- .../ui/src/constants/Services.constant.ts | 21 +++++++++++---- .../resources/ui/src/enums/entity.enum.ts | 2 +- .../resources/ui/src/enums/service.enum.ts | 2 +- .../resources/ui/src/interface/types.d.ts | 2 +- .../ui/src/pages/service/index.test.tsx | 2 +- .../ui/src/utils/ObjectStoreServiceUtils.ts | 14 +--------- .../resources/ui/src/utils/ServiceUtils.tsx | 27 +++---------------- 19 files changed, 66 insertions(+), 115 deletions(-) diff --git a/ingestion/src/metadata/ingestion/ometa/ometa_api.py b/ingestion/src/metadata/ingestion/ometa/ometa_api.py index 64c7e18854a..a1766084611 100644 --- a/ingestion/src/metadata/ingestion/ometa/ometa_api.py +++ b/ingestion/src/metadata/ingestion/ometa/ometa_api.py @@ -437,7 +437,7 @@ class OpenMetadata( ] ), ): - return "/services/objectstoreServices" + return "/services/objectStoreServices" if issubclass( entity, diff --git a/ingestion/src/metadata/ingestion/source/objectstore/objectstore_service.py b/ingestion/src/metadata/ingestion/source/objectstore/objectstore_service.py index be3a36e10b3..ff5604c49b4 100644 --- a/ingestion/src/metadata/ingestion/source/objectstore/objectstore_service.py +++ b/ingestion/src/metadata/ingestion/source/objectstore/objectstore_service.py @@ -75,27 +75,12 @@ class ObjectStoreServiceTopology(ServiceTopology): ) -class ObjectStoreSourceStatus(SourceStatus): - """ - Reports the source status after ingestion - """ - - def scanned(self, record: str) -> None: - self.success.append(record) - logger.debug(f"Scanned: {record}") - - def filter(self, key: str, reason: str) -> None: - self.filtered.append(key) - logger.debug(f"Filtered {key}: {reason}") - - class ObjectStoreServiceSource(TopologyRunnerMixin, Source, ABC): """ Base class for Object Store Services. It implements the topology and context. """ - status: ObjectStoreSourceStatus source_config: ObjectStoreServiceMetadataPipeline config: WorkflowSource metadata: OpenMetadata @@ -119,8 +104,10 @@ class ObjectStoreServiceSource(TopologyRunnerMixin, Source, ABC): self.config.sourceConfig.config ) self.connection: S3ObjectStoreClient = get_connection(self.service_connection) + + # Flag the connection for the test connection + self.connection_obj = self.connection self.test_connection() - self.status = ObjectStoreSourceStatus() @abstractmethod def get_containers(self) -> Iterable[Any]: @@ -148,7 +135,7 @@ class ObjectStoreServiceSource(TopologyRunnerMixin, Source, ABC): def test_connection(self) -> None: test_connection_fn = get_test_connection_fn(self.service_connection) - test_connection_fn(self.connection) + test_connection_fn(self.metadata, self.connection_obj, self.service_connection) def yield_create_request_objectstore_service(self, config: WorkflowSource): yield self.metadata.get_create_service_from_source( diff --git a/ingestion/src/metadata/ingestion/source/objectstore/s3/metadata.py b/ingestion/src/metadata/ingestion/source/objectstore/s3/metadata.py index d50eb3073f5..93ca1e115e7 100644 --- a/ingestion/src/metadata/ingestion/source/objectstore/s3/metadata.py +++ b/ingestion/src/metadata/ingestion/source/objectstore/s3/metadata.py @@ -126,8 +126,9 @@ class S3Source(ObjectStoreServiceSource): def get_containers(self) -> Iterable[S3ContainerDetails]: bucket_results = self.fetch_buckets() - try: - for bucket_response in bucket_results: + + for bucket_response in bucket_results: + try: metadata_config = self._load_metadata_file( bucket_name=bucket_response.name ) @@ -152,16 +153,18 @@ class S3Source(ObjectStoreServiceSource): yield self._generate_unstructured_container( bucket_response=bucket_response ) - except ValidationError as err: - logger.debug(traceback.format_exc()) - logger.warning( - f"Validation error while creating Container from bucket details - {err}" - ) - except Exception as err: - logger.debug(traceback.format_exc()) - logger.warning( - f"Wild error while creating Container from bucket details - {err}" - ) + except ValidationError as err: + error = f"Validation error while creating Container from bucket details - {err}" + logger.debug(traceback.format_exc()) + logger.warning(error) + self.status.failed(bucket_response.name, error, traceback.format_exc()) + except Exception as err: + error = ( + f"Wild error while creating Container from bucket details - {err}" + ) + logger.debug(traceback.format_exc()) + logger.warning(error) + self.status.failed(bucket_response.name, error, traceback.format_exc()) def yield_create_container_requests( self, container_details: S3ContainerDetails diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/objectstores/ContainerResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/objectstores/ContainerResource.java index aa5befeac30..f77b7c85e52 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/objectstores/ContainerResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/objectstores/ContainerResource.java @@ -112,8 +112,8 @@ public class ContainerResource extends EntityResource { - public static final String COLLECTION_PATH = "v1/services/objectstoreServices/"; + public static final String COLLECTION_PATH = "v1/services/objectStoreServices/"; static final String FIELDS = "pipelines,owner,tags"; @Override diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/ClassConverterFactory.java b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/ClassConverterFactory.java index 431b57c5995..a5702cf6451 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/ClassConverterFactory.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/ClassConverterFactory.java @@ -42,16 +42,16 @@ public class ClassConverterFactory { new HashMap<>() { { put(AirflowConnection.class, new AirflowConnectionClassConverter()); + put(BigQueryConnection.class, new BigQueryConnectionClassConverter()); put(DatalakeConnection.class, new DatalakeConnectionClassConverter()); + put(DbtGCSConfig.class, new DbtGCSConfigClassConverter()); put(DbtPipeline.class, new DbtPipelineClassConverter()); + put(GCSConfig.class, new GCSConfigClassConverter()); + put(GCSCredentials.class, new GcsCredentialsClassConverter()); + put(GcsConnection.class, new GcsConnectionClassConverter()); + put(OpenMetadataConnection.class, new OpenMetadataConnectionClassConverter()); put(SSOAuthMechanism.class, new SSOAuthMechanismClassConverter()); put(SupersetConnection.class, new SupersetConnectionClassConverter()); - put(GCSCredentials.class, new GcsCredentialsClassConverter()); - put(OpenMetadataConnection.class, new OpenMetadataConnectionClassConverter()); - put(GcsConnection.class, new GcsConnectionClassConverter()); - put(GCSConfig.class, new GCSConfigClassConverter()); - put(BigQueryConnection.class, new BigQueryConnectionClassConverter()); - put(DbtGCSConfig.class, new DbtGCSConfigClassConverter()); put(TestServiceConnectionRequest.class, new TestServiceConnectionRequestClassConverter()); put(Workflow.class, new WorkflowClassConverter()); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/TestServiceConnectionRequestClassConverter.java b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/TestServiceConnectionRequestClassConverter.java index 6431815cc13..2dde0b462fc 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/TestServiceConnectionRequestClassConverter.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/TestServiceConnectionRequestClassConverter.java @@ -21,6 +21,7 @@ import org.openmetadata.schema.entity.services.MetadataConnection; import org.openmetadata.schema.type.DashboardConnection; import org.openmetadata.schema.type.MessagingConnection; import org.openmetadata.schema.type.MlModelConnection; +import org.openmetadata.schema.type.ObjectStoreConnection; import org.openmetadata.schema.type.PipelineConnection; import org.openmetadata.service.exception.InvalidServiceConnectionException; import org.openmetadata.service.util.JsonUtils; @@ -36,7 +37,8 @@ public class TestServiceConnectionRequestClassConverter extends ClassConverter { MessagingConnection.class, PipelineConnection.class, MlModelConnection.class, - MetadataConnection.class); + MetadataConnection.class, + ObjectStoreConnection.class); public TestServiceConnectionRequestClassConverter() { super(TestServiceConnectionRequest.class); diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/services/ObjectStoreServiceResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/services/ObjectStoreServiceResourceTest.java index cdd3e041734..a0cae91eda6 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/services/ObjectStoreServiceResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/services/ObjectStoreServiceResourceTest.java @@ -39,7 +39,7 @@ public class ObjectStoreServiceResourceTest extends EntityResourceTest = ({ setTestStatus(StatusType.Failed); setMessage(failureMessage); setIsTestingConnection(false); - showErrorToast(failureMessage); return; } @@ -217,11 +211,9 @@ const TestConnection: FC = ({ } if (isTestConnectionSuccess) { - showSuccessToast(successMessage); setTestStatus(StatusType.Successful); setMessage(successMessage); } else { - showErrorToast(failureMessage); setTestStatus(StatusType.Failed); setMessage(failureMessage); } @@ -249,14 +241,12 @@ const TestConnection: FC = ({ if (!isWorkflowCompleted) { setMessage(infoMessage); - showInfoToast(infoMessage); } setIsTestingConnection(false); }, FETCHING_EXPIRY_TIME); } catch (error) { clearInterval(intervalId); - showErrorToast(error as AxiosError); setIsTestingConnection(false); setMessage(failureMessage); setTestStatus(StatusType.Failed); diff --git a/openmetadata-ui/src/main/resources/ui/src/constants/Services.constant.ts b/openmetadata-ui/src/main/resources/ui/src/constants/Services.constant.ts index ab445dcd71d..db28f74b50d 100644 --- a/openmetadata-ui/src/main/resources/ui/src/constants/Services.constant.ts +++ b/openmetadata-ui/src/main/resources/ui/src/constants/Services.constant.ts @@ -15,7 +15,8 @@ import amazonS3 from 'assets/img/service-icon-amazon-s3.svg'; import gcs from 'assets/img/service-icon-gcs.png'; import msAzure from 'assets/img/service-icon-ms-azure.png'; import { WorkflowStatus } from 'generated/entity/automations/workflow'; -import { ObjectStoreServiceType } from 'generated/entity/services/objectstoreService'; +import { ObjectStoreServiceType } from 'generated/entity/data/container'; +import { ServiceType } from 'generated/entity/services/serviceType'; import { map, startCase } from 'lodash'; import { ServiceTypes } from 'Models'; import i18n from 'utils/i18next/LocalUtil'; @@ -180,7 +181,7 @@ export const serviceTypes: Record> = { metadataServices: (Object.values(MetadataServiceType) as string[]).sort( customServiceComparator ), - objectstoreServices: (Object.values(ObjectStoreServiceType) as string[]).sort( + objectStoreServices: (Object.values(ObjectStoreServiceType) as string[]).sort( customServiceComparator ), }; @@ -191,7 +192,7 @@ export const arrServiceTypes: Array = [ 'dashboardServices', 'pipelineServices', 'mlmodelServices', - 'objectstoreServices', + 'objectStoreServices', ]; export const SERVICE_CATEGORY: { [key: string]: ServiceCategory } = { @@ -211,7 +212,7 @@ export const SERVICE_CATEGORY_TYPE = { pipelineServices: 'pipelines', mlmodelServices: 'mlModels', metadataServices: 'metadata', - objectstoreServices: 'objectStores', + objectStoreServices: 'objectStores', }; export const servicesDisplayName: { [key: string]: string } = { @@ -233,7 +234,7 @@ export const servicesDisplayName: { [key: string]: string } = { metadataServices: i18n.t('label.entity-service', { entity: i18n.t('label.metadata'), }), - objectstoreServices: i18n.t('label.entity-service', { + objectStoreServices: i18n.t('label.entity-service', { entity: i18n.t('label.object-store'), }), }; @@ -271,3 +272,13 @@ export const WORKFLOW_COMPLETE_STATUS = [ WorkflowStatus.Failed, WorkflowStatus.Successful, ]; + +export const SERVICE_TYPE_MAP = { + [ServiceCategory.DASHBOARD_SERVICES]: ServiceType.Dashboard, + [ServiceCategory.DATABASE_SERVICES]: ServiceType.Database, + [ServiceCategory.MESSAGING_SERVICES]: ServiceType.Messaging, + [ServiceCategory.ML_MODEL_SERVICES]: ServiceType.MlModel, + [ServiceCategory.METADATA_SERVICES]: ServiceType.Metadata, + [ServiceCategory.OBJECT_STORE_SERVICES]: ServiceType.ObjectStore, + [ServiceCategory.PIPELINE_SERVICES]: ServiceType.Pipeline, +}; diff --git a/openmetadata-ui/src/main/resources/ui/src/enums/entity.enum.ts b/openmetadata-ui/src/main/resources/ui/src/enums/entity.enum.ts index 12a4c87ba80..e961baa3977 100644 --- a/openmetadata-ui/src/main/resources/ui/src/enums/entity.enum.ts +++ b/openmetadata-ui/src/main/resources/ui/src/enums/entity.enum.ts @@ -27,7 +27,7 @@ export enum EntityType { DASHBOARD_SERVICE = 'dashboardService', PIPELINE_SERVICE = 'pipelineService', MLMODEL_SERVICE = 'mlmodelService', - OBJECT_STORE_SERVICE = 'objectstoreService', + OBJECT_STORE_SERVICE = 'objectStoreService', WEBHOOK = 'webhook', MLMODEL = 'mlmodel', TYPE = 'type', diff --git a/openmetadata-ui/src/main/resources/ui/src/enums/service.enum.ts b/openmetadata-ui/src/main/resources/ui/src/enums/service.enum.ts index e24858991cd..9c33c8508bc 100644 --- a/openmetadata-ui/src/main/resources/ui/src/enums/service.enum.ts +++ b/openmetadata-ui/src/main/resources/ui/src/enums/service.enum.ts @@ -18,7 +18,7 @@ export enum ServiceCategory { PIPELINE_SERVICES = 'pipelineServices', ML_MODEL_SERVICES = 'mlmodelServices', METADATA_SERVICES = 'metadataServices', - OBJECT_STORE_SERVICES = 'objectstoreServices', + OBJECT_STORE_SERVICES = 'objectStoreServices', } export enum IngestionType { diff --git a/openmetadata-ui/src/main/resources/ui/src/interface/types.d.ts b/openmetadata-ui/src/main/resources/ui/src/interface/types.d.ts index cd937f1d436..2e5e7cfb6be 100644 --- a/openmetadata-ui/src/main/resources/ui/src/interface/types.d.ts +++ b/openmetadata-ui/src/main/resources/ui/src/interface/types.d.ts @@ -140,7 +140,7 @@ declare module 'Models' { | 'pipelineServices' | 'mlmodelServices' | 'metadataServices' - | 'objectstoreServices'; + | 'objectStoreServices'; export type SearchDataFunctionType = { queryString: string; diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/service/index.test.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/service/index.test.tsx index 3e0c1279561..8b582140b02 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/service/index.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/service/index.test.tsx @@ -445,7 +445,7 @@ describe('Test ServicePage Component', () => { mockParams = { serviceFQN: 's3_object_store_sample', serviceType: 'S3', - serviceCategory: 'objectstoreServices', + serviceCategory: 'objectStoreServices', tab: 'containers', }; diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/ObjectStoreServiceUtils.ts b/openmetadata-ui/src/main/resources/ui/src/utils/ObjectStoreServiceUtils.ts index 33858f4d36b..c2f4d433a1a 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/ObjectStoreServiceUtils.ts +++ b/openmetadata-ui/src/main/resources/ui/src/utils/ObjectStoreServiceUtils.ts @@ -11,9 +11,7 @@ * limitations under the License. */ import { COMMON_UI_SCHEMA } from 'constants/Services.constant'; -import { ObjectStoreServiceType } from 'generated/entity/services/objectstoreService'; -import azureConnection from 'jsons/connectionSchemas/connections/objectstore/azureObjectStoreConnection.json'; -import gcsConnection from 'jsons/connectionSchemas/connections/objectstore/gcsObjectStoreConnection.json'; +import { ObjectStoreServiceType } from 'generated/entity/data/container'; import s3Connection from 'jsons/connectionSchemas/connections/objectstore/s3ObjectStoreConnection.json'; import { cloneDeep } from 'lodash'; @@ -21,19 +19,9 @@ export const getObjectStoreConfig = (type: ObjectStoreServiceType) => { let schema = {}; const uiSchema = { ...COMMON_UI_SCHEMA }; switch (type) { - case ObjectStoreServiceType.Azure: { - schema = azureConnection; - - break; - } case ObjectStoreServiceType.S3: { schema = s3Connection; - break; - } - case ObjectStoreServiceType.Gcs: { - schema = gcsConnection; - break; } } diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/ServiceUtils.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/ServiceUtils.tsx index bd9831f66da..57203ae9bd5 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/ServiceUtils.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/ServiceUtils.tsx @@ -58,7 +58,6 @@ import { DRUID, DYNAMODB, FIVETRAN, - GCS, GLUE, HIVE, IBMDB2, @@ -71,7 +70,6 @@ import { MLFLOW, MODE, MSSQL, - MS_AZURE, MYSQL, NIFI, ORACLE, @@ -89,6 +87,7 @@ import { SALESFORCE, SCIKIT, serviceTypes, + SERVICE_TYPE_MAP, SINGLESTORE, SNOWFLAKE, SQLITE, @@ -121,7 +120,6 @@ import { PipelineService, PipelineServiceType, } from '../generated/entity/services/pipelineService'; -import { ServiceType } from '../generated/entity/services/serviceType'; import { ServicesType } from '../interface/service.interface'; import { getEntityDeleteMessage, pluralize } from './CommonUtils'; import { getDashboardURL } from './DashboardServiceUtils'; @@ -287,15 +285,9 @@ export const serviceTypeLogo = (type: string) => { case MetadataServiceType.OpenMetadata: return LOGO; - case ObjectStoreServiceType.Azure: - return MS_AZURE; - case ObjectStoreServiceType.S3: return AMAZON_S3; - case ObjectStoreServiceType.Gcs: - return GCS; - default: { let logo; if (serviceTypes.messagingServices.includes(type)) { @@ -612,19 +604,8 @@ export const shouldTestConnection = (serviceType: string) => { ); }; -export const getTestConnectionType = (serviceCat: ServiceCategory) => { - switch (serviceCat) { - case ServiceCategory.MESSAGING_SERVICES: - return ServiceType.Messaging; - case ServiceCategory.DASHBOARD_SERVICES: - return ServiceType.Dashboard; - case ServiceCategory.PIPELINE_SERVICES: - return ServiceType.Pipeline; - case ServiceCategory.DATABASE_SERVICES: - default: - return ServiceType.Database; - } -}; +export const getTestConnectionType = (serviceCat: ServiceCategory) => + SERVICE_TYPE_MAP[serviceCat]; export const getServiceCreatedLabel = (serviceCategory: ServiceCategory) => { let serviceCat; @@ -847,7 +828,7 @@ export const getServiceRouteFromServiceType = (type: ServiceTypes) => { if (type === 'metadataServices') { return GlobalSettingOptions.METADATA; } - if (type === 'objectstoreServices') { + if (type === 'objectStoreServices') { return GlobalSettingOptions.OBJECT_STORES; }