diff --git a/ingestion/src/metadata/ingestion/ometa/ometa_api.py b/ingestion/src/metadata/ingestion/ometa/ometa_api.py index 4468b94a220..d614629fce2 100644 --- a/ingestion/src/metadata/ingestion/ometa/ometa_api.py +++ b/ingestion/src/metadata/ingestion/ometa/ometa_api.py @@ -17,65 +17,12 @@ working with OpenMetadata entities. import traceback from typing import Dict, Generic, Iterable, List, Optional, Type, TypeVar, Union -try: - from typing import get_args -except ImportError: - from typing_compat import get_args - from pydantic import BaseModel from requests.utils import quote -from metadata.generated.schema.analytics.webAnalyticEventData import ( - WebAnalyticEventData, -) -from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest -from metadata.generated.schema.dataInsight.dataInsightChart import DataInsightChart -from metadata.generated.schema.dataInsight.kpi.kpi import Kpi -from metadata.generated.schema.entity.automations.workflow import Workflow -from metadata.generated.schema.entity.classification.classification import ( - Classification, -) -from metadata.generated.schema.entity.classification.tag import Tag -from metadata.generated.schema.entity.data.chart import Chart -from metadata.generated.schema.entity.data.container import Container -from metadata.generated.schema.entity.data.dashboard import Dashboard -from metadata.generated.schema.entity.data.dashboardDataModel import DashboardDataModel -from metadata.generated.schema.entity.data.database import Database -from metadata.generated.schema.entity.data.databaseSchema import DatabaseSchema -from metadata.generated.schema.entity.data.glossary import Glossary -from metadata.generated.schema.entity.data.glossaryTerm import GlossaryTerm -from metadata.generated.schema.entity.data.metrics import Metrics -from metadata.generated.schema.entity.data.mlmodel import MlModel -from metadata.generated.schema.entity.data.pipeline import Pipeline -from metadata.generated.schema.entity.data.query import Query -from metadata.generated.schema.entity.data.report import Report -from metadata.generated.schema.entity.data.searchIndex import SearchIndex -from metadata.generated.schema.entity.data.table import Table -from metadata.generated.schema.entity.data.topic import Topic -from metadata.generated.schema.entity.policies.policy import Policy from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import ( OpenMetadataConnection, ) -from metadata.generated.schema.entity.services.connections.testConnectionDefinition import ( - TestConnectionDefinition, -) -from metadata.generated.schema.entity.services.dashboardService import DashboardService -from metadata.generated.schema.entity.services.databaseService import DatabaseService -from metadata.generated.schema.entity.services.ingestionPipelines.ingestionPipeline import ( - IngestionPipeline, -) -from metadata.generated.schema.entity.services.messagingService import MessagingService -from metadata.generated.schema.entity.services.metadataService import MetadataService -from metadata.generated.schema.entity.services.mlmodelService import MlModelService -from metadata.generated.schema.entity.services.pipelineService import PipelineService -from metadata.generated.schema.entity.services.searchService import SearchService -from metadata.generated.schema.entity.services.storageService import StorageService -from metadata.generated.schema.entity.teams.role import Role -from metadata.generated.schema.entity.teams.team import Team -from metadata.generated.schema.entity.teams.user import AuthenticationMechanism, User -from metadata.generated.schema.tests.testCase import TestCase -from metadata.generated.schema.tests.testDefinition import TestDefinition -from metadata.generated.schema.tests.testSuite import TestSuite from metadata.generated.schema.type import basic from metadata.generated.schema.type.basic import FullyQualifiedEntityName from metadata.generated.schema.type.entityHistory import EntityVersionHistory @@ -108,6 +55,7 @@ from metadata.ingestion.ometa.provider_registry import ( InvalidAuthProviderException, auth_provider_registry, ) +from metadata.ingestion.ometa.routes import ROUTES from metadata.ingestion.ometa.utils import get_entity_type, model_str from metadata.utils.logger import ometa_logger from metadata.utils.secrets.secrets_manager_factory import SecretsManagerFactory @@ -227,262 +175,20 @@ class OpenMetadata( if self.config.enableVersionValidation: self.validate_versions() - def get_suffix(self, entity: Type[T]) -> str: # pylint: disable=R0911,R0912 + @staticmethod + def get_suffix(entity: Type[T]) -> str: """ Given an entity Type from the generated sources, return the endpoint to run requests. - - Might be interesting to follow a more strict - and type-checked approach - - Disabled pylint R0911: too-many-return-statements - Disabled pylint R0912: too-many-branches """ - # Entity Schemas - if issubclass( - entity, get_args(Union[MlModel, self.get_create_entity_type(MlModel)]) - ): - return "/mlmodels" + route = ROUTES.get(entity.__name__) + if route is None: + raise MissingEntityTypeException( + f"Missing {entity} type when generating suffixes" + ) - if issubclass( - entity, get_args(Union[Chart, self.get_create_entity_type(Chart)]) - ): - return "/charts" - - if issubclass( - entity, - get_args( - Union[ - DashboardDataModel, self.get_create_entity_type(DashboardDataModel) - ] - ), - ): - return "/dashboard/datamodels" - - if issubclass( - entity, get_args(Union[Dashboard, self.get_create_entity_type(Dashboard)]) - ): - return "/dashboards" - - if issubclass( - entity, get_args(Union[Database, self.get_create_entity_type(Database)]) - ): - return "/databases" - - if issubclass( - entity, - get_args( - Union[DatabaseSchema, self.get_create_entity_type(DatabaseSchema)] - ), - ): - return "/databaseSchemas" - - if issubclass( - entity, get_args(Union[Pipeline, self.get_create_entity_type(Pipeline)]) - ): - return "/pipelines" - - if issubclass( - entity, get_args(Union[Policy, self.get_create_entity_type(Policy)]) - ): - return "/policies" - - if issubclass( - entity, get_args(Union[Table, self.get_create_entity_type(Table)]) - ): - return "/tables" - - if issubclass( - entity, get_args(Union[Topic, self.get_create_entity_type(Topic)]) - ): - return "/topics" - - if issubclass(entity, Metrics): - return "/metrics" - - if issubclass(entity, AddLineageRequest): - return "/lineage" - - if issubclass(entity, Report): - return "/reports" - - if issubclass(entity, AuthenticationMechanism): - return "/users/auth-mechanism" - - if issubclass( - entity, - get_args( - Union[ - Tag, - self.get_create_entity_type(Tag), - ] - ), - ): - return "/tags" - - if issubclass( - entity, - get_args( - Union[ - Classification, - self.get_create_entity_type(Classification), - ] - ), - ): - return "/classifications" - - if issubclass( - entity, get_args(Union[Glossary, self.get_create_entity_type(Glossary)]) - ): - return "/glossaries" - - if issubclass( - entity, - get_args(Union[GlossaryTerm, self.get_create_entity_type(GlossaryTerm)]), - ): - return "/glossaryTerms" - - if issubclass(entity, get_args(Union[Role, self.get_create_entity_type(Role)])): - return "/roles" - - if issubclass( - entity, get_args(Union[Query, self.get_create_entity_type(Query)]) - ): - return "/queries" - - if issubclass(entity, get_args(Union[Team, self.get_create_entity_type(Team)])): - return "/teams" - - if issubclass(entity, get_args(Union[User, self.get_create_entity_type(User)])): - return "/users" - - if issubclass( - entity, get_args(Union[Container, self.get_create_entity_type(Container)]) - ): - return "/containers" - - if issubclass( - entity, - get_args(Union[SearchIndex, self.get_create_entity_type(SearchIndex)]), - ): - return "/searchIndexes" - - if issubclass( - entity, get_args(Union[Workflow, self.get_create_entity_type(Workflow)]) - ): - return "/automations/workflows" - - # Services Schemas - if issubclass( - entity, - get_args( - Union[DatabaseService, self.get_create_entity_type(DatabaseService)] - ), - ): - return "/services/databaseServices" - - if issubclass( - entity, - get_args( - Union[DashboardService, self.get_create_entity_type(DashboardService)] - ), - ): - return "/services/dashboardServices" - - if issubclass( - entity, - get_args( - Union[MessagingService, self.get_create_entity_type(MessagingService)] - ), - ): - return "/services/messagingServices" - - if issubclass( - entity, - get_args( - Union[PipelineService, self.get_create_entity_type(PipelineService)] - ), - ): - return "/services/pipelineServices" - - if issubclass( - entity, - get_args( - Union[StorageService, self.get_create_entity_type(StorageService)] - ), - ): - return "/services/storageServices" - - if issubclass( - entity, - get_args( - Union[MlModelService, self.get_create_entity_type(MlModelService)] - ), - ): - return "/services/mlmodelServices" - - if issubclass( - entity, - get_args( - Union[MetadataService, self.get_create_entity_type(MetadataService)] - ), - ): - return "/services/metadataServices" - - if issubclass( - entity, - get_args(Union[SearchService, self.get_create_entity_type(SearchService)]), - ): - return "/services/searchServices" - - if issubclass( - entity, - IngestionPipeline, - ): - return "/services/ingestionPipelines" - - if issubclass( - entity, - TestConnectionDefinition, - ): - return "/services/testConnectionDefinitions" - - if issubclass( - entity, - get_args( - Union[TestDefinition, self.get_create_entity_type(TestDefinition)] - ), - ): - return "/dataQuality/testDefinitions" - - if issubclass( - entity, - get_args(Union[TestSuite, self.get_create_entity_type(TestSuite)]), - ): - return "/dataQuality/testSuites" - - if issubclass( - entity, - get_args(Union[TestCase, self.get_create_entity_type(TestCase)]), - ): - return "/dataQuality/testCases" - - if issubclass(entity, WebAnalyticEventData): - return "/analytics/web/events/collect" - - if issubclass(entity, DataInsightChart): - return "/analytics/dataInsights/charts" - - if issubclass( - entity, - Kpi, - ): - return "/kpi" - - raise MissingEntityTypeException( - f"Missing {entity} type when generating suffixes" - ) + return route def get_module_path(self, entity: Type[T]) -> str: """ @@ -499,7 +205,6 @@ class OpenMetadata( def get_create_entity_type(self, entity: Type[T]) -> Type[C]: """ imports and returns the Create Type from an Entity Type T. - We are following the expected path structure to import on-the-fly the necessary class and pass it to the consumer """ @@ -591,6 +296,7 @@ class OpenMetadata( entity: Type[T], fqn: Union[str, FullyQualifiedEntityName], fields: Optional[List[str]] = None, + nullable: bool = True, ) -> Optional[T]: """ Return entity by name or None @@ -600,6 +306,7 @@ class OpenMetadata( entity=entity, path=f"name/{quote(model_str(fqn), safe='')}", fields=fields, + nullable=nullable, ) def get_by_id( @@ -607,14 +314,24 @@ class OpenMetadata( entity: Type[T], entity_id: Union[str, basic.Uuid], fields: Optional[List[str]] = None, + nullable: bool = True, ) -> Optional[T]: """ Return entity by ID or None """ - return self._get(entity=entity, path=model_str(entity_id), fields=fields) + return self._get( + entity=entity, + path=model_str(entity_id), + fields=fields, + nullable=nullable, + ) def _get( - self, entity: Type[T], path: str, fields: Optional[List[str]] = None + self, + entity: Type[T], + path: str, + fields: Optional[List[str]] = None, + nullable: bool = True, ) -> Optional[T]: """ Generic GET operation for an entity @@ -631,6 +348,12 @@ class OpenMetadata( ) return entity(**resp) except APIError as err: + # We can expect some GET calls to return us a None and manage it in following steps. + # No need to pollute the logs in these cases. + if err.code == 404 and nullable: + return None + + # Any other API errors will be passed to the client logger.debug(traceback.format_exc()) logger.debug( "GET %s for %s. Error %s - %s", @@ -639,7 +362,7 @@ class OpenMetadata( err.status_code, err, ) - return None + raise err def get_entity_reference( self, entity: Type[T], fqn: str @@ -667,7 +390,7 @@ class OpenMetadata( self, entity: Type[T], fields: Optional[List[str]] = None, - after: str = None, + after: Optional[str] = None, limit: int = 100, params: Optional[Dict[str, str]] = None, ) -> EntityList[T]: diff --git a/ingestion/src/metadata/ingestion/ometa/routes.py b/ingestion/src/metadata/ingestion/ometa/routes.py new file mode 100644 index 00000000000..b57b8321491 --- /dev/null +++ b/ingestion/src/metadata/ingestion/ometa/routes.py @@ -0,0 +1,207 @@ +# Copyright 2021 Collate +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +OMeta API endpoints +""" +from metadata.generated.schema.analytics.webAnalyticEventData import ( + WebAnalyticEventData, +) +from metadata.generated.schema.api.automations.createWorkflow import ( + CreateWorkflowRequest, +) +from metadata.generated.schema.api.classification.createClassification import ( + CreateClassificationRequest, +) +from metadata.generated.schema.api.classification.createTag import CreateTagRequest +from metadata.generated.schema.api.data.createChart import CreateChartRequest +from metadata.generated.schema.api.data.createContainer import CreateContainerRequest +from metadata.generated.schema.api.data.createDashboard import CreateDashboardRequest +from metadata.generated.schema.api.data.createDashboardDataModel import ( + CreateDashboardDataModelRequest, +) +from metadata.generated.schema.api.data.createDatabase import CreateDatabaseRequest +from metadata.generated.schema.api.data.createDatabaseSchema import ( + CreateDatabaseSchemaRequest, +) +from metadata.generated.schema.api.data.createGlossary import CreateGlossaryRequest +from metadata.generated.schema.api.data.createGlossaryTerm import ( + CreateGlossaryTermRequest, +) +from metadata.generated.schema.api.data.createMlModel import CreateMlModelRequest +from metadata.generated.schema.api.data.createPipeline import CreatePipelineRequest +from metadata.generated.schema.api.data.createQuery import CreateQueryRequest +from metadata.generated.schema.api.data.createSearchIndex import ( + CreateSearchIndexRequest, +) +from metadata.generated.schema.api.data.createTable import CreateTableRequest +from metadata.generated.schema.api.data.createTopic import CreateTopicRequest +from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest +from metadata.generated.schema.api.policies.createPolicy import CreatePolicyRequest +from metadata.generated.schema.api.services.createDashboardService import ( + CreateDashboardServiceRequest, +) +from metadata.generated.schema.api.services.createDatabaseService import ( + CreateDatabaseServiceRequest, +) +from metadata.generated.schema.api.services.createMessagingService import ( + CreateMessagingServiceRequest, +) +from metadata.generated.schema.api.services.createMetadataService import ( + CreateMetadataServiceRequest, +) +from metadata.generated.schema.api.services.createMlModelService import ( + CreateMlModelServiceRequest, +) +from metadata.generated.schema.api.services.createPipelineService import ( + CreatePipelineServiceRequest, +) +from metadata.generated.schema.api.services.createSearchService import ( + CreateSearchServiceRequest, +) +from metadata.generated.schema.api.services.createStorageService import ( + CreateStorageServiceRequest, +) +from metadata.generated.schema.api.teams.createRole import CreateRoleRequest +from metadata.generated.schema.api.teams.createTeam import CreateTeamRequest +from metadata.generated.schema.api.teams.createUser import CreateUserRequest +from metadata.generated.schema.api.tests.createTestCase import CreateTestCaseRequest +from metadata.generated.schema.api.tests.createTestDefinition import ( + CreateTestDefinitionRequest, +) +from metadata.generated.schema.api.tests.createTestSuite import CreateTestSuiteRequest +from metadata.generated.schema.dataInsight.dataInsightChart import DataInsightChart +from metadata.generated.schema.dataInsight.kpi.kpi import Kpi +from metadata.generated.schema.entity.automations.workflow import Workflow +from metadata.generated.schema.entity.classification.classification import ( + Classification, +) +from metadata.generated.schema.entity.classification.tag import Tag +from metadata.generated.schema.entity.data.chart import Chart +from metadata.generated.schema.entity.data.container import Container +from metadata.generated.schema.entity.data.dashboard import Dashboard +from metadata.generated.schema.entity.data.dashboardDataModel import DashboardDataModel +from metadata.generated.schema.entity.data.database import Database +from metadata.generated.schema.entity.data.databaseSchema import DatabaseSchema +from metadata.generated.schema.entity.data.glossary import Glossary +from metadata.generated.schema.entity.data.glossaryTerm import GlossaryTerm +from metadata.generated.schema.entity.data.metrics import Metrics +from metadata.generated.schema.entity.data.mlmodel import MlModel +from metadata.generated.schema.entity.data.pipeline import Pipeline +from metadata.generated.schema.entity.data.query import Query +from metadata.generated.schema.entity.data.report import Report +from metadata.generated.schema.entity.data.searchIndex import SearchIndex +from metadata.generated.schema.entity.data.table import Table +from metadata.generated.schema.entity.data.topic import Topic +from metadata.generated.schema.entity.policies.policy import Policy +from metadata.generated.schema.entity.services.connections.testConnectionDefinition import ( + TestConnectionDefinition, +) +from metadata.generated.schema.entity.services.dashboardService import DashboardService +from metadata.generated.schema.entity.services.databaseService import DatabaseService +from metadata.generated.schema.entity.services.ingestionPipelines.ingestionPipeline import ( + IngestionPipeline, +) +from metadata.generated.schema.entity.services.messagingService import MessagingService +from metadata.generated.schema.entity.services.metadataService import MetadataService +from metadata.generated.schema.entity.services.mlmodelService import MlModelService +from metadata.generated.schema.entity.services.pipelineService import PipelineService +from metadata.generated.schema.entity.services.searchService import SearchService +from metadata.generated.schema.entity.services.storageService import StorageService +from metadata.generated.schema.entity.teams.role import Role +from metadata.generated.schema.entity.teams.team import Team +from metadata.generated.schema.entity.teams.user import AuthenticationMechanism, User +from metadata.generated.schema.tests.testCase import TestCase +from metadata.generated.schema.tests.testDefinition import TestDefinition +from metadata.generated.schema.tests.testSuite import TestSuite + +ROUTES = { + MlModel.__name__: "/mlmodels", + CreateMlModelRequest.__name__: "/mlmodels", + Chart.__name__: "/charts", + CreateChartRequest.__name__: "/charts", + DashboardDataModel.__name__: "/dashboard/datamodels", + CreateDashboardDataModelRequest.__name__: "/dashboard/datamodels", + Dashboard.__name__: "/dashboards", + CreateDashboardRequest.__name__: "/dashboards", + Database.__name__: "/databases", + CreateDatabaseRequest.__name__: "/databases", + DatabaseSchema.__name__: "/databaseSchemas", + CreateDatabaseSchemaRequest.__name__: "/databaseSchemas", + Pipeline.__name__: "/pipelines", + CreatePipelineRequest.__name__: "/pipelines", + Table.__name__: "/tables", + CreateTableRequest.__name__: "/tables", + Topic.__name__: "/topics", + CreateTopicRequest.__name__: "/topics", + Metrics.__name__: "/metrics", + AddLineageRequest.__name__: "/lineage", + Report.__name__: "/reports", + Query.__name__: "/queries", + CreateQueryRequest.__name__: "/queries", + Container.__name__: "/containers", + CreateContainerRequest.__name__: "/containers", + SearchIndex.__name__: "/searchIndexes", + CreateSearchIndexRequest.__name__: "/searchIndexes", + # Classifications + Tag.__name__: "/tags", + CreateTagRequest.__name__: "/tags", + Classification.__name__: "/classifications", + CreateClassificationRequest.__name__: "/classifications", + # Glossaries + Glossary.__name__: "/glossaries", + CreateGlossaryRequest.__name__: "/glossaries", + GlossaryTerm.__name__: "/glossaryTerms", + CreateGlossaryTermRequest.__name__: "/glossaryTerms", + # Users + Team.__name__: "/teams", + CreateTeamRequest.__name__: "/teams", + User.__name__: "/users", + CreateUserRequest.__name__: "/users", + AuthenticationMechanism.__name__: "/users/auth-mechanism", + # Roles + Role.__name__: "/roles", + CreateRoleRequest.__name__: "/roles", + Policy.__name__: "/policies", + CreatePolicyRequest.__name__: "/policies", + # Automations + Workflow.__name__: "/automations/workflows", + CreateWorkflowRequest.__name__: "/automations/workflows", + # Services + DatabaseService.__name__: "/services/databaseServices", + CreateDatabaseServiceRequest.__name__: "/services/databaseServices", + DashboardService.__name__: "/services/dashboardServices", + CreateDashboardServiceRequest.__name__: "/services/dashboardServices", + MessagingService.__name__: "/services/messagingServices", + CreateMessagingServiceRequest.__name__: "/services/messagingServices", + PipelineService.__name__: "/services/pipelineServices", + CreatePipelineServiceRequest.__name__: "/services/pipelineServices", + StorageService.__name__: "/services/storageServices", + CreateStorageServiceRequest.__name__: "/services/storageServices", + MlModelService.__name__: "/services/mlmodelServices", + CreateMlModelServiceRequest.__name__: "/services/mlmodelServices", + MetadataService.__name__: "/services/metadataServices", + CreateMetadataServiceRequest.__name__: "/services/metadataServices", + SearchService.__name__: "/services/searchServices", + CreateSearchServiceRequest.__name__: "/services/searchServices", + IngestionPipeline.__name__: "/services/ingestionPipelines", + TestConnectionDefinition.__name__: "/services/testConnectionDefinitions", + # Data Quality + TestDefinition.__name__: "/dataQuality/testDefinitions", + CreateTestDefinitionRequest.__name__: "/dataQuality/testDefinitions", + TestSuite.__name__: "/dataQuality/testSuites", + CreateTestSuiteRequest.__name__: "/dataQuality/testSuites", + TestCase.__name__: "/dataQuality/testCases", + CreateTestCaseRequest.__name__: "/dataQuality/testCases", + # Analytics + WebAnalyticEventData.__name__: "/analytics/web/events/collect", + DataInsightChart.__name__: "/analytics/dataInsights/charts", + Kpi.__name__: "/kpi", +} diff --git a/ingestion/tests/integration/ometa/test_ometa_table_api.py b/ingestion/tests/integration/ometa/test_ometa_table_api.py index 35a0a11d982..0439ddd31a1 100644 --- a/ingestion/tests/integration/ometa/test_ometa_table_api.py +++ b/ingestion/tests/integration/ometa/test_ometa_table_api.py @@ -210,6 +210,10 @@ class OMetaTableTest(TestCase): ) self.assertEqual(res.name, self.entity.name) + # Now check that we get a None if the table does not exist + nullable_res = self.metadata.get_by_name(entity=Table, fqn="something.made.up") + self.assertIsNone(nullable_res) + def test_get_id(self): """ We can fetch a Table by ID and get it back as Entity diff --git a/ingestion/tests/unit/test_ometa_endpoints.py b/ingestion/tests/unit/test_ometa_endpoints.py index 836e457adcf..b3cc71b2af1 100644 --- a/ingestion/tests/unit/test_ometa_endpoints.py +++ b/ingestion/tests/unit/test_ometa_endpoints.py @@ -105,20 +105,6 @@ class OMetaEndpointTest(TestCase): """ self.assertEqual(self.metadata.get_suffix(User), "/users") - def test_get_create_entity_type(self): - """ - Validate the mapping from Entity to CreateEntity - """ - - create = self.metadata.get_create_entity_type(Topic) - assert issubclass(create, CreateTopicRequest) - - create = self.metadata.get_create_entity_type(DatabaseService) - assert issubclass(create, CreateDatabaseServiceRequest) - - create = self.metadata.get_create_entity_type(User) - assert issubclass(create, CreateUserRequest) - def test_get_entity_from_create(self): """ Validate the mapping from CreateEntity to Entity diff --git a/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/common.py b/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/common.py index 27a927a188f..82c256bd90e 100644 --- a/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/common.py +++ b/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/common.py @@ -109,6 +109,9 @@ def build_source(ingestion_pipeline: IngestionPipeline) -> WorkflowSource: try: metadata = OpenMetadata(config=ingestion_pipeline.openMetadataServerConnection) + + # check we can access OM server + metadata.health_check() except Exception as exc: raise ClientInitializationError( f"Failed to initialize the OpenMetadata client due to: {exc}." @@ -122,8 +125,6 @@ def build_source(ingestion_pipeline: IngestionPipeline) -> WorkflowSource: entity_class = None try: if service_type == "testSuite": - # check we can access OM server - metadata.health_check() return WorkflowSource( type=service_type, serviceName=ingestion_pipeline.service.name, @@ -134,37 +135,51 @@ def build_source(ingestion_pipeline: IngestionPipeline) -> WorkflowSource: if service_type == "databaseService": entity_class = DatabaseService service: DatabaseService = metadata.get_by_name( - entity=entity_class, fqn=ingestion_pipeline.service.name + entity=entity_class, + fqn=ingestion_pipeline.service.name, + nullable=False, ) elif service_type == "pipelineService": entity_class = PipelineService service: PipelineService = metadata.get_by_name( - entity=entity_class, fqn=ingestion_pipeline.service.name + entity=entity_class, + fqn=ingestion_pipeline.service.name, + nullable=False, ) elif service_type == "dashboardService": entity_class = DashboardService service: DashboardService = metadata.get_by_name( - entity=entity_class, fqn=ingestion_pipeline.service.name + entity=entity_class, + fqn=ingestion_pipeline.service.name, + nullable=False, ) elif service_type == "messagingService": entity_class = MessagingService service: MessagingService = metadata.get_by_name( - entity=entity_class, fqn=ingestion_pipeline.service.name + entity=entity_class, + fqn=ingestion_pipeline.service.name, + nullable=False, ) elif service_type == "mlmodelService": entity_class = MlModelService service: MlModelService = metadata.get_by_name( - entity=entity_class, fqn=ingestion_pipeline.service.name + entity=entity_class, + fqn=ingestion_pipeline.service.name, + nullable=False, ) elif service_type == "metadataService": entity_class = MetadataService service: MetadataService = metadata.get_by_name( - entity=entity_class, fqn=ingestion_pipeline.service.name + entity=entity_class, + fqn=ingestion_pipeline.service.name, + nullable=False, ) elif service_type == "storageService": entity_class = StorageService service: StorageService = metadata.get_by_name( - entity=entity_class, fqn=ingestion_pipeline.service.name + entity=entity_class, + fqn=ingestion_pipeline.service.name, + nullable=False, ) else: raise InvalidServiceException(f"Invalid Service Type: {service_type}") diff --git a/openmetadata-docs/content/v1.1.1/connectors/dashboard/index.md b/openmetadata-docs/content/v1.1.1/connectors/dashboard/index.md index 4f78309d328..c8e02cac30c 100644 --- a/openmetadata-docs/content/v1.1.1/connectors/dashboard/index.md +++ b/openmetadata-docs/content/v1.1.1/connectors/dashboard/index.md @@ -12,6 +12,7 @@ This is the supported list of connectors for Dashboard Services: - [Metabase](/connectors/dashboard/metabase) - [Mode](/connectors/dashboard/mode) - [PowerBI](/connectors/dashboard/powerbi) +- [Qliksense](/connectors/dashboard/qliksense) - [QuickSight](/connectors/dashboard/quicksight) - [Redash](/connectors/dashboard/redash) - [Superset](/connectors/dashboard/superset) diff --git a/openmetadata-docs/content/v1.1.1/connectors/database/deltalake/index.md b/openmetadata-docs/content/v1.1.1/connectors/database/deltalake/index.md index 6c2e55c6802..1d13e771d8d 100644 --- a/openmetadata-docs/content/v1.1.1/connectors/database/deltalake/index.md +++ b/openmetadata-docs/content/v1.1.1/connectors/database/deltalake/index.md @@ -15,7 +15,7 @@ slug: /connectors/database/deltalake | Data Profiler | {% icon iconName="cross" /%} | | Data Quality | {% icon iconName="cross" /%} | | Lineage | Partially via Views | -| DBT | {% icon iconName="cross" /%} | +| DBT | {% icon iconName="check" /%} | | Supported Versions | -- | | Feature | Status | diff --git a/openmetadata-docs/content/v1.1.1/connectors/database/deltalake/yaml.md b/openmetadata-docs/content/v1.1.1/connectors/database/deltalake/yaml.md index 2f73a9b7c46..e42d2787909 100644 --- a/openmetadata-docs/content/v1.1.1/connectors/database/deltalake/yaml.md +++ b/openmetadata-docs/content/v1.1.1/connectors/database/deltalake/yaml.md @@ -15,7 +15,7 @@ slug: /connectors/database/deltalake/yaml | Data Profiler | {% icon iconName="cross" /%} | | Data Quality | {% icon iconName="cross" /%} | | Lineage | Partially via Views | -| DBT | {% icon iconName="cross" /%} | +| DBT | {% icon iconName="check" /%} | | Supported Versions | -- | | Feature | Status | @@ -144,7 +144,7 @@ The `sourceConfig` is defined [here](https://github.com/open-metadata/OpenMetada **includeViews**: true or false, to ingest views definitions. -**databaseFilterPattern**, **schemaFilterPattern**, **tableFilterPattern**: Note that the filter supports regex as include or exclude. You can find examples [here](/connectors/ingestion/workflows/metadata/filter-patterns/database) +**databaseFilterPattern**, **schemaFilterPattern**, **tableFilternPattern**: Note that the filter supports regex as include or exclude. You can find examples [here](/connectors/ingestion/workflows/metadata/filter-patterns/database) {% /codeInfo %} diff --git a/openmetadata-docs/content/v1.1.1/connectors/database/index.md b/openmetadata-docs/content/v1.1.1/connectors/database/index.md index a8456aba099..0599b0dbed9 100644 --- a/openmetadata-docs/content/v1.1.1/connectors/database/index.md +++ b/openmetadata-docs/content/v1.1.1/connectors/database/index.md @@ -27,6 +27,7 @@ This is the supported list of connectors for Database Services: - [Postgres](/connectors/database/postgres) - [Presto](/connectors/database/presto) - [Redshift](/connectors/database/redshift) +- [Sap Hana](/connectors/database/saphana) - [Salesforce](/connectors/database/salesforce) - [SingleStore](/connectors/database/singlestore) - [Snowflake](/connectors/database/snowflake) diff --git a/openmetadata-docs/content/v1.2.0-SNAPSHOT/connectors/database/deltalake/index.md b/openmetadata-docs/content/v1.2.0-SNAPSHOT/connectors/database/deltalake/index.md index 5ed7c8d91cc..13fadaed015 100644 --- a/openmetadata-docs/content/v1.2.0-SNAPSHOT/connectors/database/deltalake/index.md +++ b/openmetadata-docs/content/v1.2.0-SNAPSHOT/connectors/database/deltalake/index.md @@ -15,7 +15,7 @@ slug: /connectors/database/deltalake | Data Profiler | {% icon iconName="cross" /%} | | Data Quality | {% icon iconName="cross" /%} | | Lineage | Partially via Views | -| DBT | {% icon iconName="cross" /%} | +| DBT | {% icon iconName="check" /%} | | Supported Versions | -- | | Feature | Status | diff --git a/openmetadata-docs/content/v1.2.0-SNAPSHOT/connectors/database/deltalake/yaml.md b/openmetadata-docs/content/v1.2.0-SNAPSHOT/connectors/database/deltalake/yaml.md index b98cfb01096..3c28dbc247d 100644 --- a/openmetadata-docs/content/v1.2.0-SNAPSHOT/connectors/database/deltalake/yaml.md +++ b/openmetadata-docs/content/v1.2.0-SNAPSHOT/connectors/database/deltalake/yaml.md @@ -15,7 +15,7 @@ slug: /connectors/database/deltalake/yaml | Data Profiler | {% icon iconName="cross" /%} | | Data Quality | {% icon iconName="cross" /%} | | Lineage | Partially via Views | -| DBT | {% icon iconName="cross" /%} | +| DBT | {% icon iconName="check" /%} | | Supported Versions | -- | | Feature | Status | diff --git a/openmetadata-docs/images/connectors/mongodb.png b/openmetadata-docs/images/connectors/mongodb.png new file mode 100644 index 00000000000..99bf30d394c Binary files /dev/null and b/openmetadata-docs/images/connectors/mongodb.png differ diff --git a/openmetadata-docs/images/connectors/qliksense.png b/openmetadata-docs/images/connectors/qliksense.png new file mode 100644 index 00000000000..6327edc3466 Binary files /dev/null and b/openmetadata-docs/images/connectors/qliksense.png differ diff --git a/openmetadata-docs/images/connectors/saphana.png b/openmetadata-docs/images/connectors/saphana.png new file mode 100644 index 00000000000..88076387631 Binary files /dev/null and b/openmetadata-docs/images/connectors/saphana.png differ