Fix #8885 - Misleading 404 logs & suffix cleanup (#12916)

* Fix #8885 - Misleading 404 debug logs

* Clean get_suffix complexity

* Prepare airflow config

* Add bootstrap comment

* Revert "Add bootstrap comment"

This reverts commit cc053513df6e25bcc5e742786adb91339461fdd2.

* Revert "Prepare airflow config"

This reverts commit f57645fe953e17d6153652daa1f42c14b97c942d.

* add removed method

* Add icons for docs

* Add missing routes

ingestion/src/metadata/ingestion/ometa/ometa_api.py

@@ -17,65 +17,12 @@ working with OpenMetadata entities.
 import traceback
 from typing import Dict, Generic, Iterable, List, Optional, Type, TypeVar, Union
 
-try:
-    from typing import get_args
-except ImportError:
-    from typing_compat import get_args
-
 from pydantic import BaseModel
 from requests.utils import quote
 
-from metadata.generated.schema.analytics.webAnalyticEventData import (
-    WebAnalyticEventData,
-)
-from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
-from metadata.generated.schema.dataInsight.dataInsightChart import DataInsightChart
-from metadata.generated.schema.dataInsight.kpi.kpi import Kpi
-from metadata.generated.schema.entity.automations.workflow import Workflow
-from metadata.generated.schema.entity.classification.classification import (
-    Classification,
-)
-from metadata.generated.schema.entity.classification.tag import Tag
-from metadata.generated.schema.entity.data.chart import Chart
-from metadata.generated.schema.entity.data.container import Container
-from metadata.generated.schema.entity.data.dashboard import Dashboard
-from metadata.generated.schema.entity.data.dashboardDataModel import DashboardDataModel
-from metadata.generated.schema.entity.data.database import Database
-from metadata.generated.schema.entity.data.databaseSchema import DatabaseSchema
-from metadata.generated.schema.entity.data.glossary import Glossary
-from metadata.generated.schema.entity.data.glossaryTerm import GlossaryTerm
-from metadata.generated.schema.entity.data.metrics import Metrics
-from metadata.generated.schema.entity.data.mlmodel import MlModel
-from metadata.generated.schema.entity.data.pipeline import Pipeline
-from metadata.generated.schema.entity.data.query import Query
-from metadata.generated.schema.entity.data.report import Report
-from metadata.generated.schema.entity.data.searchIndex import SearchIndex
-from metadata.generated.schema.entity.data.table import Table
-from metadata.generated.schema.entity.data.topic import Topic
-from metadata.generated.schema.entity.policies.policy import Policy
 from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import (
     OpenMetadataConnection,
 )
-from metadata.generated.schema.entity.services.connections.testConnectionDefinition import (
-    TestConnectionDefinition,
-)
-from metadata.generated.schema.entity.services.dashboardService import DashboardService
-from metadata.generated.schema.entity.services.databaseService import DatabaseService
-from metadata.generated.schema.entity.services.ingestionPipelines.ingestionPipeline import (
-    IngestionPipeline,
-)
-from metadata.generated.schema.entity.services.messagingService import MessagingService
-from metadata.generated.schema.entity.services.metadataService import MetadataService
-from metadata.generated.schema.entity.services.mlmodelService import MlModelService
-from metadata.generated.schema.entity.services.pipelineService import PipelineService
-from metadata.generated.schema.entity.services.searchService import SearchService
-from metadata.generated.schema.entity.services.storageService import StorageService
-from metadata.generated.schema.entity.teams.role import Role
-from metadata.generated.schema.entity.teams.team import Team
-from metadata.generated.schema.entity.teams.user import AuthenticationMechanism, User
-from metadata.generated.schema.tests.testCase import TestCase
-from metadata.generated.schema.tests.testDefinition import TestDefinition
-from metadata.generated.schema.tests.testSuite import TestSuite
 from metadata.generated.schema.type import basic
 from metadata.generated.schema.type.basic import FullyQualifiedEntityName
 from metadata.generated.schema.type.entityHistory import EntityVersionHistory
@@ -108,6 +55,7 @@ from metadata.ingestion.ometa.provider_registry import (
     InvalidAuthProviderException,
     auth_provider_registry,
 )
+from metadata.ingestion.ometa.routes import ROUTES
 from metadata.ingestion.ometa.utils import get_entity_type, model_str
 from metadata.utils.logger import ometa_logger
 from metadata.utils.secrets.secrets_manager_factory import SecretsManagerFactory
@@ -227,262 +175,20 @@ class OpenMetadata(
         if self.config.enableVersionValidation:
             self.validate_versions()
 
-    def get_suffix(self, entity: Type[T]) -> str:  # pylint: disable=R0911,R0912
+    @staticmethod
+    def get_suffix(entity: Type[T]) -> str:
         """
         Given an entity Type from the generated sources,
         return the endpoint to run requests.
-
-        Might be interesting to follow a more strict
-        and type-checked approach
-
-        Disabled pylint R0911: too-many-return-statements
-        Disabled pylint R0912: too-many-branches
         """
 
-        # Entity Schemas
-        if issubclass(
-            entity, get_args(Union[MlModel, self.get_create_entity_type(MlModel)])
-        ):
-            return "/mlmodels"
+        route = ROUTES.get(entity.__name__)
+        if route is None:
+            raise MissingEntityTypeException(
+                f"Missing {entity} type when generating suffixes"
+            )
 
-        if issubclass(
-            entity, get_args(Union[Chart, self.get_create_entity_type(Chart)])
-        ):
-            return "/charts"
-
-        if issubclass(
-            entity,
-            get_args(
-                Union[
-                    DashboardDataModel, self.get_create_entity_type(DashboardDataModel)
-                ]
-            ),
-        ):
-            return "/dashboard/datamodels"
-
-        if issubclass(
-            entity, get_args(Union[Dashboard, self.get_create_entity_type(Dashboard)])
-        ):
-            return "/dashboards"
-
-        if issubclass(
-            entity, get_args(Union[Database, self.get_create_entity_type(Database)])
-        ):
-            return "/databases"
-
-        if issubclass(
-            entity,
-            get_args(
-                Union[DatabaseSchema, self.get_create_entity_type(DatabaseSchema)]
-            ),
-        ):
-            return "/databaseSchemas"
-
-        if issubclass(
-            entity, get_args(Union[Pipeline, self.get_create_entity_type(Pipeline)])
-        ):
-            return "/pipelines"
-
-        if issubclass(
-            entity, get_args(Union[Policy, self.get_create_entity_type(Policy)])
-        ):
-            return "/policies"
-
-        if issubclass(
-            entity, get_args(Union[Table, self.get_create_entity_type(Table)])
-        ):
-            return "/tables"
-
-        if issubclass(
-            entity, get_args(Union[Topic, self.get_create_entity_type(Topic)])
-        ):
-            return "/topics"
-
-        if issubclass(entity, Metrics):
-            return "/metrics"
-
-        if issubclass(entity, AddLineageRequest):
-            return "/lineage"
-
-        if issubclass(entity, Report):
-            return "/reports"
-
-        if issubclass(entity, AuthenticationMechanism):
-            return "/users/auth-mechanism"
-
-        if issubclass(
-            entity,
-            get_args(
-                Union[
-                    Tag,
-                    self.get_create_entity_type(Tag),
-                ]
-            ),
-        ):
-            return "/tags"
-
-        if issubclass(
-            entity,
-            get_args(
-                Union[
-                    Classification,
-                    self.get_create_entity_type(Classification),
-                ]
-            ),
-        ):
-            return "/classifications"
-
-        if issubclass(
-            entity, get_args(Union[Glossary, self.get_create_entity_type(Glossary)])
-        ):
-            return "/glossaries"
-
-        if issubclass(
-            entity,
-            get_args(Union[GlossaryTerm, self.get_create_entity_type(GlossaryTerm)]),
-        ):
-            return "/glossaryTerms"
-
-        if issubclass(entity, get_args(Union[Role, self.get_create_entity_type(Role)])):
-            return "/roles"
-
-        if issubclass(
-            entity, get_args(Union[Query, self.get_create_entity_type(Query)])
-        ):
-            return "/queries"
-
-        if issubclass(entity, get_args(Union[Team, self.get_create_entity_type(Team)])):
-            return "/teams"
-
-        if issubclass(entity, get_args(Union[User, self.get_create_entity_type(User)])):
-            return "/users"
-
-        if issubclass(
-            entity, get_args(Union[Container, self.get_create_entity_type(Container)])
-        ):
-            return "/containers"
-
-        if issubclass(
-            entity,
-            get_args(Union[SearchIndex, self.get_create_entity_type(SearchIndex)]),
-        ):
-            return "/searchIndexes"
-
-        if issubclass(
-            entity, get_args(Union[Workflow, self.get_create_entity_type(Workflow)])
-        ):
-            return "/automations/workflows"
-
-        # Services Schemas
-        if issubclass(
-            entity,
-            get_args(
-                Union[DatabaseService, self.get_create_entity_type(DatabaseService)]
-            ),
-        ):
-            return "/services/databaseServices"
-
-        if issubclass(
-            entity,
-            get_args(
-                Union[DashboardService, self.get_create_entity_type(DashboardService)]
-            ),
-        ):
-            return "/services/dashboardServices"
-
-        if issubclass(
-            entity,
-            get_args(
-                Union[MessagingService, self.get_create_entity_type(MessagingService)]
-            ),
-        ):
-            return "/services/messagingServices"
-
-        if issubclass(
-            entity,
-            get_args(
-                Union[PipelineService, self.get_create_entity_type(PipelineService)]
-            ),
-        ):
-            return "/services/pipelineServices"
-
-        if issubclass(
-            entity,
-            get_args(
-                Union[StorageService, self.get_create_entity_type(StorageService)]
-            ),
-        ):
-            return "/services/storageServices"
-
-        if issubclass(
-            entity,
-            get_args(
-                Union[MlModelService, self.get_create_entity_type(MlModelService)]
-            ),
-        ):
-            return "/services/mlmodelServices"
-
-        if issubclass(
-            entity,
-            get_args(
-                Union[MetadataService, self.get_create_entity_type(MetadataService)]
-            ),
-        ):
-            return "/services/metadataServices"
-
-        if issubclass(
-            entity,
-            get_args(Union[SearchService, self.get_create_entity_type(SearchService)]),
-        ):
-            return "/services/searchServices"
-
-        if issubclass(
-            entity,
-            IngestionPipeline,
-        ):
-            return "/services/ingestionPipelines"
-
-        if issubclass(
-            entity,
-            TestConnectionDefinition,
-        ):
-            return "/services/testConnectionDefinitions"
-
-        if issubclass(
-            entity,
-            get_args(
-                Union[TestDefinition, self.get_create_entity_type(TestDefinition)]
-            ),
-        ):
-            return "/dataQuality/testDefinitions"
-
-        if issubclass(
-            entity,
-            get_args(Union[TestSuite, self.get_create_entity_type(TestSuite)]),
-        ):
-            return "/dataQuality/testSuites"
-
-        if issubclass(
-            entity,
-            get_args(Union[TestCase, self.get_create_entity_type(TestCase)]),
-        ):
-            return "/dataQuality/testCases"
-
-        if issubclass(entity, WebAnalyticEventData):
-            return "/analytics/web/events/collect"
-
-        if issubclass(entity, DataInsightChart):
-            return "/analytics/dataInsights/charts"
-
-        if issubclass(
-            entity,
-            Kpi,
-        ):
-            return "/kpi"
-
-        raise MissingEntityTypeException(
-            f"Missing {entity} type when generating suffixes"
-        )
+        return route
 
     def get_module_path(self, entity: Type[T]) -> str:
         """
@@ -499,7 +205,6 @@ class OpenMetadata(
     def get_create_entity_type(self, entity: Type[T]) -> Type[C]:
         """
         imports and returns the Create Type from an Entity Type T.
-
         We are following the expected path structure to import
         on-the-fly the necessary class and pass it to the consumer
         """
@@ -591,6 +296,7 @@ class OpenMetadata(
         entity: Type[T],
         fqn: Union[str, FullyQualifiedEntityName],
         fields: Optional[List[str]] = None,
+        nullable: bool = True,
     ) -> Optional[T]:
         """
         Return entity by name or None
@@ -600,6 +306,7 @@ class OpenMetadata(
             entity=entity,
             path=f"name/{quote(model_str(fqn), safe='')}",
             fields=fields,
+            nullable=nullable,
         )
 
     def get_by_id(
@@ -607,14 +314,24 @@ class OpenMetadata(
         entity: Type[T],
         entity_id: Union[str, basic.Uuid],
         fields: Optional[List[str]] = None,
+        nullable: bool = True,
     ) -> Optional[T]:
         """
         Return entity by ID or None
         """
-        return self._get(entity=entity, path=model_str(entity_id), fields=fields)
+        return self._get(
+            entity=entity,
+            path=model_str(entity_id),
+            fields=fields,
+            nullable=nullable,
+        )
 
     def _get(
-        self, entity: Type[T], path: str, fields: Optional[List[str]] = None
+        self,
+        entity: Type[T],
+        path: str,
+        fields: Optional[List[str]] = None,
+        nullable: bool = True,
     ) -> Optional[T]:
         """
         Generic GET operation for an entity
@@ -631,6 +348,12 @@ class OpenMetadata(
                 )
             return entity(**resp)
         except APIError as err:
+            # We can expect some GET calls to return us a None and manage it in following steps.
+            # No need to pollute the logs in these cases.
+            if err.code == 404 and nullable:
+                return None
+
+            # Any other API errors will be passed to the client
             logger.debug(traceback.format_exc())
             logger.debug(
                 "GET %s for %s. Error %s - %s",
@@ -639,7 +362,7 @@ class OpenMetadata(
                 err.status_code,
                 err,
             )
-            return None
+            raise err
 
     def get_entity_reference(
         self, entity: Type[T], fqn: str
@@ -667,7 +390,7 @@ class OpenMetadata(
         self,
         entity: Type[T],
         fields: Optional[List[str]] = None,
-        after: str = None,
+        after: Optional[str] = None,
         limit: int = 100,
         params: Optional[Dict[str, str]] = None,
     ) -> EntityList[T]:

ingestion/src/metadata/ingestion/ometa/routes.py

@@ -0,0 +1,207 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""
+OMeta API endpoints
+"""
+from metadata.generated.schema.analytics.webAnalyticEventData import (
+    WebAnalyticEventData,
+)
+from metadata.generated.schema.api.automations.createWorkflow import (
+    CreateWorkflowRequest,
+)
+from metadata.generated.schema.api.classification.createClassification import (
+    CreateClassificationRequest,
+)
+from metadata.generated.schema.api.classification.createTag import CreateTagRequest
+from metadata.generated.schema.api.data.createChart import CreateChartRequest
+from metadata.generated.schema.api.data.createContainer import CreateContainerRequest
+from metadata.generated.schema.api.data.createDashboard import CreateDashboardRequest
+from metadata.generated.schema.api.data.createDashboardDataModel import (
+    CreateDashboardDataModelRequest,
+)
+from metadata.generated.schema.api.data.createDatabase import CreateDatabaseRequest
+from metadata.generated.schema.api.data.createDatabaseSchema import (
+    CreateDatabaseSchemaRequest,
+)
+from metadata.generated.schema.api.data.createGlossary import CreateGlossaryRequest
+from metadata.generated.schema.api.data.createGlossaryTerm import (
+    CreateGlossaryTermRequest,
+)
+from metadata.generated.schema.api.data.createMlModel import CreateMlModelRequest
+from metadata.generated.schema.api.data.createPipeline import CreatePipelineRequest
+from metadata.generated.schema.api.data.createQuery import CreateQueryRequest
+from metadata.generated.schema.api.data.createSearchIndex import (
+    CreateSearchIndexRequest,
+)
+from metadata.generated.schema.api.data.createTable import CreateTableRequest
+from metadata.generated.schema.api.data.createTopic import CreateTopicRequest
+from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
+from metadata.generated.schema.api.policies.createPolicy import CreatePolicyRequest
+from metadata.generated.schema.api.services.createDashboardService import (
+    CreateDashboardServiceRequest,
+)
+from metadata.generated.schema.api.services.createDatabaseService import (
+    CreateDatabaseServiceRequest,
+)
+from metadata.generated.schema.api.services.createMessagingService import (
+    CreateMessagingServiceRequest,
+)
+from metadata.generated.schema.api.services.createMetadataService import (
+    CreateMetadataServiceRequest,
+)
+from metadata.generated.schema.api.services.createMlModelService import (
+    CreateMlModelServiceRequest,
+)
+from metadata.generated.schema.api.services.createPipelineService import (
+    CreatePipelineServiceRequest,
+)
+from metadata.generated.schema.api.services.createSearchService import (
+    CreateSearchServiceRequest,
+)
+from metadata.generated.schema.api.services.createStorageService import (
+    CreateStorageServiceRequest,
+)
+from metadata.generated.schema.api.teams.createRole import CreateRoleRequest
+from metadata.generated.schema.api.teams.createTeam import CreateTeamRequest
+from metadata.generated.schema.api.teams.createUser import CreateUserRequest
+from metadata.generated.schema.api.tests.createTestCase import CreateTestCaseRequest
+from metadata.generated.schema.api.tests.createTestDefinition import (
+    CreateTestDefinitionRequest,
+)
+from metadata.generated.schema.api.tests.createTestSuite import CreateTestSuiteRequest
+from metadata.generated.schema.dataInsight.dataInsightChart import DataInsightChart
+from metadata.generated.schema.dataInsight.kpi.kpi import Kpi
+from metadata.generated.schema.entity.automations.workflow import Workflow
+from metadata.generated.schema.entity.classification.classification import (
+    Classification,
+)
+from metadata.generated.schema.entity.classification.tag import Tag
+from metadata.generated.schema.entity.data.chart import Chart
+from metadata.generated.schema.entity.data.container import Container
+from metadata.generated.schema.entity.data.dashboard import Dashboard
+from metadata.generated.schema.entity.data.dashboardDataModel import DashboardDataModel
+from metadata.generated.schema.entity.data.database import Database
+from metadata.generated.schema.entity.data.databaseSchema import DatabaseSchema
+from metadata.generated.schema.entity.data.glossary import Glossary
+from metadata.generated.schema.entity.data.glossaryTerm import GlossaryTerm
+from metadata.generated.schema.entity.data.metrics import Metrics
+from metadata.generated.schema.entity.data.mlmodel import MlModel
+from metadata.generated.schema.entity.data.pipeline import Pipeline
+from metadata.generated.schema.entity.data.query import Query
+from metadata.generated.schema.entity.data.report import Report
+from metadata.generated.schema.entity.data.searchIndex import SearchIndex
+from metadata.generated.schema.entity.data.table import Table
+from metadata.generated.schema.entity.data.topic import Topic
+from metadata.generated.schema.entity.policies.policy import Policy
+from metadata.generated.schema.entity.services.connections.testConnectionDefinition import (
+    TestConnectionDefinition,
+)
+from metadata.generated.schema.entity.services.dashboardService import DashboardService
+from metadata.generated.schema.entity.services.databaseService import DatabaseService
+from metadata.generated.schema.entity.services.ingestionPipelines.ingestionPipeline import (
+    IngestionPipeline,
+)
+from metadata.generated.schema.entity.services.messagingService import MessagingService
+from metadata.generated.schema.entity.services.metadataService import MetadataService
+from metadata.generated.schema.entity.services.mlmodelService import MlModelService
+from metadata.generated.schema.entity.services.pipelineService import PipelineService
+from metadata.generated.schema.entity.services.searchService import SearchService
+from metadata.generated.schema.entity.services.storageService import StorageService
+from metadata.generated.schema.entity.teams.role import Role
+from metadata.generated.schema.entity.teams.team import Team
+from metadata.generated.schema.entity.teams.user import AuthenticationMechanism, User
+from metadata.generated.schema.tests.testCase import TestCase
+from metadata.generated.schema.tests.testDefinition import TestDefinition
+from metadata.generated.schema.tests.testSuite import TestSuite
+
+ROUTES = {
+    MlModel.__name__: "/mlmodels",
+    CreateMlModelRequest.__name__: "/mlmodels",
+    Chart.__name__: "/charts",
+    CreateChartRequest.__name__: "/charts",
+    DashboardDataModel.__name__: "/dashboard/datamodels",
+    CreateDashboardDataModelRequest.__name__: "/dashboard/datamodels",
+    Dashboard.__name__: "/dashboards",
+    CreateDashboardRequest.__name__: "/dashboards",
+    Database.__name__: "/databases",
+    CreateDatabaseRequest.__name__: "/databases",
+    DatabaseSchema.__name__: "/databaseSchemas",
+    CreateDatabaseSchemaRequest.__name__: "/databaseSchemas",
+    Pipeline.__name__: "/pipelines",
+    CreatePipelineRequest.__name__: "/pipelines",
+    Table.__name__: "/tables",
+    CreateTableRequest.__name__: "/tables",
+    Topic.__name__: "/topics",
+    CreateTopicRequest.__name__: "/topics",
+    Metrics.__name__: "/metrics",
+    AddLineageRequest.__name__: "/lineage",
+    Report.__name__: "/reports",
+    Query.__name__: "/queries",
+    CreateQueryRequest.__name__: "/queries",
+    Container.__name__: "/containers",
+    CreateContainerRequest.__name__: "/containers",
+    SearchIndex.__name__: "/searchIndexes",
+    CreateSearchIndexRequest.__name__: "/searchIndexes",
+    # Classifications
+    Tag.__name__: "/tags",
+    CreateTagRequest.__name__: "/tags",
+    Classification.__name__: "/classifications",
+    CreateClassificationRequest.__name__: "/classifications",
+    # Glossaries
+    Glossary.__name__: "/glossaries",
+    CreateGlossaryRequest.__name__: "/glossaries",
+    GlossaryTerm.__name__: "/glossaryTerms",
+    CreateGlossaryTermRequest.__name__: "/glossaryTerms",
+    # Users
+    Team.__name__: "/teams",
+    CreateTeamRequest.__name__: "/teams",
+    User.__name__: "/users",
+    CreateUserRequest.__name__: "/users",
+    AuthenticationMechanism.__name__: "/users/auth-mechanism",
+    # Roles
+    Role.__name__: "/roles",
+    CreateRoleRequest.__name__: "/roles",
+    Policy.__name__: "/policies",
+    CreatePolicyRequest.__name__: "/policies",
+    # Automations
+    Workflow.__name__: "/automations/workflows",
+    CreateWorkflowRequest.__name__: "/automations/workflows",
+    # Services
+    DatabaseService.__name__: "/services/databaseServices",
+    CreateDatabaseServiceRequest.__name__: "/services/databaseServices",
+    DashboardService.__name__: "/services/dashboardServices",
+    CreateDashboardServiceRequest.__name__: "/services/dashboardServices",
+    MessagingService.__name__: "/services/messagingServices",
+    CreateMessagingServiceRequest.__name__: "/services/messagingServices",
+    PipelineService.__name__: "/services/pipelineServices",
+    CreatePipelineServiceRequest.__name__: "/services/pipelineServices",
+    StorageService.__name__: "/services/storageServices",
+    CreateStorageServiceRequest.__name__: "/services/storageServices",
+    MlModelService.__name__: "/services/mlmodelServices",
+    CreateMlModelServiceRequest.__name__: "/services/mlmodelServices",
+    MetadataService.__name__: "/services/metadataServices",
+    CreateMetadataServiceRequest.__name__: "/services/metadataServices",
+    SearchService.__name__: "/services/searchServices",
+    CreateSearchServiceRequest.__name__: "/services/searchServices",
+    IngestionPipeline.__name__: "/services/ingestionPipelines",
+    TestConnectionDefinition.__name__: "/services/testConnectionDefinitions",
+    # Data Quality
+    TestDefinition.__name__: "/dataQuality/testDefinitions",
+    CreateTestDefinitionRequest.__name__: "/dataQuality/testDefinitions",
+    TestSuite.__name__: "/dataQuality/testSuites",
+    CreateTestSuiteRequest.__name__: "/dataQuality/testSuites",
+    TestCase.__name__: "/dataQuality/testCases",
+    CreateTestCaseRequest.__name__: "/dataQuality/testCases",
+    # Analytics
+    WebAnalyticEventData.__name__: "/analytics/web/events/collect",
+    DataInsightChart.__name__: "/analytics/dataInsights/charts",
+    Kpi.__name__: "/kpi",
+}

ingestion/tests/integration/ometa/test_ometa_table_api.py

@@ -210,6 +210,10 @@ class OMetaTableTest(TestCase):
         )
         self.assertEqual(res.name, self.entity.name)
 
+        # Now check that we get a None if the table does not exist
+        nullable_res = self.metadata.get_by_name(entity=Table, fqn="something.made.up")
+        self.assertIsNone(nullable_res)
+
     def test_get_id(self):
         """
         We can fetch a Table by ID and get it back as Entity

ingestion/tests/unit/test_ometa_endpoints.py

@@ -105,20 +105,6 @@ class OMetaEndpointTest(TestCase):
         """
         self.assertEqual(self.metadata.get_suffix(User), "/users")
 
-    def test_get_create_entity_type(self):
-        """
-        Validate the mapping from Entity to CreateEntity
-        """
-
-        create = self.metadata.get_create_entity_type(Topic)
-        assert issubclass(create, CreateTopicRequest)
-
-        create = self.metadata.get_create_entity_type(DatabaseService)
-        assert issubclass(create, CreateDatabaseServiceRequest)
-
-        create = self.metadata.get_create_entity_type(User)
-        assert issubclass(create, CreateUserRequest)
-
     def test_get_entity_from_create(self):
         """
         Validate the mapping from CreateEntity to Entity

openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/common.py

@@ -109,6 +109,9 @@ def build_source(ingestion_pipeline: IngestionPipeline) -> WorkflowSource:
 
     try:
         metadata = OpenMetadata(config=ingestion_pipeline.openMetadataServerConnection)
+
+        # check we can access OM server
+        metadata.health_check()
     except Exception as exc:
         raise ClientInitializationError(
             f"Failed to initialize the OpenMetadata client due to: {exc}."
@@ -122,8 +125,6 @@ def build_source(ingestion_pipeline: IngestionPipeline) -> WorkflowSource:
     entity_class = None
     try:
         if service_type == "testSuite":
-            # check we can access OM server
-            metadata.health_check()
             return WorkflowSource(
                 type=service_type,
                 serviceName=ingestion_pipeline.service.name,
@@ -134,37 +135,51 @@ def build_source(ingestion_pipeline: IngestionPipeline) -> WorkflowSource:
         if service_type == "databaseService":
             entity_class = DatabaseService
             service: DatabaseService = metadata.get_by_name(
-                entity=entity_class, fqn=ingestion_pipeline.service.name
+                entity=entity_class,
+                fqn=ingestion_pipeline.service.name,
+                nullable=False,
             )
         elif service_type == "pipelineService":
             entity_class = PipelineService
             service: PipelineService = metadata.get_by_name(
-                entity=entity_class, fqn=ingestion_pipeline.service.name
+                entity=entity_class,
+                fqn=ingestion_pipeline.service.name,
+                nullable=False,
             )
         elif service_type == "dashboardService":
             entity_class = DashboardService
             service: DashboardService = metadata.get_by_name(
-                entity=entity_class, fqn=ingestion_pipeline.service.name
+                entity=entity_class,
+                fqn=ingestion_pipeline.service.name,
+                nullable=False,
             )
         elif service_type == "messagingService":
             entity_class = MessagingService
             service: MessagingService = metadata.get_by_name(
-                entity=entity_class, fqn=ingestion_pipeline.service.name
+                entity=entity_class,
+                fqn=ingestion_pipeline.service.name,
+                nullable=False,
             )
         elif service_type == "mlmodelService":
             entity_class = MlModelService
             service: MlModelService = metadata.get_by_name(
-                entity=entity_class, fqn=ingestion_pipeline.service.name
+                entity=entity_class,
+                fqn=ingestion_pipeline.service.name,
+                nullable=False,
             )
         elif service_type == "metadataService":
             entity_class = MetadataService
             service: MetadataService = metadata.get_by_name(
-                entity=entity_class, fqn=ingestion_pipeline.service.name
+                entity=entity_class,
+                fqn=ingestion_pipeline.service.name,
+                nullable=False,
             )
         elif service_type == "storageService":
             entity_class = StorageService
             service: StorageService = metadata.get_by_name(
-                entity=entity_class, fqn=ingestion_pipeline.service.name
+                entity=entity_class,
+                fqn=ingestion_pipeline.service.name,
+                nullable=False,
             )
         else:
             raise InvalidServiceException(f"Invalid Service Type: {service_type}")

openmetadata-docs/content/v1.1.1/connectors/dashboard/index.md

@@ -12,6 +12,7 @@ This is the supported list of connectors for Dashboard Services:
 - [Metabase](/connectors/dashboard/metabase)
 - [Mode](/connectors/dashboard/mode)
 - [PowerBI](/connectors/dashboard/powerbi)
+- [Qliksense](/connectors/dashboard/qliksense)
 - [QuickSight](/connectors/dashboard/quicksight)
 - [Redash](/connectors/dashboard/redash)
 - [Superset](/connectors/dashboard/superset)

openmetadata-docs/content/v1.1.1/connectors/database/deltalake/index.md

@@ -15,7 +15,7 @@ slug: /connectors/database/deltalake
 | Data Profiler      | {% icon iconName="cross" /%} |
 | Data Quality       | {% icon iconName="cross" /%} |
 | Lineage            | Partially via Views          |
-| DBT                | {% icon iconName="cross" /%} |
+| DBT                | {% icon iconName="check" /%}     |
 | Supported Versions | --                           |
 
 | Feature      | Status                       |

openmetadata-docs/content/v1.1.1/connectors/database/deltalake/yaml.md

@@ -15,7 +15,7 @@ slug: /connectors/database/deltalake/yaml
 | Data Profiler      | {% icon iconName="cross" /%} |
 | Data Quality       | {% icon iconName="cross" /%} |
 | Lineage            | Partially via Views          |
-| DBT                | {% icon iconName="cross" /%} |
+| DBT                | {% icon iconName="check" /%} |
 | Supported Versions | --                           |
 
 | Feature      | Status                       |
@@ -144,7 +144,7 @@ The `sourceConfig` is defined [here](https://github.com/open-metadata/OpenMetada
 
 **includeViews**: true or false, to ingest views definitions.
 
-**databaseFilterPattern**, **schemaFilterPattern**, **tableFilterPattern**: Note that the filter supports regex as include or exclude. You can find examples [here](/connectors/ingestion/workflows/metadata/filter-patterns/database)
+**databaseFilterPattern**, **schemaFilterPattern**, **tableFilternPattern**: Note that the filter supports regex as include or exclude. You can find examples [here](/connectors/ingestion/workflows/metadata/filter-patterns/database)
 
 {% /codeInfo %}
 

openmetadata-docs/content/v1.1.1/connectors/database/index.md

@@ -27,6 +27,7 @@ This is the supported list of connectors for Database Services:
 - [Postgres](/connectors/database/postgres)
 - [Presto](/connectors/database/presto)
 - [Redshift](/connectors/database/redshift)
+- [Sap Hana](/connectors/database/saphana)
 - [Salesforce](/connectors/database/salesforce)
 - [SingleStore](/connectors/database/singlestore)
 - [Snowflake](/connectors/database/snowflake)

openmetadata-docs/content/v1.2.0-SNAPSHOT/connectors/database/deltalake/index.md

@@ -15,7 +15,7 @@ slug: /connectors/database/deltalake
 | Data Profiler      | {% icon iconName="cross" /%} |
 | Data Quality       | {% icon iconName="cross" /%} |
 | Lineage            | Partially via Views          |
-| DBT                | {% icon iconName="cross" /%} |
+| DBT                | {% icon iconName="check" /%} |
 | Supported Versions | --                           |
 
 | Feature      | Status                       |

openmetadata-docs/content/v1.2.0-SNAPSHOT/connectors/database/deltalake/yaml.md

@@ -15,7 +15,7 @@ slug: /connectors/database/deltalake/yaml
 | Data Profiler      | {% icon iconName="cross" /%} |
 | Data Quality       | {% icon iconName="cross" /%} |
 | Lineage            | Partially via Views          |
-| DBT                | {% icon iconName="cross" /%} |
+| DBT                | {% icon iconName="check" /%} |
 | Supported Versions | --                           |
 
 | Feature      | Status                       |