Fix #13794 - Add domain support to the Python SDK (#13931)

ingestion/src/metadata/ingestion/ometa/mixins/patch_mixin.py

@@ -25,6 +25,7 @@ from metadata.generated.schema.entity.automations.workflow import (
 )
 from metadata.generated.schema.entity.automations.workflow import WorkflowStatus
 from metadata.generated.schema.entity.data.table import Column, Table, TableConstraint
+from metadata.generated.schema.entity.domains.domain import Domain
 from metadata.generated.schema.entity.services.connections.testConnectionResult import (
     TestConnectionResult,
 )
@@ -485,7 +486,9 @@ class OMetaPatchMixin(OMetaPatchMixinBase):
                 f"Error trying to PATCH status for automation workflow [{model_str(automation_workflow)}]: {exc}"
             )
 
-    def patch_life_cycle(self, entity: Entity, life_cycle: LifeCycle) -> None:
+    def patch_life_cycle(
+        self, entity: Entity, life_cycle: LifeCycle
+    ) -> Optional[Entity]:
         """
         Patch life cycle data for a entity
 
@@ -495,9 +498,27 @@ class OMetaPatchMixin(OMetaPatchMixinBase):
         try:
             destination = entity.copy(deep=True)
             destination.lifeCycle = life_cycle
-            self.patch(entity=type(entity), source=entity, destination=destination)
+            return self.patch(
+                entity=type(entity), source=entity, destination=destination
+            )
         except Exception as exc:
             logger.debug(traceback.format_exc())
             logger.warning(
                 f"Error trying to Patch life cycle data for {entity.fullyQualifiedName.__root__}: {exc}"
             )
+            return None
+
+    def patch_domain(self, entity: Entity, domain: Domain) -> Optional[Entity]:
+        """Patch domain data for an Entity"""
+        try:
+            destination: Entity = entity.copy(deep=True)
+            destination.domain = EntityReference(id=domain.id, type="domain")
+            return self.patch(
+                entity=type(entity), source=entity, destination=destination
+            )
+        except Exception as exc:
+            logger.debug(traceback.format_exc())
+            logger.warning(
+                f"Error trying to Patch Domain for {entity.fullyQualifiedName.__root__}: {exc}"
+            )
+            return None

ingestion/src/metadata/ingestion/ometa/ometa_api.py

@@ -239,6 +239,7 @@ class OpenMetadata(
             .replace("searchindex", "searchIndex")
             .replace("storedprocedure", "storedProcedure")
             .replace("ingestionpipeline", "ingestionPipeline")
+            .replace("dataproduct", "dataProduct")
         )
         class_path = ".".join(
             filter(

ingestion/src/metadata/ingestion/ometa/routes.py

@@ -46,6 +46,10 @@ from metadata.generated.schema.api.data.createStoredProcedure import (
 )
 from metadata.generated.schema.api.data.createTable import CreateTableRequest
 from metadata.generated.schema.api.data.createTopic import CreateTopicRequest
+from metadata.generated.schema.api.domains.createDataProduct import (
+    CreateDataProductRequest,
+)
+from metadata.generated.schema.api.domains.createDomain import CreateDomainRequest
 from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
 from metadata.generated.schema.api.policies.createPolicy import CreatePolicyRequest
 from metadata.generated.schema.api.services.createDashboardService import (
@@ -107,6 +111,8 @@ from metadata.generated.schema.entity.data.searchIndex import SearchIndex
 from metadata.generated.schema.entity.data.storedProcedure import StoredProcedure
 from metadata.generated.schema.entity.data.table import Table
 from metadata.generated.schema.entity.data.topic import Topic
+from metadata.generated.schema.entity.domains.dataProduct import DataProduct
+from metadata.generated.schema.entity.domains.domain import Domain
 from metadata.generated.schema.entity.policies.policy import Policy
 from metadata.generated.schema.entity.services.connections.testConnectionDefinition import (
     TestConnectionDefinition,
@@ -214,4 +220,9 @@ ROUTES = {
     WebAnalyticEventData.__name__: "/analytics/web/events/collect",
     DataInsightChart.__name__: "/analytics/dataInsights/charts",
     Kpi.__name__: "/kpi",
+    # Domains & Data Products
+    Domain.__name__: "/domains",
+    CreateDomainRequest.__name__: "/domains",
+    DataProduct.__name__: "/dataProducts",
+    CreateDataProductRequest.__name__: "/dataProducts",
 }

ingestion/tests/integration/ometa/test_ometa_domains_api.py

@@ -0,0 +1,192 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""
+OpenMetadata high-level API Domains & Data Products test
+"""
+from unittest import TestCase
+
+from metadata.generated.schema.api.data.createDashboard import CreateDashboardRequest
+from metadata.generated.schema.api.domains.createDataProduct import (
+    CreateDataProductRequest,
+)
+from metadata.generated.schema.api.domains.createDomain import CreateDomainRequest
+from metadata.generated.schema.api.services.createDashboardService import (
+    CreateDashboardServiceRequest,
+)
+from metadata.generated.schema.api.teams.createUser import CreateUserRequest
+from metadata.generated.schema.entity.data.dashboard import Dashboard
+from metadata.generated.schema.entity.domains.dataProduct import DataProduct
+from metadata.generated.schema.entity.domains.domain import Domain, DomainType
+from metadata.generated.schema.entity.services.connections.dashboard.lookerConnection import (
+    LookerConnection,
+)
+from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import (
+    OpenMetadataConnection,
+)
+from metadata.generated.schema.entity.services.dashboardService import (
+    DashboardConnection,
+    DashboardService,
+    DashboardServiceType,
+)
+from metadata.generated.schema.security.client.openMetadataJWTClientConfig import (
+    OpenMetadataJWTClientConfig,
+)
+from metadata.generated.schema.type.entityReference import EntityReference
+from metadata.ingestion.ometa.ometa_api import OpenMetadata
+
+
+class OMetaDomainTest(TestCase):
+    """
+    Run this integration test with the local API available
+    Install the ingestion package before running the tests
+    """
+
+    service_entity_id = None
+
+    server_config = OpenMetadataConnection(
+        hostPort="http://localhost:8585/api",
+        authProvider="openmetadata",
+        securityConfig=OpenMetadataJWTClientConfig(
+            jwtToken="eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg"
+        ),
+    )
+    metadata = OpenMetadata(server_config)
+
+    assert metadata.health_check()
+
+    user = metadata.create_or_update(
+        data=CreateUserRequest(name="random-user", email="random@user.com"),
+    )
+    owner = EntityReference(id=user.id, type="user")
+
+    service = CreateDashboardServiceRequest(
+        name="test-service-dashboard",
+        serviceType=DashboardServiceType.Looker,
+        connection=DashboardConnection(
+            config=LookerConnection(
+                hostPort="http://hostPort", clientId="id", clientSecret="secret"
+            )
+        ),
+    )
+    service_type = "dashboardService"
+
+    create_domain = CreateDomainRequest(
+        domainType=DomainType.Consumer_aligned, name="TestDomain", description="random"
+    )
+
+    create_data_product = CreateDataProductRequest(
+        name="TestDataProduct",
+        description="random",
+        domain="TestDomain",
+    )
+
+    @classmethod
+    def setUpClass(cls) -> None:
+        """
+        Prepare ingredients
+        """
+        cls.service_entity = cls.metadata.create_or_update(data=cls.service)
+
+        cls.dashboard: Dashboard = cls.metadata.create_or_update(
+            CreateDashboardRequest(
+                name="test",
+                service=cls.service_entity.fullyQualifiedName,
+            )
+        )
+
+    @classmethod
+    def tearDownClass(cls) -> None:
+        """
+        Clean up
+        """
+
+        service_id = str(
+            cls.metadata.get_by_name(
+                entity=DashboardService, fqn=cls.service.name.__root__
+            ).id.__root__
+        )
+
+        cls.metadata.delete(
+            entity=DashboardService,
+            entity_id=service_id,
+            recursive=True,
+            hard_delete=True,
+        )
+
+        domain: Domain = cls.metadata.get_by_name(
+            entity=Domain, fqn=cls.create_domain.name.__root__
+        )
+
+        cls.metadata.delete(
+            entity=Domain, entity_id=domain.id, recursive=True, hard_delete=True
+        )
+
+    def test_create(self):
+        """
+        We can create a Domain and we receive it back as Entity
+        """
+
+        res: Domain = self.metadata.create_or_update(data=self.create_domain)
+        self.assertEquals(res.name, self.create_domain.name)
+        self.assertEquals(res.description, self.create_domain.description)
+
+        res: DataProduct = self.metadata.create_or_update(data=self.create_data_product)
+        self.assertEquals(res.name, self.create_data_product.name)
+        self.assertEquals(res.description, self.create_data_product.description)
+        self.assertEquals(res.domain.name, self.create_data_product.domain.__root__)
+
+    def test_get_name(self):
+        """We can fetch Domains & Data Products by name"""
+        self.metadata.create_or_update(data=self.create_domain)
+
+        res: Domain = self.metadata.get_by_name(
+            entity=Domain, fqn=self.create_domain.name.__root__
+        )
+        self.assertEqual(res.name, self.create_domain.name)
+
+        self.metadata.create_or_update(data=self.create_data_product)
+
+        res: DataProduct = self.metadata.get_by_name(
+            entity=DataProduct, fqn=self.create_data_product.name.__root__
+        )
+        self.assertEqual(res.name, self.create_data_product.name)
+
+    def test_get_id(self):
+        """We can fetch Domains & Data Products by ID"""
+        self.metadata.create_or_update(data=self.create_domain)
+
+        res_name: Domain = self.metadata.get_by_name(
+            entity=Domain, fqn=self.create_domain.name.__root__
+        )
+        res: Domain = self.metadata.get_by_id(entity=Domain, entity_id=res_name.id)
+        self.assertEqual(res.name, self.create_domain.name)
+
+        self.metadata.create_or_update(data=self.create_data_product)
+
+        res_name: DataProduct = self.metadata.get_by_name(
+            entity=DataProduct, fqn=self.create_data_product.name.__root__
+        )
+        res: DataProduct = self.metadata.get_by_id(
+            entity=DataProduct, entity_id=res_name.id
+        )
+        self.assertEqual(res.name, self.create_data_product.name)
+
+    def test_patch_domain(self):
+        """We can add domain to an asset"""
+        domain: Domain = self.metadata.create_or_update(data=self.create_domain)
+        self.metadata.patch_domain(entity=self.dashboard, domain=domain)
+
+        updated_dashboard: Dashboard = self.metadata.get_by_name(
+            entity=Dashboard, fqn=self.dashboard.fullyQualifiedName, fields=["domain"]
+        )
+
+        self.assertEquals(updated_dashboard.domain.name, domain.name.__root__)

openmetadata-spec/src/main/resources/json/schema/api/domains/createDataProduct.json

@@ -1,7 +1,7 @@
 {
   "$id": "https://open-metadata.org/schema/entity/domains/createDataProduct.json",
   "$schema": "http://json-schema.org/draft-07/schema#",
-  "title": "createDataProduct",
+  "title": "CreateDataProductRequest",
   "description": "Create DataProduct API request",
   "type": "object",
   "javaType": "org.openmetadata.schema.api.domains.CreateDataProduct",

openmetadata-spec/src/main/resources/json/schema/api/domains/createDomain.json

@@ -1,7 +1,7 @@
 {
   "$id": "https://open-metadata.org/schema/entity/domains/createDomain.json",
   "$schema": "http://json-schema.org/draft-07/schema#",
-  "title": "createDomain",
+  "title": "CreateDomainRequest",
   "description": "Create Domain API request",
   "type": "object",
   "javaType": "org.openmetadata.schema.api.domains.CreateDomain",