diff --git a/ingestion/src/metadata/ingestion/ometa/mixins/patch_mixin.py b/ingestion/src/metadata/ingestion/ometa/mixins/patch_mixin.py index a7f5a62517b..f9994858a94 100644 --- a/ingestion/src/metadata/ingestion/ometa/mixins/patch_mixin.py +++ b/ingestion/src/metadata/ingestion/ometa/mixins/patch_mixin.py @@ -25,6 +25,7 @@ from metadata.generated.schema.entity.automations.workflow import ( ) from metadata.generated.schema.entity.automations.workflow import WorkflowStatus from metadata.generated.schema.entity.data.table import Column, Table, TableConstraint +from metadata.generated.schema.entity.domains.domain import Domain from metadata.generated.schema.entity.services.connections.testConnectionResult import ( TestConnectionResult, ) @@ -485,7 +486,9 @@ class OMetaPatchMixin(OMetaPatchMixinBase): f"Error trying to PATCH status for automation workflow [{model_str(automation_workflow)}]: {exc}" ) - def patch_life_cycle(self, entity: Entity, life_cycle: LifeCycle) -> None: + def patch_life_cycle( + self, entity: Entity, life_cycle: LifeCycle + ) -> Optional[Entity]: """ Patch life cycle data for a entity @@ -495,9 +498,27 @@ class OMetaPatchMixin(OMetaPatchMixinBase): try: destination = entity.copy(deep=True) destination.lifeCycle = life_cycle - self.patch(entity=type(entity), source=entity, destination=destination) + return self.patch( + entity=type(entity), source=entity, destination=destination + ) except Exception as exc: logger.debug(traceback.format_exc()) logger.warning( f"Error trying to Patch life cycle data for {entity.fullyQualifiedName.__root__}: {exc}" ) + return None + + def patch_domain(self, entity: Entity, domain: Domain) -> Optional[Entity]: + """Patch domain data for an Entity""" + try: + destination: Entity = entity.copy(deep=True) + destination.domain = EntityReference(id=domain.id, type="domain") + return self.patch( + entity=type(entity), source=entity, destination=destination + ) + except Exception as exc: + logger.debug(traceback.format_exc()) + logger.warning( + f"Error trying to Patch Domain for {entity.fullyQualifiedName.__root__}: {exc}" + ) + return None diff --git a/ingestion/src/metadata/ingestion/ometa/ometa_api.py b/ingestion/src/metadata/ingestion/ometa/ometa_api.py index 8e8df29101c..84fc0d65526 100644 --- a/ingestion/src/metadata/ingestion/ometa/ometa_api.py +++ b/ingestion/src/metadata/ingestion/ometa/ometa_api.py @@ -239,6 +239,7 @@ class OpenMetadata( .replace("searchindex", "searchIndex") .replace("storedprocedure", "storedProcedure") .replace("ingestionpipeline", "ingestionPipeline") + .replace("dataproduct", "dataProduct") ) class_path = ".".join( filter( diff --git a/ingestion/src/metadata/ingestion/ometa/routes.py b/ingestion/src/metadata/ingestion/ometa/routes.py index c8e6669c841..6cf70529ad0 100644 --- a/ingestion/src/metadata/ingestion/ometa/routes.py +++ b/ingestion/src/metadata/ingestion/ometa/routes.py @@ -46,6 +46,10 @@ from metadata.generated.schema.api.data.createStoredProcedure import ( ) from metadata.generated.schema.api.data.createTable import CreateTableRequest from metadata.generated.schema.api.data.createTopic import CreateTopicRequest +from metadata.generated.schema.api.domains.createDataProduct import ( + CreateDataProductRequest, +) +from metadata.generated.schema.api.domains.createDomain import CreateDomainRequest from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest from metadata.generated.schema.api.policies.createPolicy import CreatePolicyRequest from metadata.generated.schema.api.services.createDashboardService import ( @@ -107,6 +111,8 @@ from metadata.generated.schema.entity.data.searchIndex import SearchIndex from metadata.generated.schema.entity.data.storedProcedure import StoredProcedure from metadata.generated.schema.entity.data.table import Table from metadata.generated.schema.entity.data.topic import Topic +from metadata.generated.schema.entity.domains.dataProduct import DataProduct +from metadata.generated.schema.entity.domains.domain import Domain from metadata.generated.schema.entity.policies.policy import Policy from metadata.generated.schema.entity.services.connections.testConnectionDefinition import ( TestConnectionDefinition, @@ -214,4 +220,9 @@ ROUTES = { WebAnalyticEventData.__name__: "/analytics/web/events/collect", DataInsightChart.__name__: "/analytics/dataInsights/charts", Kpi.__name__: "/kpi", + # Domains & Data Products + Domain.__name__: "/domains", + CreateDomainRequest.__name__: "/domains", + DataProduct.__name__: "/dataProducts", + CreateDataProductRequest.__name__: "/dataProducts", } diff --git a/ingestion/tests/integration/ometa/test_ometa_domains_api.py b/ingestion/tests/integration/ometa/test_ometa_domains_api.py new file mode 100644 index 00000000000..032f08adbb6 --- /dev/null +++ b/ingestion/tests/integration/ometa/test_ometa_domains_api.py @@ -0,0 +1,192 @@ +# Copyright 2021 Collate +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +OpenMetadata high-level API Domains & Data Products test +""" +from unittest import TestCase + +from metadata.generated.schema.api.data.createDashboard import CreateDashboardRequest +from metadata.generated.schema.api.domains.createDataProduct import ( + CreateDataProductRequest, +) +from metadata.generated.schema.api.domains.createDomain import CreateDomainRequest +from metadata.generated.schema.api.services.createDashboardService import ( + CreateDashboardServiceRequest, +) +from metadata.generated.schema.api.teams.createUser import CreateUserRequest +from metadata.generated.schema.entity.data.dashboard import Dashboard +from metadata.generated.schema.entity.domains.dataProduct import DataProduct +from metadata.generated.schema.entity.domains.domain import Domain, DomainType +from metadata.generated.schema.entity.services.connections.dashboard.lookerConnection import ( + LookerConnection, +) +from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import ( + OpenMetadataConnection, +) +from metadata.generated.schema.entity.services.dashboardService import ( + DashboardConnection, + DashboardService, + DashboardServiceType, +) +from metadata.generated.schema.security.client.openMetadataJWTClientConfig import ( + OpenMetadataJWTClientConfig, +) +from metadata.generated.schema.type.entityReference import EntityReference +from metadata.ingestion.ometa.ometa_api import OpenMetadata + + +class OMetaDomainTest(TestCase): + """ + Run this integration test with the local API available + Install the ingestion package before running the tests + """ + + service_entity_id = None + + server_config = OpenMetadataConnection( + hostPort="http://localhost:8585/api", + authProvider="openmetadata", + securityConfig=OpenMetadataJWTClientConfig( + jwtToken="eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg" + ), + ) + metadata = OpenMetadata(server_config) + + assert metadata.health_check() + + user = metadata.create_or_update( + data=CreateUserRequest(name="random-user", email="random@user.com"), + ) + owner = EntityReference(id=user.id, type="user") + + service = CreateDashboardServiceRequest( + name="test-service-dashboard", + serviceType=DashboardServiceType.Looker, + connection=DashboardConnection( + config=LookerConnection( + hostPort="http://hostPort", clientId="id", clientSecret="secret" + ) + ), + ) + service_type = "dashboardService" + + create_domain = CreateDomainRequest( + domainType=DomainType.Consumer_aligned, name="TestDomain", description="random" + ) + + create_data_product = CreateDataProductRequest( + name="TestDataProduct", + description="random", + domain="TestDomain", + ) + + @classmethod + def setUpClass(cls) -> None: + """ + Prepare ingredients + """ + cls.service_entity = cls.metadata.create_or_update(data=cls.service) + + cls.dashboard: Dashboard = cls.metadata.create_or_update( + CreateDashboardRequest( + name="test", + service=cls.service_entity.fullyQualifiedName, + ) + ) + + @classmethod + def tearDownClass(cls) -> None: + """ + Clean up + """ + + service_id = str( + cls.metadata.get_by_name( + entity=DashboardService, fqn=cls.service.name.__root__ + ).id.__root__ + ) + + cls.metadata.delete( + entity=DashboardService, + entity_id=service_id, + recursive=True, + hard_delete=True, + ) + + domain: Domain = cls.metadata.get_by_name( + entity=Domain, fqn=cls.create_domain.name.__root__ + ) + + cls.metadata.delete( + entity=Domain, entity_id=domain.id, recursive=True, hard_delete=True + ) + + def test_create(self): + """ + We can create a Domain and we receive it back as Entity + """ + + res: Domain = self.metadata.create_or_update(data=self.create_domain) + self.assertEquals(res.name, self.create_domain.name) + self.assertEquals(res.description, self.create_domain.description) + + res: DataProduct = self.metadata.create_or_update(data=self.create_data_product) + self.assertEquals(res.name, self.create_data_product.name) + self.assertEquals(res.description, self.create_data_product.description) + self.assertEquals(res.domain.name, self.create_data_product.domain.__root__) + + def test_get_name(self): + """We can fetch Domains & Data Products by name""" + self.metadata.create_or_update(data=self.create_domain) + + res: Domain = self.metadata.get_by_name( + entity=Domain, fqn=self.create_domain.name.__root__ + ) + self.assertEqual(res.name, self.create_domain.name) + + self.metadata.create_or_update(data=self.create_data_product) + + res: DataProduct = self.metadata.get_by_name( + entity=DataProduct, fqn=self.create_data_product.name.__root__ + ) + self.assertEqual(res.name, self.create_data_product.name) + + def test_get_id(self): + """We can fetch Domains & Data Products by ID""" + self.metadata.create_or_update(data=self.create_domain) + + res_name: Domain = self.metadata.get_by_name( + entity=Domain, fqn=self.create_domain.name.__root__ + ) + res: Domain = self.metadata.get_by_id(entity=Domain, entity_id=res_name.id) + self.assertEqual(res.name, self.create_domain.name) + + self.metadata.create_or_update(data=self.create_data_product) + + res_name: DataProduct = self.metadata.get_by_name( + entity=DataProduct, fqn=self.create_data_product.name.__root__ + ) + res: DataProduct = self.metadata.get_by_id( + entity=DataProduct, entity_id=res_name.id + ) + self.assertEqual(res.name, self.create_data_product.name) + + def test_patch_domain(self): + """We can add domain to an asset""" + domain: Domain = self.metadata.create_or_update(data=self.create_domain) + self.metadata.patch_domain(entity=self.dashboard, domain=domain) + + updated_dashboard: Dashboard = self.metadata.get_by_name( + entity=Dashboard, fqn=self.dashboard.fullyQualifiedName, fields=["domain"] + ) + + self.assertEquals(updated_dashboard.domain.name, domain.name.__root__) diff --git a/openmetadata-spec/src/main/resources/json/schema/api/domains/createDataProduct.json b/openmetadata-spec/src/main/resources/json/schema/api/domains/createDataProduct.json index 5d20fb401c3..f9267b58769 100644 --- a/openmetadata-spec/src/main/resources/json/schema/api/domains/createDataProduct.json +++ b/openmetadata-spec/src/main/resources/json/schema/api/domains/createDataProduct.json @@ -1,7 +1,7 @@ { "$id": "https://open-metadata.org/schema/entity/domains/createDataProduct.json", "$schema": "http://json-schema.org/draft-07/schema#", - "title": "createDataProduct", + "title": "CreateDataProductRequest", "description": "Create DataProduct API request", "type": "object", "javaType": "org.openmetadata.schema.api.domains.CreateDataProduct", diff --git a/openmetadata-spec/src/main/resources/json/schema/api/domains/createDomain.json b/openmetadata-spec/src/main/resources/json/schema/api/domains/createDomain.json index 9a77fff9564..3c677fc067d 100644 --- a/openmetadata-spec/src/main/resources/json/schema/api/domains/createDomain.json +++ b/openmetadata-spec/src/main/resources/json/schema/api/domains/createDomain.json @@ -1,7 +1,7 @@ { "$id": "https://open-metadata.org/schema/entity/domains/createDomain.json", "$schema": "http://json-schema.org/draft-07/schema#", - "title": "createDomain", + "title": "CreateDomainRequest", "description": "Create Domain API request", "type": "object", "javaType": "org.openmetadata.schema.api.domains.CreateDomain",