diff --git a/ingestion/setup.py b/ingestion/setup.py index e21dc55e34f..06f6dd90290 100644 --- a/ingestion/setup.py +++ b/ingestion/setup.py @@ -9,6 +9,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +""" +Python Dependencies +""" + import os from typing import Dict, Set @@ -17,8 +21,8 @@ from setuptools import find_namespace_packages, setup def get_long_description(): root = os.path.dirname(__file__) - with open(os.path.join(root, "README.md")) as f: - description = f.read() + with open(os.path.join(root, "README.md"), encoding="UTF-8") as file: + description = file.read() return description diff --git a/ingestion/src/metadata/ingestion/source/metadata/atlas/metadata.py b/ingestion/src/metadata/ingestion/source/metadata/atlas/metadata.py index cd9916ebd6f..ef5b992bd60 100644 --- a/ingestion/src/metadata/ingestion/source/metadata/atlas/metadata.py +++ b/ingestion/src/metadata/ingestion/source/metadata/atlas/metadata.py @@ -207,6 +207,7 @@ class AtlasSource(Source): for table in entity: table_entity = self.atlas_client.get_entity(table) tbl_entities = table_entity["entities"] + db_entity = None for tbl_entity in tbl_entities: try: diff --git a/ingestion/tests/unit/resources/datasets/atlas_dataset.json b/ingestion/tests/unit/resources/datasets/atlas_dataset.json new file mode 100644 index 00000000000..71a965aea03 --- /dev/null +++ b/ingestion/tests/unit/resources/datasets/atlas_dataset.json @@ -0,0 +1,461 @@ +{ + "referredEntities":{ + "e82a090a-0f21-4694-aae5-11eac90ada6d":{ + "typeName":"Column", + "attributes":{ + "owner":"None", + "replicatedTo":"None", + "userDescription":"None", + "replicatedFrom":"None", + "qualifiedName":"time_id@cl1", + "displayName":"None", + "dataType":"int", + "name":"time_id", + "description":"None", + "comment":"time id" + }, + "guid":"e82a090a-0f21-4694-aae5-11eac90ada6d", + "isIncomplete":false, + "status":"ACTIVE", + "createdBy":"admin", + "updatedBy":"admin", + "createTime":1650276602280, + "updateTime":1650276617480, + "version":0, + "relationshipAttributes":{ + "inputToProcesses":[ + + ], + "schema":[ + + ], + "meanings":[ + + ], + "table":{ + "guid":"fc4eae96-a7ff-4139-8bd0-3f7f8e59761b", + "typeName":"Table", + "entityStatus":"ACTIVE", + "displayText":"logging_fact_monthly_mv", + "relationshipType":"Table_Columns", + "relationshipGuid":"bd5b2145-237f-421b-b0c0-92a8da045d4f", + "relationshipStatus":"ACTIVE", + "relationshipAttributes":{ + "typeName":"Table_Columns" + } + }, + "outputFromProcesses":[ + + ] + }, + "labels":[ + + ] + }, + "f0f82283-d4af-413f-ad4a-eb908f184e85":{ + "typeName":"Column", + "attributes":{ + "owner":"None", + "replicatedTo":"None", + "userDescription":"None", + "replicatedFrom":"None", + "qualifiedName":"customer_id@cl1", + "displayName":"None", + "dataType":"int", + "name":"customer_id", + "description":"None", + "comment":"customer id" + }, + "guid":"f0f82283-d4af-413f-ad4a-eb908f184e85", + "isIncomplete":false, + "status":"ACTIVE", + "createdBy":"admin", + "updatedBy":"admin", + "createTime":1650276602280, + "updateTime":1650276617480, + "version":0, + "relationshipAttributes":{ + "inputToProcesses":[ + + ], + "schema":[ + + ], + "meanings":[ + + ], + "table":{ + "guid":"aabdc6c0-96c1-451c-98b5-7a7e48c319be", + "typeName":"Table", + "entityStatus":"ACTIVE", + "displayText":"customer_dim", + "relationshipType":"Table_Columns", + "relationshipGuid":"66022976-9fb6-492d-8389-170082212e11", + "relationshipStatus":"ACTIVE", + "relationshipAttributes":{ + "typeName":"Table_Columns" + } + }, + "outputFromProcesses":[ + + ] + }, + "classifications":[ + { + "typeName":"PII", + "entityGuid":"f0f82283-d4af-413f-ad4a-eb908f184e85", + "entityStatus":"ACTIVE", + "propagate":true, + "removePropagationsOnEntityDelete":false + } + ], + "labels":[ + + ] + }, + "6e389306-3fde-4d1b-bf1a-00940ea69414":{ + "typeName":"Column", + "attributes":{ + "owner":"None", + "replicatedTo":"None", + "userDescription":"None", + "replicatedFrom":"None", + "qualifiedName":"sales@cl1", + "displayName":"None", + "dataType":"double", + "name":"sales", + "description":"None", + "comment":"product id" + }, + "guid":"6e389306-3fde-4d1b-bf1a-00940ea69414", + "isIncomplete":false, + "status":"ACTIVE", + "createdBy":"admin", + "updatedBy":"admin", + "createTime":1650276602280, + "updateTime":1650276617480, + "version":0, + "relationshipAttributes":{ + "inputToProcesses":[ + + ], + "schema":[ + + ], + "meanings":[ + + ], + "table":{ + "guid":"84135ba4-03ee-4f2e-ae55-29aa2682a699", + "typeName":"Table", + "entityStatus":"ACTIVE", + "displayText":"sales_fact_monthly_mv", + "relationshipType":"Table_Columns", + "relationshipGuid":"8642936e-6112-4814-a538-07e1dc2db00e", + "relationshipStatus":"ACTIVE", + "relationshipAttributes":{ + "typeName":"Table_Columns" + } + }, + "outputFromProcesses":[ + + ] + }, + "classifications":[ + { + "typeName":"Metric", + "entityGuid":"6e389306-3fde-4d1b-bf1a-00940ea69414", + "entityStatus":"ACTIVE", + "propagate":true, + "removePropagationsOnEntityDelete":false + } + ], + "labels":[ + + ] + }, + "e23e6fe3-e772-40bf-baa5-a738e7d00cc3":{ + "typeName":"StorageDesc", + "attributes":{ + "owner":"None", + "replicatedTo":"None", + "userDescription":"None", + "replicatedFrom":"None", + "qualifiedName":"sd:hdfs://host:8000/apps/warehouse/sales@cl1", + "displayName":"None", + "name":"sd:hdfs://host:8000/apps/warehouse/sales", + "description":"None", + "location":"hdfs://host:8000/apps/warehouse/sales", + "compressed":true, + "inputFormat":"TextInputFormat", + "outputFormat":"TextOutputFormat" + }, + "guid":"e23e6fe3-e772-40bf-baa5-a738e7d00cc3", + "isIncomplete":false, + "status":"ACTIVE", + "createdBy":"admin", + "updatedBy":"admin", + "createTime":1650276602280, + "updateTime":1650276617480, + "version":0, + "relationshipAttributes":{ + "inputToProcesses":[ + + ], + "schema":[ + + ], + "meanings":[ + + ], + "table":{ + "guid":"fc4eae96-a7ff-4139-8bd0-3f7f8e59761b", + "typeName":"Table", + "entityStatus":"ACTIVE", + "displayText":"logging_fact_monthly_mv", + "relationshipType":"Table_StorageDesc", + "relationshipGuid":"0e5181e7-a06b-4e03-898a-93eb9cb0d1eb", + "relationshipStatus":"ACTIVE", + "relationshipAttributes":{ + "typeName":"Table_StorageDesc" + } + }, + "outputFromProcesses":[ + + ] + }, + "labels":[ + + ] + }, + "cf2425df-dd64-49b1-8b9f-b3726e9ce11d":{ + "typeName":"Column", + "attributes":{ + "owner":"None", + "replicatedTo":"None", + "userDescription":"None", + "replicatedFrom":"None", + "qualifiedName":"product_id@cl1", + "displayName":"None", + "dataType":"int", + "name":"product_id", + "description":"None", + "comment":"product id" + }, + "guid":"cf2425df-dd64-49b1-8b9f-b3726e9ce11d", + "isIncomplete":false, + "status":"ACTIVE", + "createdBy":"admin", + "updatedBy":"admin", + "createTime":1650276602280, + "updateTime":1650276617480, + "version":0, + "relationshipAttributes":{ + "inputToProcesses":[ + + ], + "schema":[ + + ], + "meanings":[ + + ], + "table":{ + "guid":"1697e9bc-2fb0-4d11-8294-7e0da9c74830", + "typeName":"Table", + "entityStatus":"ACTIVE", + "displayText":"product_dim", + "relationshipType":"Table_Columns", + "relationshipGuid":"fea21306-ae96-4e2e-85f0-b121e967969c", + "relationshipStatus":"ACTIVE", + "relationshipAttributes":{ + "typeName":"Table_Columns" + } + }, + "outputFromProcesses":[ + + ] + }, + "labels":[ + + ] + } + }, + "entities":[ + { + "typeName":"Table", + "attributes":{ + "owner":"Joe BI", + "temporary":false, + "lastAccessTime":1650276616988, + "replicatedTo":"None", + "userDescription":"None", + "replicatedFrom":"None", + "qualifiedName":"sales_fact_daily_mv@cl1", + "displayName":"None", + "description":"sales fact daily materialized view", + "viewExpandedText":"None", + "tableType":"Managed", + "createTime":1650276616988, + "name":"sales_fact_daily_mv", + "retention":1650276616988, + "viewOriginalText":"None" + }, + "guid":"eff274ef-f84d-4c58-81d6-c309663b887d", + "isIncomplete":false, + "status":"ACTIVE", + "createdBy":"admin", + "updatedBy":"admin", + "createTime":1650276616997, + "updateTime":1650276619535, + "version":0, + "relationshipAttributes":{ + "inputToProcesses":[ + { + "guid":"2194856a-9c10-40a1-8e85-8460e53cf15b", + "typeName":"LoadProcess", + "entityStatus":"ACTIVE", + "displayText":"loadSalesMonthly", + "relationshipType":"dataset_process_inputs", + "relationshipGuid":"77bb8a16-61d8-48f5-a8fb-b61c493eb008", + "relationshipStatus":"ACTIVE", + "relationshipAttributes":{ + "typeName":"dataset_process_inputs" + } + } + ], + "schema":[ + + ], + "sd":{ + "guid":"e23e6fe3-e772-40bf-baa5-a738e7d00cc3", + "typeName":"StorageDesc", + "entityStatus":"ACTIVE", + "displayText":"sd:hdfs://host:8000/apps/warehouse/sales", + "relationshipType":"Table_StorageDesc", + "relationshipGuid":"8da6f6d8-9998-425a-b409-f267ebf66631", + "relationshipStatus":"DELETED", + "relationshipAttributes":{ + "typeName":"Table_StorageDesc" + } + }, + "columns":[ + { + "guid":"e82a090a-0f21-4694-aae5-11eac90ada6d", + "typeName":"Column", + "entityStatus":"ACTIVE", + "displayText":"time_id", + "relationshipType":"Table_Columns", + "relationshipGuid":"fcc25fe7-da42-4c39-abf6-ee862c96bf39", + "relationshipStatus":"DELETED", + "relationshipAttributes":{ + "typeName":"Table_Columns" + } + }, + { + "guid":"f0f82283-d4af-413f-ad4a-eb908f184e85", + "typeName":"Column", + "entityStatus":"ACTIVE", + "displayText":"customer_id", + "relationshipType":"Table_Columns", + "relationshipGuid":"3f062158-b7db-4a8a-b92e-c2f8aec4944f", + "relationshipStatus":"DELETED", + "relationshipAttributes":{ + "typeName":"Table_Columns" + } + }, + { + "guid":"6e389306-3fde-4d1b-bf1a-00940ea69414", + "typeName":"Column", + "entityStatus":"ACTIVE", + "displayText":"sales", + "relationshipType":"Table_Columns", + "relationshipGuid":"8a2aa73f-b76e-4a71-a99f-36d6b5145c6e", + "relationshipStatus":"DELETED", + "relationshipAttributes":{ + "typeName":"Table_Columns" + } + }, + { + "guid":"cf2425df-dd64-49b1-8b9f-b3726e9ce11d", + "typeName":"Column", + "entityStatus":"ACTIVE", + "displayText":"product_id", + "relationshipType":"Table_Columns", + "relationshipGuid":"e8f45f67-b7c2-48fb-a59a-e23067c2300a", + "relationshipStatus":"DELETED", + "relationshipAttributes":{ + "typeName":"Table_Columns" + } + } + ], + "meanings":[ + + ], + "db":{ + "guid":"388f1d3f-31d4-4ee5-898d-c35e6e752b38", + "typeName":"DB", + "entityStatus":"ACTIVE", + "displayText":"Reporting", + "description": "THIS IS TEST_DESCRIPTION FOR DATABASE", + "relationshipType":"Table_DB", + "relationshipGuid":"cdbddf6a-ec6f-4263-9238-38281d73b4ea", + "relationshipStatus":"ACTIVE", + "relationshipAttributes":{ + "typeName":"Table_DB" + } + }, + "outputFromProcesses":[ + { + "guid":"6d6ab7e5-646f-4161-937c-dc340d7c9d06", + "typeName":"LoadProcess", + "entityStatus":"ACTIVE", + "displayText":"loadSalesDaily", + "relationshipType":"process_dataset_outputs", + "relationshipGuid":"2573b9b2-82e6-4089-b3f1-8b0b1b378491", + "relationshipStatus":"ACTIVE", + "relationshipAttributes":{ + "typeName":"process_dataset_outputs" + } + } + ] + }, + "classifications":[ + { + "typeName":"Metric", + "entityGuid":"eff274ef-f84d-4c58-81d6-c309663b887d", + "entityStatus":"ACTIVE", + "propagate":true, + "removePropagationsOnEntityDelete":false + }, + { + "typeName":"Fact", + "entityGuid":"b233b2ae-8a4a-44a3-b446-4027462b2cc6", + "entityStatus":"ACTIVE", + "propagate":true, + "removePropagationsOnEntityDelete":false + }, + { + "typeName":"Dimension", + "attributes":{ + "customers":"None" + }, + "entityGuid":"67101f5b-205a-4fd8-8273-273ccb45ca74", + "entityStatus":"ACTIVE", + "propagate":true, + "removePropagationsOnEntityDelete":false + }, + { + "typeName":"ETL", + "entityGuid":"6d6ab7e5-646f-4161-937c-dc340d7c9d06", + "entityStatus":"ACTIVE", + "propagate":true, + "removePropagationsOnEntityDelete":false + } + ], + "labels":[ + + ] + } + ] + } \ No newline at end of file diff --git a/ingestion/tests/unit/topology/metadata/test_atlas.py b/ingestion/tests/unit/topology/metadata/test_atlas.py new file mode 100644 index 00000000000..842f27c921b --- /dev/null +++ b/ingestion/tests/unit/topology/metadata/test_atlas.py @@ -0,0 +1,506 @@ +# Copyright 2021 Collate +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Test Atlas using the topology +""" +import json +from pathlib import Path +from unittest import TestCase +from unittest.mock import patch + +from pydantic import AnyUrl + +from metadata.generated.schema.api.classification.createClassification import ( + CreateClassificationRequest, +) +from metadata.generated.schema.api.classification.createTag import CreateTagRequest +from metadata.generated.schema.api.data.createDatabase import CreateDatabaseRequest +from metadata.generated.schema.api.data.createDatabaseSchema import ( + CreateDatabaseSchemaRequest, +) +from metadata.generated.schema.api.data.createTable import CreateTableRequest +from metadata.generated.schema.api.services.createDatabaseService import ( + CreateDatabaseServiceRequest, +) +from metadata.generated.schema.entity.data.database import Database +from metadata.generated.schema.entity.data.databaseSchema import DatabaseSchema +from metadata.generated.schema.entity.data.table import Column, Table +from metadata.generated.schema.entity.services.connections.database.hiveConnection import ( + HiveConnection, +) +from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import ( + OpenMetadataConnection, +) +from metadata.generated.schema.entity.services.databaseService import ( + DatabaseConnection, + DatabaseService, +) +from metadata.generated.schema.metadataIngestion.workflow import ( + OpenMetadataWorkflowConfig, +) +from metadata.generated.schema.type.basic import Href +from metadata.generated.schema.type.entityReference import EntityReference +from metadata.generated.schema.type.tagLabel import TagLabel +from metadata.ingestion.ometa.ometa_api import OpenMetadata +from metadata.ingestion.source.metadata.atlas.client import AtlasClient +from metadata.ingestion.source.metadata.atlas.metadata import AtlasSource + +mock_atlas_config = { + "source": { + "type": "Atlas", + "serviceName": "local_atlas", + "serviceConnection": { + "config": { + "type": "Atlas", + "hostPort": "http://192.168.1.8:21000", + "username": "username", + "password": "password", + "databaseServiceName": ["hive"], + "messagingServiceName": [], + } + }, + "sourceConfig": {"config": {"type": "DatabaseMetadata"}}, + }, + "sink": {"type": "metadata-rest", "config": {}}, + "workflowConfig": { + "openMetadataServerConfig": { + "hostPort": "http://localhost:8585/api", + "authProvider": "openmetadata", + "securityConfig": { + "jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGc" + "iOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE" + "2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXB" + "iEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fN" + "r3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3u" + "d-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg" + }, + } + }, +} + + +mock_file_path = ( + Path(__file__).parent.parent.parent / "resources/datasets/atlas_dataset.json" +) +with open(mock_file_path, encoding="UTF-8") as file: + mock_data: dict = json.load(file) + + +LIST_ENTITIES = [ + "b233b2ae-8a4a-44a3-b446-4027462b2cc6", + "eff274ef-f84d-4c58-81d6-c309663b887d", + "1697e9bc-2fb0-4d11-8294-7e0da9c74830", + "aabdc6c0-96c1-451c-98b5-7a7e48c319be", +] + + +def mock_get_entity(self, table): # pylint: disable=unused-argument + return mock_data + + +def mock_list_entities(self, entity_type): # pylint: disable=unused-argument + return LIST_ENTITIES + + +EXPECTED_DATABASE_DESCRIPTION = "THIS IS TEST_DESCRIPTION FOR DATABASE" + +EXPTECTED_DATABASE_SCHEMA_DESCRIPTION = "THIS IS TEST_DESCRIPTION FOR DATABASE" + +EXPTECTED_TABLE = Table( + id="124d078d-dcf2-43a8-b59e-33bc7953f680", + name="delta_test_table", + displayName=None, + fullyQualifiedName="hive.Reporting.Reporting.delta_test_table", + description="sales fact daily materialized view", + version=0.1, + updatedAt=1673413042524, + updatedBy="admin", + href=Href( + __root__=AnyUrl( + "http://localhost:8585/api/v1/tables/124d078d-dcf2-43a8-b59e-33bc7953f680", + scheme="http", + host="localhost", + host_type="int_domain", + port="8585", + path="/api/v1/tables/124d078d-dcf2-43a8-b59e-33bc7953f680", + ) + ), + tableType="Regular", + columns=[ + Column( + name="col2", + displayName=None, + dataType="STRUCT", + arrayDataType=None, + dataLength=1, + precision=None, + scale=None, + dataTypeDisplay="struct", + description="col2 nested delta column", + fullyQualifiedName="hive.Reporting.Reporting.delta_test_table.col2", + tags=None, + constraint=None, + ordinalPosition=None, + jsonSchema=None, + children=[ + Column( + name="col3", + displayName=None, + dataType="STRING", + arrayDataType=None, + dataLength=None, + precision=None, + scale=None, + dataTypeDisplay="string", + description=None, + fullyQualifiedName="hive.Reporting.Reporting.delta_test_table.col2.col3", + tags=None, + constraint=None, + ordinalPosition=None, + jsonSchema=None, + children=None, + customMetrics=None, + profile=None, + ), + Column( + name="col4", + displayName=None, + dataType="BIGINT", + arrayDataType=None, + dataLength=None, + precision=None, + scale=None, + dataTypeDisplay="bigint", + description=None, + fullyQualifiedName="hive.Reporting.Reporting.delta_test_table.col2.col4", + tags=None, + constraint=None, + ordinalPosition=None, + jsonSchema=None, + children=None, + customMetrics=None, + profile=None, + ), + ], + customMetrics=None, + profile=None, + ), + Column( + name="col1", + displayName=None, + dataType="BIGINT", + arrayDataType=None, + dataLength=1, + precision=None, + scale=None, + dataTypeDisplay="bigint", + description="col1 description", + fullyQualifiedName="hive.Reporting.Reporting.delta_test_table.col1", + tags=None, + constraint=None, + ordinalPosition=None, + jsonSchema=None, + children=None, + customMetrics=None, + profile=None, + ), + ], + tableConstraints=None, + tablePartition=None, + owner=None, + databaseSchema=EntityReference( + id="4cf6ee7e-9d24-4153-9318-82aa1167259b", + type="databaseSchema", + name="Reporting", + fullyQualifiedName="hive.Reporting.Reporting", + description="THIS IS TEST_DESCRIPTION FOR DATABASE", + displayName=None, + deleted=False, + href=Href( + __root__=AnyUrl( + "http://localhost:8585/api/v1/databaseSchemas/4cf6ee7e-9d24-4153-9318-82aa1167259b", + scheme="http", + host="localhost", + host_type="int_domain", + port="8585", + path="/api/v1/databaseSchemas/4cf6ee7e-9d24-4153-9318-82aa1167259b", + ) + ), + ), + database=EntityReference( + id="367f53b5-d6c2-44be-bf5d-a0a1dc98a9dd", + type="database", + name="Reporting", + fullyQualifiedName="hive.Reporting", + description="THIS IS TEST_DESCRIPTION FOR DATABASE", + displayName=None, + deleted=False, + href=Href( + __root__=AnyUrl( + "http://localhost:8585/api/v1/databases/367f53b5-d6c2-44be-bf5d-a0a1dc98a9dd", + scheme="http", + host="localhost", + host_type="int_domain", + port="8585", + path="/api/v1/databases/367f53b5-d6c2-44be-bf5d-a0a1dc98a9dd", + ) + ), + ), + service=EntityReference( + id="f2ab0e7a-5224-4acb-a189-74158851733f", + type="databaseService", + name="hive", + fullyQualifiedName="hive", + description=None, + displayName=None, + deleted=False, + href=Href( + __root__=AnyUrl( + "http://localhost:8585/api/v1/services/databaseServices/f2ab0e7a-5224-4acb-a189-74158851733f", + scheme="http", + host="localhost", + host_type="int_domain", + port="8585", + path="/api/v1/services/databaseServices/f2ab0e7a-5224-4acb-a189-74158851733f", + ) + ), + ), + serviceType="Hive", + location=None, + viewDefinition=None, + tags=[ + TagLabel( + tagFQN="AtlasMetadata.atlas_table", + description="test tag", + source="Tag", + labelType="Automated", + state="Confirmed", + href=None, + ) + ], + usageSummary=None, + followers=None, + joins=None, + sampleData=None, + tableProfilerConfig=None, + profile=None, + tableQueries=None, + dataModel=None, + changeDescription=None, + deleted=False, + extension=None, +) + + +class AtlasUnitTest(TestCase): + """ + Implements the necessary methods to extract + Atlas Metadata Unit Test + """ + + @patch( + "metadata.ingestion.source.pipeline.pipeline_service.PipelineServiceSource.test_connection" + ) + def __init__(self, methodName, test_connection) -> None: + super().__init__(methodName) + test_connection.return_value = False + self.config = OpenMetadataWorkflowConfig.parse_obj(mock_atlas_config) + self.atlas_source = AtlasSource.create( + mock_atlas_config["source"], + self.config.workflowConfig.openMetadataServerConfig, + ) + self.metadata = OpenMetadata( + OpenMetadataConnection.parse_obj( + mock_atlas_config["workflowConfig"]["openMetadataServerConfig"] + ) + ) + + self.database_service = ( + mock_database_service_object + ) = self.metadata.create_or_update( + CreateDatabaseServiceRequest( + name="hive", + serviceType="Hive", + connection=DatabaseConnection( + config=HiveConnection( + type="Hive", + scheme="hive", + username=None, + password=None, + hostPort="http://nohost:6000", + ) + ), + ) + ) + + mock_database_object = self.metadata.create_or_update( + CreateDatabaseRequest( + name="Reporting", + displayName=None, + description=None, + tags=None, + owner=None, + service=EntityReference( + id=mock_database_service_object.id, + type="databaseService", + ), + ) + ) + mock_database_schema_object = self.metadata.create_or_update( + CreateDatabaseSchemaRequest( + name="Reporting", + database=EntityReference(id=mock_database_object.id, type="database"), + ) + ) + _ = self.metadata.create_or_update( + CreateTableRequest( + name="sales_fact_daily_mv", + tableType="Regular", + columns=[ + Column( + name="col2", + displayName=None, + dataType="STRUCT", + arrayDataType=None, + dataLength=1, + precision=None, + scale=None, + dataTypeDisplay="struct", + description="col2 nested delta column", + fullyQualifiedName="delta.default.test_schema.delta_test_table.col2", + tags=None, + constraint=None, + ordinalPosition=None, + jsonSchema=None, + children=[ + Column( + name="col3", + displayName=None, + dataType="STRING", + arrayDataType=None, + dataLength=None, + precision=None, + scale=None, + dataTypeDisplay="string", + description=None, + fullyQualifiedName="delta.default.test_schema.delta_test_table.col2.col3", + tags=None, + constraint=None, + ordinalPosition=None, + jsonSchema=None, + children=None, + customMetrics=None, + profile=None, + ), + Column( + name="col4", + displayName=None, + dataType="BIGINT", + arrayDataType=None, + dataLength=None, + precision=None, + scale=None, + dataTypeDisplay="bigint", + description=None, + fullyQualifiedName="delta.default.test_schema.delta_test_table.col2.col4", + tags=None, + constraint=None, + ordinalPosition=None, + jsonSchema=None, + children=None, + customMetrics=None, + profile=None, + ), + ], + customMetrics=None, + profile=None, + ), + Column( + name="col1", + displayName=None, + dataType="BIGINT", + arrayDataType=None, + dataLength=1, + precision=None, + scale=None, + dataTypeDisplay="bigint", + description="col1 description", + fullyQualifiedName="delta.default.test_schema.delta_test_table.col1", + tags=None, + constraint=None, + ordinalPosition=None, + jsonSchema=None, + children=None, + customMetrics=None, + profile=None, + ), + ], + databaseSchema=EntityReference( + id=mock_database_schema_object.id, type="databaseSchema" + ), + ), + ) + + def mock_ingest_lineage(self, source_guid, name): # pylint: disable=unused-argument + return [] + + def mock_create_tag(self): + classification = CreateClassificationRequest( + description="test tag", name="AtlasMetadata" + ) + + self.metadata.create_or_update(classification) + self.metadata.create_or_update( + CreateTagRequest( + classification="AtlasMetadata", + name="atlas_table", + description="test tag", + ) + ) + + @patch.object(AtlasClient, "list_entities", mock_list_entities) + @patch.object(AtlasClient, "get_entity", mock_get_entity) + @patch.object(AtlasSource, "ingest_lineage", mock_ingest_lineage) + @patch.object(AtlasSource, "create_tag", mock_create_tag) + def test_description(self): + """ + Testing description updated for database, databaseSchema, table + """ + _ = list(self.atlas_source.next_record()) + updated_database = self.metadata.get_by_name( + entity=Database, fqn="hive.Reporting" + ) + assert updated_database.description.__root__ == EXPECTED_DATABASE_DESCRIPTION + + updated_database_schema = self.metadata.get_by_name( + entity=DatabaseSchema, fqn="hive.Reporting.Reporting" + ) + assert ( + updated_database_schema.description.__root__ + == EXPTECTED_DATABASE_SCHEMA_DESCRIPTION + ) + + updated_table = self.metadata.get_by_name( + entity=Table, + fqn="hive.Reporting.Reporting.sales_fact_daily_mv", + fields=["tags"], + ) + + assert updated_table.description == EXPTECTED_TABLE.description + + assert updated_table.tags == EXPTECTED_TABLE.tags + + self.metadata.delete( + entity=DatabaseService, + entity_id=self.atlas_source.service.id, + recursive=True, + hard_delete=True, + )