mirror of
https://github.com/open-metadata/OpenMetadata.git
synced 2025-08-14 20:17:05 +00:00
Fix: added test case for atlas (#9678)
* Fix: added test case for atlas * Fix: resolved conflict * Fix: changing back neo4j to old version * Fix: changing back neo4j to old version * Fix: changes as per comment * Fix: changes as per comment * Fix: python checkstyle
This commit is contained in:
NiharDoshi99
GitHub
parent
35b9cef3fa
commit
029dbe892e
@ -9,6 +9,10 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""
|
||||
Python Dependencies
|
||||
"""
|
||||
|
||||
import os
|
||||
from typing import Dict, Set
|
||||
|
||||
@ -17,8 +21,8 @@ from setuptools import find_namespace_packages, setup
|
||||
|
||||
def get_long_description():
|
||||
root = os.path.dirname(__file__)
|
||||
with open(os.path.join(root, "README.md")) as f:
|
||||
description = f.read()
|
||||
with open(os.path.join(root, "README.md"), encoding="UTF-8") as file:
|
||||
description = file.read()
|
||||
return description
|
||||
|
||||
|
||||
|
||||
@ -207,6 +207,7 @@ class AtlasSource(Source):
|
||||
for table in entity:
|
||||
table_entity = self.atlas_client.get_entity(table)
|
||||
tbl_entities = table_entity["entities"]
|
||||
db_entity = None
|
||||
for tbl_entity in tbl_entities:
|
||||
try:
|
||||
|
||||
|
||||
461
ingestion/tests/unit/resources/datasets/atlas_dataset.json
Normal file
461
ingestion/tests/unit/resources/datasets/atlas_dataset.json
Normal file
@ -0,0 +1,461 @@
|
||||
{
|
||||
"referredEntities":{
|
||||
"e82a090a-0f21-4694-aae5-11eac90ada6d":{
|
||||
"typeName":"Column",
|
||||
"attributes":{
|
||||
"owner":"None",
|
||||
"replicatedTo":"None",
|
||||
"userDescription":"None",
|
||||
"replicatedFrom":"None",
|
||||
"qualifiedName":"time_id@cl1",
|
||||
"displayName":"None",
|
||||
"dataType":"int",
|
||||
"name":"time_id",
|
||||
"description":"None",
|
||||
"comment":"time id"
|
||||
},
|
||||
"guid":"e82a090a-0f21-4694-aae5-11eac90ada6d",
|
||||
"isIncomplete":false,
|
||||
"status":"ACTIVE",
|
||||
"createdBy":"admin",
|
||||
"updatedBy":"admin",
|
||||
"createTime":1650276602280,
|
||||
"updateTime":1650276617480,
|
||||
"version":0,
|
||||
"relationshipAttributes":{
|
||||
"inputToProcesses":[
|
||||
|
||||
],
|
||||
"schema":[
|
||||
|
||||
],
|
||||
"meanings":[
|
||||
|
||||
],
|
||||
"table":{
|
||||
"guid":"fc4eae96-a7ff-4139-8bd0-3f7f8e59761b",
|
||||
"typeName":"Table",
|
||||
"entityStatus":"ACTIVE",
|
||||
"displayText":"logging_fact_monthly_mv",
|
||||
"relationshipType":"Table_Columns",
|
||||
"relationshipGuid":"bd5b2145-237f-421b-b0c0-92a8da045d4f",
|
||||
"relationshipStatus":"ACTIVE",
|
||||
"relationshipAttributes":{
|
||||
"typeName":"Table_Columns"
|
||||
}
|
||||
},
|
||||
"outputFromProcesses":[
|
||||
|
||||
]
|
||||
},
|
||||
"labels":[
|
||||
|
||||
]
|
||||
},
|
||||
"f0f82283-d4af-413f-ad4a-eb908f184e85":{
|
||||
"typeName":"Column",
|
||||
"attributes":{
|
||||
"owner":"None",
|
||||
"replicatedTo":"None",
|
||||
"userDescription":"None",
|
||||
"replicatedFrom":"None",
|
||||
"qualifiedName":"customer_id@cl1",
|
||||
"displayName":"None",
|
||||
"dataType":"int",
|
||||
"name":"customer_id",
|
||||
"description":"None",
|
||||
"comment":"customer id"
|
||||
},
|
||||
"guid":"f0f82283-d4af-413f-ad4a-eb908f184e85",
|
||||
"isIncomplete":false,
|
||||
"status":"ACTIVE",
|
||||
"createdBy":"admin",
|
||||
"updatedBy":"admin",
|
||||
"createTime":1650276602280,
|
||||
"updateTime":1650276617480,
|
||||
"version":0,
|
||||
"relationshipAttributes":{
|
||||
"inputToProcesses":[
|
||||
|
||||
],
|
||||
"schema":[
|
||||
|
||||
],
|
||||
"meanings":[
|
||||
|
||||
],
|
||||
"table":{
|
||||
"guid":"aabdc6c0-96c1-451c-98b5-7a7e48c319be",
|
||||
"typeName":"Table",
|
||||
"entityStatus":"ACTIVE",
|
||||
"displayText":"customer_dim",
|
||||
"relationshipType":"Table_Columns",
|
||||
"relationshipGuid":"66022976-9fb6-492d-8389-170082212e11",
|
||||
"relationshipStatus":"ACTIVE",
|
||||
"relationshipAttributes":{
|
||||
"typeName":"Table_Columns"
|
||||
}
|
||||
},
|
||||
"outputFromProcesses":[
|
||||
|
||||
]
|
||||
},
|
||||
"classifications":[
|
||||
{
|
||||
"typeName":"PII",
|
||||
"entityGuid":"f0f82283-d4af-413f-ad4a-eb908f184e85",
|
||||
"entityStatus":"ACTIVE",
|
||||
"propagate":true,
|
||||
"removePropagationsOnEntityDelete":false
|
||||
}
|
||||
],
|
||||
"labels":[
|
||||
|
||||
]
|
||||
},
|
||||
"6e389306-3fde-4d1b-bf1a-00940ea69414":{
|
||||
"typeName":"Column",
|
||||
"attributes":{
|
||||
"owner":"None",
|
||||
"replicatedTo":"None",
|
||||
"userDescription":"None",
|
||||
"replicatedFrom":"None",
|
||||
"qualifiedName":"sales@cl1",
|
||||
"displayName":"None",
|
||||
"dataType":"double",
|
||||
"name":"sales",
|
||||
"description":"None",
|
||||
"comment":"product id"
|
||||
},
|
||||
"guid":"6e389306-3fde-4d1b-bf1a-00940ea69414",
|
||||
"isIncomplete":false,
|
||||
"status":"ACTIVE",
|
||||
"createdBy":"admin",
|
||||
"updatedBy":"admin",
|
||||
"createTime":1650276602280,
|
||||
"updateTime":1650276617480,
|
||||
"version":0,
|
||||
"relationshipAttributes":{
|
||||
"inputToProcesses":[
|
||||
|
||||
],
|
||||
"schema":[
|
||||
|
||||
],
|
||||
"meanings":[
|
||||
|
||||
],
|
||||
"table":{
|
||||
"guid":"84135ba4-03ee-4f2e-ae55-29aa2682a699",
|
||||
"typeName":"Table",
|
||||
"entityStatus":"ACTIVE",
|
||||
"displayText":"sales_fact_monthly_mv",
|
||||
"relationshipType":"Table_Columns",
|
||||
"relationshipGuid":"8642936e-6112-4814-a538-07e1dc2db00e",
|
||||
"relationshipStatus":"ACTIVE",
|
||||
"relationshipAttributes":{
|
||||
"typeName":"Table_Columns"
|
||||
}
|
||||
},
|
||||
"outputFromProcesses":[
|
||||
|
||||
]
|
||||
},
|
||||
"classifications":[
|
||||
{
|
||||
"typeName":"Metric",
|
||||
"entityGuid":"6e389306-3fde-4d1b-bf1a-00940ea69414",
|
||||
"entityStatus":"ACTIVE",
|
||||
"propagate":true,
|
||||
"removePropagationsOnEntityDelete":false
|
||||
}
|
||||
],
|
||||
"labels":[
|
||||
|
||||
]
|
||||
},
|
||||
"e23e6fe3-e772-40bf-baa5-a738e7d00cc3":{
|
||||
"typeName":"StorageDesc",
|
||||
"attributes":{
|
||||
"owner":"None",
|
||||
"replicatedTo":"None",
|
||||
"userDescription":"None",
|
||||
"replicatedFrom":"None",
|
||||
"qualifiedName":"sd:hdfs://host:8000/apps/warehouse/sales@cl1",
|
||||
"displayName":"None",
|
||||
"name":"sd:hdfs://host:8000/apps/warehouse/sales",
|
||||
"description":"None",
|
||||
"location":"hdfs://host:8000/apps/warehouse/sales",
|
||||
"compressed":true,
|
||||
"inputFormat":"TextInputFormat",
|
||||
"outputFormat":"TextOutputFormat"
|
||||
},
|
||||
"guid":"e23e6fe3-e772-40bf-baa5-a738e7d00cc3",
|
||||
"isIncomplete":false,
|
||||
"status":"ACTIVE",
|
||||
"createdBy":"admin",
|
||||
"updatedBy":"admin",
|
||||
"createTime":1650276602280,
|
||||
"updateTime":1650276617480,
|
||||
"version":0,
|
||||
"relationshipAttributes":{
|
||||
"inputToProcesses":[
|
||||
|
||||
],
|
||||
"schema":[
|
||||
|
||||
],
|
||||
"meanings":[
|
||||
|
||||
],
|
||||
"table":{
|
||||
"guid":"fc4eae96-a7ff-4139-8bd0-3f7f8e59761b",
|
||||
"typeName":"Table",
|
||||
"entityStatus":"ACTIVE",
|
||||
"displayText":"logging_fact_monthly_mv",
|
||||
"relationshipType":"Table_StorageDesc",
|
||||
"relationshipGuid":"0e5181e7-a06b-4e03-898a-93eb9cb0d1eb",
|
||||
"relationshipStatus":"ACTIVE",
|
||||
"relationshipAttributes":{
|
||||
"typeName":"Table_StorageDesc"
|
||||
}
|
||||
},
|
||||
"outputFromProcesses":[
|
||||
|
||||
]
|
||||
},
|
||||
"labels":[
|
||||
|
||||
]
|
||||
},
|
||||
"cf2425df-dd64-49b1-8b9f-b3726e9ce11d":{
|
||||
"typeName":"Column",
|
||||
"attributes":{
|
||||
"owner":"None",
|
||||
"replicatedTo":"None",
|
||||
"userDescription":"None",
|
||||
"replicatedFrom":"None",
|
||||
"qualifiedName":"product_id@cl1",
|
||||
"displayName":"None",
|
||||
"dataType":"int",
|
||||
"name":"product_id",
|
||||
"description":"None",
|
||||
"comment":"product id"
|
||||
},
|
||||
"guid":"cf2425df-dd64-49b1-8b9f-b3726e9ce11d",
|
||||
"isIncomplete":false,
|
||||
"status":"ACTIVE",
|
||||
"createdBy":"admin",
|
||||
"updatedBy":"admin",
|
||||
"createTime":1650276602280,
|
||||
"updateTime":1650276617480,
|
||||
"version":0,
|
||||
"relationshipAttributes":{
|
||||
"inputToProcesses":[
|
||||
|
||||
],
|
||||
"schema":[
|
||||
|
||||
],
|
||||
"meanings":[
|
||||
|
||||
],
|
||||
"table":{
|
||||
"guid":"1697e9bc-2fb0-4d11-8294-7e0da9c74830",
|
||||
"typeName":"Table",
|
||||
"entityStatus":"ACTIVE",
|
||||
"displayText":"product_dim",
|
||||
"relationshipType":"Table_Columns",
|
||||
"relationshipGuid":"fea21306-ae96-4e2e-85f0-b121e967969c",
|
||||
"relationshipStatus":"ACTIVE",
|
||||
"relationshipAttributes":{
|
||||
"typeName":"Table_Columns"
|
||||
}
|
||||
},
|
||||
"outputFromProcesses":[
|
||||
|
||||
]
|
||||
},
|
||||
"labels":[
|
||||
|
||||
]
|
||||
}
|
||||
},
|
||||
"entities":[
|
||||
{
|
||||
"typeName":"Table",
|
||||
"attributes":{
|
||||
"owner":"Joe BI",
|
||||
"temporary":false,
|
||||
"lastAccessTime":1650276616988,
|
||||
"replicatedTo":"None",
|
||||
"userDescription":"None",
|
||||
"replicatedFrom":"None",
|
||||
"qualifiedName":"sales_fact_daily_mv@cl1",
|
||||
"displayName":"None",
|
||||
"description":"sales fact daily materialized view",
|
||||
"viewExpandedText":"None",
|
||||
"tableType":"Managed",
|
||||
"createTime":1650276616988,
|
||||
"name":"sales_fact_daily_mv",
|
||||
"retention":1650276616988,
|
||||
"viewOriginalText":"None"
|
||||
},
|
||||
"guid":"eff274ef-f84d-4c58-81d6-c309663b887d",
|
||||
"isIncomplete":false,
|
||||
"status":"ACTIVE",
|
||||
"createdBy":"admin",
|
||||
"updatedBy":"admin",
|
||||
"createTime":1650276616997,
|
||||
"updateTime":1650276619535,
|
||||
"version":0,
|
||||
"relationshipAttributes":{
|
||||
"inputToProcesses":[
|
||||
{
|
||||
"guid":"2194856a-9c10-40a1-8e85-8460e53cf15b",
|
||||
"typeName":"LoadProcess",
|
||||
"entityStatus":"ACTIVE",
|
||||
"displayText":"loadSalesMonthly",
|
||||
"relationshipType":"dataset_process_inputs",
|
||||
"relationshipGuid":"77bb8a16-61d8-48f5-a8fb-b61c493eb008",
|
||||
"relationshipStatus":"ACTIVE",
|
||||
"relationshipAttributes":{
|
||||
"typeName":"dataset_process_inputs"
|
||||
}
|
||||
}
|
||||
],
|
||||
"schema":[
|
||||
|
||||
],
|
||||
"sd":{
|
||||
"guid":"e23e6fe3-e772-40bf-baa5-a738e7d00cc3",
|
||||
"typeName":"StorageDesc",
|
||||
"entityStatus":"ACTIVE",
|
||||
"displayText":"sd:hdfs://host:8000/apps/warehouse/sales",
|
||||
"relationshipType":"Table_StorageDesc",
|
||||
"relationshipGuid":"8da6f6d8-9998-425a-b409-f267ebf66631",
|
||||
"relationshipStatus":"DELETED",
|
||||
"relationshipAttributes":{
|
||||
"typeName":"Table_StorageDesc"
|
||||
}
|
||||
},
|
||||
"columns":[
|
||||
{
|
||||
"guid":"e82a090a-0f21-4694-aae5-11eac90ada6d",
|
||||
"typeName":"Column",
|
||||
"entityStatus":"ACTIVE",
|
||||
"displayText":"time_id",
|
||||
"relationshipType":"Table_Columns",
|
||||
"relationshipGuid":"fcc25fe7-da42-4c39-abf6-ee862c96bf39",
|
||||
"relationshipStatus":"DELETED",
|
||||
"relationshipAttributes":{
|
||||
"typeName":"Table_Columns"
|
||||
}
|
||||
},
|
||||
{
|
||||
"guid":"f0f82283-d4af-413f-ad4a-eb908f184e85",
|
||||
"typeName":"Column",
|
||||
"entityStatus":"ACTIVE",
|
||||
"displayText":"customer_id",
|
||||
"relationshipType":"Table_Columns",
|
||||
"relationshipGuid":"3f062158-b7db-4a8a-b92e-c2f8aec4944f",
|
||||
"relationshipStatus":"DELETED",
|
||||
"relationshipAttributes":{
|
||||
"typeName":"Table_Columns"
|
||||
}
|
||||
},
|
||||
{
|
||||
"guid":"6e389306-3fde-4d1b-bf1a-00940ea69414",
|
||||
"typeName":"Column",
|
||||
"entityStatus":"ACTIVE",
|
||||
"displayText":"sales",
|
||||
"relationshipType":"Table_Columns",
|
||||
"relationshipGuid":"8a2aa73f-b76e-4a71-a99f-36d6b5145c6e",
|
||||
"relationshipStatus":"DELETED",
|
||||
"relationshipAttributes":{
|
||||
"typeName":"Table_Columns"
|
||||
}
|
||||
},
|
||||
{
|
||||
"guid":"cf2425df-dd64-49b1-8b9f-b3726e9ce11d",
|
||||
"typeName":"Column",
|
||||
"entityStatus":"ACTIVE",
|
||||
"displayText":"product_id",
|
||||
"relationshipType":"Table_Columns",
|
||||
"relationshipGuid":"e8f45f67-b7c2-48fb-a59a-e23067c2300a",
|
||||
"relationshipStatus":"DELETED",
|
||||
"relationshipAttributes":{
|
||||
"typeName":"Table_Columns"
|
||||
}
|
||||
}
|
||||
],
|
||||
"meanings":[
|
||||
|
||||
],
|
||||
"db":{
|
||||
"guid":"388f1d3f-31d4-4ee5-898d-c35e6e752b38",
|
||||
"typeName":"DB",
|
||||
"entityStatus":"ACTIVE",
|
||||
"displayText":"Reporting",
|
||||
"description": "THIS IS TEST_DESCRIPTION FOR DATABASE",
|
||||
"relationshipType":"Table_DB",
|
||||
"relationshipGuid":"cdbddf6a-ec6f-4263-9238-38281d73b4ea",
|
||||
"relationshipStatus":"ACTIVE",
|
||||
"relationshipAttributes":{
|
||||
"typeName":"Table_DB"
|
||||
}
|
||||
},
|
||||
"outputFromProcesses":[
|
||||
{
|
||||
"guid":"6d6ab7e5-646f-4161-937c-dc340d7c9d06",
|
||||
"typeName":"LoadProcess",
|
||||
"entityStatus":"ACTIVE",
|
||||
"displayText":"loadSalesDaily",
|
||||
"relationshipType":"process_dataset_outputs",
|
||||
"relationshipGuid":"2573b9b2-82e6-4089-b3f1-8b0b1b378491",
|
||||
"relationshipStatus":"ACTIVE",
|
||||
"relationshipAttributes":{
|
||||
"typeName":"process_dataset_outputs"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"classifications":[
|
||||
{
|
||||
"typeName":"Metric",
|
||||
"entityGuid":"eff274ef-f84d-4c58-81d6-c309663b887d",
|
||||
"entityStatus":"ACTIVE",
|
||||
"propagate":true,
|
||||
"removePropagationsOnEntityDelete":false
|
||||
},
|
||||
{
|
||||
"typeName":"Fact",
|
||||
"entityGuid":"b233b2ae-8a4a-44a3-b446-4027462b2cc6",
|
||||
"entityStatus":"ACTIVE",
|
||||
"propagate":true,
|
||||
"removePropagationsOnEntityDelete":false
|
||||
},
|
||||
{
|
||||
"typeName":"Dimension",
|
||||
"attributes":{
|
||||
"customers":"None"
|
||||
},
|
||||
"entityGuid":"67101f5b-205a-4fd8-8273-273ccb45ca74",
|
||||
"entityStatus":"ACTIVE",
|
||||
"propagate":true,
|
||||
"removePropagationsOnEntityDelete":false
|
||||
},
|
||||
{
|
||||
"typeName":"ETL",
|
||||
"entityGuid":"6d6ab7e5-646f-4161-937c-dc340d7c9d06",
|
||||
"entityStatus":"ACTIVE",
|
||||
"propagate":true,
|
||||
"removePropagationsOnEntityDelete":false
|
||||
}
|
||||
],
|
||||
"labels":[
|
||||
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
506
ingestion/tests/unit/topology/metadata/test_atlas.py
Normal file
506
ingestion/tests/unit/topology/metadata/test_atlas.py
Normal file
@ -0,0 +1,506 @@
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""
|
||||
Test Atlas using the topology
|
||||
"""
|
||||
import json
|
||||
from pathlib import Path
|
||||
from unittest import TestCase
|
||||
from unittest.mock import patch
|
||||
|
||||
from pydantic import AnyUrl
|
||||
|
||||
from metadata.generated.schema.api.classification.createClassification import (
|
||||
CreateClassificationRequest,
|
||||
)
|
||||
from metadata.generated.schema.api.classification.createTag import CreateTagRequest
|
||||
from metadata.generated.schema.api.data.createDatabase import CreateDatabaseRequest
|
||||
from metadata.generated.schema.api.data.createDatabaseSchema import (
|
||||
CreateDatabaseSchemaRequest,
|
||||
)
|
||||
from metadata.generated.schema.api.data.createTable import CreateTableRequest
|
||||
from metadata.generated.schema.api.services.createDatabaseService import (
|
||||
CreateDatabaseServiceRequest,
|
||||
)
|
||||
from metadata.generated.schema.entity.data.database import Database
|
||||
from metadata.generated.schema.entity.data.databaseSchema import DatabaseSchema
|
||||
from metadata.generated.schema.entity.data.table import Column, Table
|
||||
from metadata.generated.schema.entity.services.connections.database.hiveConnection import (
|
||||
HiveConnection,
|
||||
)
|
||||
from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import (
|
||||
OpenMetadataConnection,
|
||||
)
|
||||
from metadata.generated.schema.entity.services.databaseService import (
|
||||
DatabaseConnection,
|
||||
DatabaseService,
|
||||
)
|
||||
from metadata.generated.schema.metadataIngestion.workflow import (
|
||||
OpenMetadataWorkflowConfig,
|
||||
)
|
||||
from metadata.generated.schema.type.basic import Href
|
||||
from metadata.generated.schema.type.entityReference import EntityReference
|
||||
from metadata.generated.schema.type.tagLabel import TagLabel
|
||||
from metadata.ingestion.ometa.ometa_api import OpenMetadata
|
||||
from metadata.ingestion.source.metadata.atlas.client import AtlasClient
|
||||
from metadata.ingestion.source.metadata.atlas.metadata import AtlasSource
|
||||
|
||||
mock_atlas_config = {
|
||||
"source": {
|
||||
"type": "Atlas",
|
||||
"serviceName": "local_atlas",
|
||||
"serviceConnection": {
|
||||
"config": {
|
||||
"type": "Atlas",
|
||||
"hostPort": "http://192.168.1.8:21000",
|
||||
"username": "username",
|
||||
"password": "password",
|
||||
"databaseServiceName": ["hive"],
|
||||
"messagingServiceName": [],
|
||||
}
|
||||
},
|
||||
"sourceConfig": {"config": {"type": "DatabaseMetadata"}},
|
||||
},
|
||||
"sink": {"type": "metadata-rest", "config": {}},
|
||||
"workflowConfig": {
|
||||
"openMetadataServerConfig": {
|
||||
"hostPort": "http://localhost:8585/api",
|
||||
"authProvider": "openmetadata",
|
||||
"securityConfig": {
|
||||
"jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGc"
|
||||
"iOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE"
|
||||
"2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXB"
|
||||
"iEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fN"
|
||||
"r3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3u"
|
||||
"d-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg"
|
||||
},
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
mock_file_path = (
|
||||
Path(__file__).parent.parent.parent / "resources/datasets/atlas_dataset.json"
|
||||
)
|
||||
with open(mock_file_path, encoding="UTF-8") as file:
|
||||
mock_data: dict = json.load(file)
|
||||
|
||||
|
||||
LIST_ENTITIES = [
|
||||
"b233b2ae-8a4a-44a3-b446-4027462b2cc6",
|
||||
"eff274ef-f84d-4c58-81d6-c309663b887d",
|
||||
"1697e9bc-2fb0-4d11-8294-7e0da9c74830",
|
||||
"aabdc6c0-96c1-451c-98b5-7a7e48c319be",
|
||||
]
|
||||
|
||||
|
||||
def mock_get_entity(self, table): # pylint: disable=unused-argument
|
||||
return mock_data
|
||||
|
||||
|
||||
def mock_list_entities(self, entity_type): # pylint: disable=unused-argument
|
||||
return LIST_ENTITIES
|
||||
|
||||
|
||||
EXPECTED_DATABASE_DESCRIPTION = "THIS IS TEST_DESCRIPTION FOR DATABASE"
|
||||
|
||||
EXPTECTED_DATABASE_SCHEMA_DESCRIPTION = "THIS IS TEST_DESCRIPTION FOR DATABASE"
|
||||
|
||||
EXPTECTED_TABLE = Table(
|
||||
id="124d078d-dcf2-43a8-b59e-33bc7953f680",
|
||||
name="delta_test_table",
|
||||
displayName=None,
|
||||
fullyQualifiedName="hive.Reporting.Reporting.delta_test_table",
|
||||
description="sales fact daily materialized view",
|
||||
version=0.1,
|
||||
updatedAt=1673413042524,
|
||||
updatedBy="admin",
|
||||
href=Href(
|
||||
__root__=AnyUrl(
|
||||
"http://localhost:8585/api/v1/tables/124d078d-dcf2-43a8-b59e-33bc7953f680",
|
||||
scheme="http",
|
||||
host="localhost",
|
||||
host_type="int_domain",
|
||||
port="8585",
|
||||
path="/api/v1/tables/124d078d-dcf2-43a8-b59e-33bc7953f680",
|
||||
)
|
||||
),
|
||||
tableType="Regular",
|
||||
columns=[
|
||||
Column(
|
||||
name="col2",
|
||||
displayName=None,
|
||||
dataType="STRUCT",
|
||||
arrayDataType=None,
|
||||
dataLength=1,
|
||||
precision=None,
|
||||
scale=None,
|
||||
dataTypeDisplay="struct<col3:string,col4:bigint>",
|
||||
description="col2 nested delta column",
|
||||
fullyQualifiedName="hive.Reporting.Reporting.delta_test_table.col2",
|
||||
tags=None,
|
||||
constraint=None,
|
||||
ordinalPosition=None,
|
||||
jsonSchema=None,
|
||||
children=[
|
||||
Column(
|
||||
name="col3",
|
||||
displayName=None,
|
||||
dataType="STRING",
|
||||
arrayDataType=None,
|
||||
dataLength=None,
|
||||
precision=None,
|
||||
scale=None,
|
||||
dataTypeDisplay="string",
|
||||
description=None,
|
||||
fullyQualifiedName="hive.Reporting.Reporting.delta_test_table.col2.col3",
|
||||
tags=None,
|
||||
constraint=None,
|
||||
ordinalPosition=None,
|
||||
jsonSchema=None,
|
||||
children=None,
|
||||
customMetrics=None,
|
||||
profile=None,
|
||||
),
|
||||
Column(
|
||||
name="col4",
|
||||
displayName=None,
|
||||
dataType="BIGINT",
|
||||
arrayDataType=None,
|
||||
dataLength=None,
|
||||
precision=None,
|
||||
scale=None,
|
||||
dataTypeDisplay="bigint",
|
||||
description=None,
|
||||
fullyQualifiedName="hive.Reporting.Reporting.delta_test_table.col2.col4",
|
||||
tags=None,
|
||||
constraint=None,
|
||||
ordinalPosition=None,
|
||||
jsonSchema=None,
|
||||
children=None,
|
||||
customMetrics=None,
|
||||
profile=None,
|
||||
),
|
||||
],
|
||||
customMetrics=None,
|
||||
profile=None,
|
||||
),
|
||||
Column(
|
||||
name="col1",
|
||||
displayName=None,
|
||||
dataType="BIGINT",
|
||||
arrayDataType=None,
|
||||
dataLength=1,
|
||||
precision=None,
|
||||
scale=None,
|
||||
dataTypeDisplay="bigint",
|
||||
description="col1 description",
|
||||
fullyQualifiedName="hive.Reporting.Reporting.delta_test_table.col1",
|
||||
tags=None,
|
||||
constraint=None,
|
||||
ordinalPosition=None,
|
||||
jsonSchema=None,
|
||||
children=None,
|
||||
customMetrics=None,
|
||||
profile=None,
|
||||
),
|
||||
],
|
||||
tableConstraints=None,
|
||||
tablePartition=None,
|
||||
owner=None,
|
||||
databaseSchema=EntityReference(
|
||||
id="4cf6ee7e-9d24-4153-9318-82aa1167259b",
|
||||
type="databaseSchema",
|
||||
name="Reporting",
|
||||
fullyQualifiedName="hive.Reporting.Reporting",
|
||||
description="THIS IS TEST_DESCRIPTION FOR DATABASE",
|
||||
displayName=None,
|
||||
deleted=False,
|
||||
href=Href(
|
||||
__root__=AnyUrl(
|
||||
"http://localhost:8585/api/v1/databaseSchemas/4cf6ee7e-9d24-4153-9318-82aa1167259b",
|
||||
scheme="http",
|
||||
host="localhost",
|
||||
host_type="int_domain",
|
||||
port="8585",
|
||||
path="/api/v1/databaseSchemas/4cf6ee7e-9d24-4153-9318-82aa1167259b",
|
||||
)
|
||||
),
|
||||
),
|
||||
database=EntityReference(
|
||||
id="367f53b5-d6c2-44be-bf5d-a0a1dc98a9dd",
|
||||
type="database",
|
||||
name="Reporting",
|
||||
fullyQualifiedName="hive.Reporting",
|
||||
description="THIS IS TEST_DESCRIPTION FOR DATABASE",
|
||||
displayName=None,
|
||||
deleted=False,
|
||||
href=Href(
|
||||
__root__=AnyUrl(
|
||||
"http://localhost:8585/api/v1/databases/367f53b5-d6c2-44be-bf5d-a0a1dc98a9dd",
|
||||
scheme="http",
|
||||
host="localhost",
|
||||
host_type="int_domain",
|
||||
port="8585",
|
||||
path="/api/v1/databases/367f53b5-d6c2-44be-bf5d-a0a1dc98a9dd",
|
||||
)
|
||||
),
|
||||
),
|
||||
service=EntityReference(
|
||||
id="f2ab0e7a-5224-4acb-a189-74158851733f",
|
||||
type="databaseService",
|
||||
name="hive",
|
||||
fullyQualifiedName="hive",
|
||||
description=None,
|
||||
displayName=None,
|
||||
deleted=False,
|
||||
href=Href(
|
||||
__root__=AnyUrl(
|
||||
"http://localhost:8585/api/v1/services/databaseServices/f2ab0e7a-5224-4acb-a189-74158851733f",
|
||||
scheme="http",
|
||||
host="localhost",
|
||||
host_type="int_domain",
|
||||
port="8585",
|
||||
path="/api/v1/services/databaseServices/f2ab0e7a-5224-4acb-a189-74158851733f",
|
||||
)
|
||||
),
|
||||
),
|
||||
serviceType="Hive",
|
||||
location=None,
|
||||
viewDefinition=None,
|
||||
tags=[
|
||||
TagLabel(
|
||||
tagFQN="AtlasMetadata.atlas_table",
|
||||
description="test tag",
|
||||
source="Tag",
|
||||
labelType="Automated",
|
||||
state="Confirmed",
|
||||
href=None,
|
||||
)
|
||||
],
|
||||
usageSummary=None,
|
||||
followers=None,
|
||||
joins=None,
|
||||
sampleData=None,
|
||||
tableProfilerConfig=None,
|
||||
profile=None,
|
||||
tableQueries=None,
|
||||
dataModel=None,
|
||||
changeDescription=None,
|
||||
deleted=False,
|
||||
extension=None,
|
||||
)
|
||||
|
||||
|
||||
class AtlasUnitTest(TestCase):
|
||||
"""
|
||||
Implements the necessary methods to extract
|
||||
Atlas Metadata Unit Test
|
||||
"""
|
||||
|
||||
@patch(
|
||||
"metadata.ingestion.source.pipeline.pipeline_service.PipelineServiceSource.test_connection"
|
||||
)
|
||||
def __init__(self, methodName, test_connection) -> None:
|
||||
super().__init__(methodName)
|
||||
test_connection.return_value = False
|
||||
self.config = OpenMetadataWorkflowConfig.parse_obj(mock_atlas_config)
|
||||
self.atlas_source = AtlasSource.create(
|
||||
mock_atlas_config["source"],
|
||||
self.config.workflowConfig.openMetadataServerConfig,
|
||||
)
|
||||
self.metadata = OpenMetadata(
|
||||
OpenMetadataConnection.parse_obj(
|
||||
mock_atlas_config["workflowConfig"]["openMetadataServerConfig"]
|
||||
)
|
||||
)
|
||||
|
||||
self.database_service = (
|
||||
mock_database_service_object
|
||||
) = self.metadata.create_or_update(
|
||||
CreateDatabaseServiceRequest(
|
||||
name="hive",
|
||||
serviceType="Hive",
|
||||
connection=DatabaseConnection(
|
||||
config=HiveConnection(
|
||||
type="Hive",
|
||||
scheme="hive",
|
||||
username=None,
|
||||
password=None,
|
||||
hostPort="http://nohost:6000",
|
||||
)
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
mock_database_object = self.metadata.create_or_update(
|
||||
CreateDatabaseRequest(
|
||||
name="Reporting",
|
||||
displayName=None,
|
||||
description=None,
|
||||
tags=None,
|
||||
owner=None,
|
||||
service=EntityReference(
|
||||
id=mock_database_service_object.id,
|
||||
type="databaseService",
|
||||
),
|
||||
)
|
||||
)
|
||||
mock_database_schema_object = self.metadata.create_or_update(
|
||||
CreateDatabaseSchemaRequest(
|
||||
name="Reporting",
|
||||
database=EntityReference(id=mock_database_object.id, type="database"),
|
||||
)
|
||||
)
|
||||
_ = self.metadata.create_or_update(
|
||||
CreateTableRequest(
|
||||
name="sales_fact_daily_mv",
|
||||
tableType="Regular",
|
||||
columns=[
|
||||
Column(
|
||||
name="col2",
|
||||
displayName=None,
|
||||
dataType="STRUCT",
|
||||
arrayDataType=None,
|
||||
dataLength=1,
|
||||
precision=None,
|
||||
scale=None,
|
||||
dataTypeDisplay="struct<col3:string,col4:bigint>",
|
||||
description="col2 nested delta column",
|
||||
fullyQualifiedName="delta.default.test_schema.delta_test_table.col2",
|
||||
tags=None,
|
||||
constraint=None,
|
||||
ordinalPosition=None,
|
||||
jsonSchema=None,
|
||||
children=[
|
||||
Column(
|
||||
name="col3",
|
||||
displayName=None,
|
||||
dataType="STRING",
|
||||
arrayDataType=None,
|
||||
dataLength=None,
|
||||
precision=None,
|
||||
scale=None,
|
||||
dataTypeDisplay="string",
|
||||
description=None,
|
||||
fullyQualifiedName="delta.default.test_schema.delta_test_table.col2.col3",
|
||||
tags=None,
|
||||
constraint=None,
|
||||
ordinalPosition=None,
|
||||
jsonSchema=None,
|
||||
children=None,
|
||||
customMetrics=None,
|
||||
profile=None,
|
||||
),
|
||||
Column(
|
||||
name="col4",
|
||||
displayName=None,
|
||||
dataType="BIGINT",
|
||||
arrayDataType=None,
|
||||
dataLength=None,
|
||||
precision=None,
|
||||
scale=None,
|
||||
dataTypeDisplay="bigint",
|
||||
description=None,
|
||||
fullyQualifiedName="delta.default.test_schema.delta_test_table.col2.col4",
|
||||
tags=None,
|
||||
constraint=None,
|
||||
ordinalPosition=None,
|
||||
jsonSchema=None,
|
||||
children=None,
|
||||
customMetrics=None,
|
||||
profile=None,
|
||||
),
|
||||
],
|
||||
customMetrics=None,
|
||||
profile=None,
|
||||
),
|
||||
Column(
|
||||
name="col1",
|
||||
displayName=None,
|
||||
dataType="BIGINT",
|
||||
arrayDataType=None,
|
||||
dataLength=1,
|
||||
precision=None,
|
||||
scale=None,
|
||||
dataTypeDisplay="bigint",
|
||||
description="col1 description",
|
||||
fullyQualifiedName="delta.default.test_schema.delta_test_table.col1",
|
||||
tags=None,
|
||||
constraint=None,
|
||||
ordinalPosition=None,
|
||||
jsonSchema=None,
|
||||
children=None,
|
||||
customMetrics=None,
|
||||
profile=None,
|
||||
),
|
||||
],
|
||||
databaseSchema=EntityReference(
|
||||
id=mock_database_schema_object.id, type="databaseSchema"
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
def mock_ingest_lineage(self, source_guid, name): # pylint: disable=unused-argument
|
||||
return []
|
||||
|
||||
def mock_create_tag(self):
|
||||
classification = CreateClassificationRequest(
|
||||
description="test tag", name="AtlasMetadata"
|
||||
)
|
||||
|
||||
self.metadata.create_or_update(classification)
|
||||
self.metadata.create_or_update(
|
||||
CreateTagRequest(
|
||||
classification="AtlasMetadata",
|
||||
name="atlas_table",
|
||||
description="test tag",
|
||||
)
|
||||
)
|
||||
|
||||
@patch.object(AtlasClient, "list_entities", mock_list_entities)
|
||||
@patch.object(AtlasClient, "get_entity", mock_get_entity)
|
||||
@patch.object(AtlasSource, "ingest_lineage", mock_ingest_lineage)
|
||||
@patch.object(AtlasSource, "create_tag", mock_create_tag)
|
||||
def test_description(self):
|
||||
"""
|
||||
Testing description updated for database, databaseSchema, table
|
||||
"""
|
||||
_ = list(self.atlas_source.next_record())
|
||||
updated_database = self.metadata.get_by_name(
|
||||
entity=Database, fqn="hive.Reporting"
|
||||
)
|
||||
assert updated_database.description.__root__ == EXPECTED_DATABASE_DESCRIPTION
|
||||
|
||||
updated_database_schema = self.metadata.get_by_name(
|
||||
entity=DatabaseSchema, fqn="hive.Reporting.Reporting"
|
||||
)
|
||||
assert (
|
||||
updated_database_schema.description.__root__
|
||||
== EXPTECTED_DATABASE_SCHEMA_DESCRIPTION
|
||||
)
|
||||
|
||||
updated_table = self.metadata.get_by_name(
|
||||
entity=Table,
|
||||
fqn="hive.Reporting.Reporting.sales_fact_daily_mv",
|
||||
fields=["tags"],
|
||||
)
|
||||
|
||||
assert updated_table.description == EXPTECTED_TABLE.description
|
||||
|
||||
assert updated_table.tags == EXPTECTED_TABLE.tags
|
||||
|
||||
self.metadata.delete(
|
||||
entity=DatabaseService,
|
||||
entity_id=self.atlas_source.service.id,
|
||||
recursive=True,
|
||||
hard_delete=True,
|
||||
)
|
||||
Loading…
x
Reference in New Issue
Block a user