mirror of
https://github.com/open-metadata/OpenMetadata.git
synced 2025-08-15 04:26:59 +00:00
Fix: added test case for atlas (#9678)
* Fix: added test case for atlas * Fix: resolved conflict * Fix: changing back neo4j to old version * Fix: changing back neo4j to old version * Fix: changes as per comment * Fix: changes as per comment * Fix: python checkstyle
This commit is contained in:
NiharDoshi99
GitHub
parent
35b9cef3fa
commit
029dbe892e
@ -9,6 +9,10 @@
|
|||||||
# See the License for the specific language governing permissions and
|
# See the License for the specific language governing permissions and
|
||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
|
|
||||||
|
"""
|
||||||
|
Python Dependencies
|
||||||
|
"""
|
||||||
|
|
||||||
import os
|
import os
|
||||||
from typing import Dict, Set
|
from typing import Dict, Set
|
||||||
|
|
||||||
@ -17,8 +21,8 @@ from setuptools import find_namespace_packages, setup
|
|||||||
|
|
||||||
def get_long_description():
|
def get_long_description():
|
||||||
root = os.path.dirname(__file__)
|
root = os.path.dirname(__file__)
|
||||||
with open(os.path.join(root, "README.md")) as f:
|
with open(os.path.join(root, "README.md"), encoding="UTF-8") as file:
|
||||||
description = f.read()
|
description = file.read()
|
||||||
return description
|
return description
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@ -207,6 +207,7 @@ class AtlasSource(Source):
|
|||||||
for table in entity:
|
for table in entity:
|
||||||
table_entity = self.atlas_client.get_entity(table)
|
table_entity = self.atlas_client.get_entity(table)
|
||||||
tbl_entities = table_entity["entities"]
|
tbl_entities = table_entity["entities"]
|
||||||
|
db_entity = None
|
||||||
for tbl_entity in tbl_entities:
|
for tbl_entity in tbl_entities:
|
||||||
try:
|
try:
|
||||||
|
|
||||||
|
|||||||
461
ingestion/tests/unit/resources/datasets/atlas_dataset.json
Normal file
461
ingestion/tests/unit/resources/datasets/atlas_dataset.json
Normal file
@ -0,0 +1,461 @@
|
|||||||
|
{
|
||||||
|
"referredEntities":{
|
||||||
|
"e82a090a-0f21-4694-aae5-11eac90ada6d":{
|
||||||
|
"typeName":"Column",
|
||||||
|
"attributes":{
|
||||||
|
"owner":"None",
|
||||||
|
"replicatedTo":"None",
|
||||||
|
"userDescription":"None",
|
||||||
|
"replicatedFrom":"None",
|
||||||
|
"qualifiedName":"time_id@cl1",
|
||||||
|
"displayName":"None",
|
||||||
|
"dataType":"int",
|
||||||
|
"name":"time_id",
|
||||||
|
"description":"None",
|
||||||
|
"comment":"time id"
|
||||||
|
},
|
||||||
|
"guid":"e82a090a-0f21-4694-aae5-11eac90ada6d",
|
||||||
|
"isIncomplete":false,
|
||||||
|
"status":"ACTIVE",
|
||||||
|
"createdBy":"admin",
|
||||||
|
"updatedBy":"admin",
|
||||||
|
"createTime":1650276602280,
|
||||||
|
"updateTime":1650276617480,
|
||||||
|
"version":0,
|
||||||
|
"relationshipAttributes":{
|
||||||
|
"inputToProcesses":[
|
||||||
|
|
||||||
|
],
|
||||||
|
"schema":[
|
||||||
|
|
||||||
|
],
|
||||||
|
"meanings":[
|
||||||
|
|
||||||
|
],
|
||||||
|
"table":{
|
||||||
|
"guid":"fc4eae96-a7ff-4139-8bd0-3f7f8e59761b",
|
||||||
|
"typeName":"Table",
|
||||||
|
"entityStatus":"ACTIVE",
|
||||||
|
"displayText":"logging_fact_monthly_mv",
|
||||||
|
"relationshipType":"Table_Columns",
|
||||||
|
"relationshipGuid":"bd5b2145-237f-421b-b0c0-92a8da045d4f",
|
||||||
|
"relationshipStatus":"ACTIVE",
|
||||||
|
"relationshipAttributes":{
|
||||||
|
"typeName":"Table_Columns"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"outputFromProcesses":[
|
||||||
|
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"labels":[
|
||||||
|
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"f0f82283-d4af-413f-ad4a-eb908f184e85":{
|
||||||
|
"typeName":"Column",
|
||||||
|
"attributes":{
|
||||||
|
"owner":"None",
|
||||||
|
"replicatedTo":"None",
|
||||||
|
"userDescription":"None",
|
||||||
|
"replicatedFrom":"None",
|
||||||
|
"qualifiedName":"customer_id@cl1",
|
||||||
|
"displayName":"None",
|
||||||
|
"dataType":"int",
|
||||||
|
"name":"customer_id",
|
||||||
|
"description":"None",
|
||||||
|
"comment":"customer id"
|
||||||
|
},
|
||||||
|
"guid":"f0f82283-d4af-413f-ad4a-eb908f184e85",
|
||||||
|
"isIncomplete":false,
|
||||||
|
"status":"ACTIVE",
|
||||||
|
"createdBy":"admin",
|
||||||
|
"updatedBy":"admin",
|
||||||
|
"createTime":1650276602280,
|
||||||
|
"updateTime":1650276617480,
|
||||||
|
"version":0,
|
||||||
|
"relationshipAttributes":{
|
||||||
|
"inputToProcesses":[
|
||||||
|
|
||||||
|
],
|
||||||
|
"schema":[
|
||||||
|
|
||||||
|
],
|
||||||
|
"meanings":[
|
||||||
|
|
||||||
|
],
|
||||||
|
"table":{
|
||||||
|
"guid":"aabdc6c0-96c1-451c-98b5-7a7e48c319be",
|
||||||
|
"typeName":"Table",
|
||||||
|
"entityStatus":"ACTIVE",
|
||||||
|
"displayText":"customer_dim",
|
||||||
|
"relationshipType":"Table_Columns",
|
||||||
|
"relationshipGuid":"66022976-9fb6-492d-8389-170082212e11",
|
||||||
|
"relationshipStatus":"ACTIVE",
|
||||||
|
"relationshipAttributes":{
|
||||||
|
"typeName":"Table_Columns"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"outputFromProcesses":[
|
||||||
|
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"classifications":[
|
||||||
|
{
|
||||||
|
"typeName":"PII",
|
||||||
|
"entityGuid":"f0f82283-d4af-413f-ad4a-eb908f184e85",
|
||||||
|
"entityStatus":"ACTIVE",
|
||||||
|
"propagate":true,
|
||||||
|
"removePropagationsOnEntityDelete":false
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"labels":[
|
||||||
|
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"6e389306-3fde-4d1b-bf1a-00940ea69414":{
|
||||||
|
"typeName":"Column",
|
||||||
|
"attributes":{
|
||||||
|
"owner":"None",
|
||||||
|
"replicatedTo":"None",
|
||||||
|
"userDescription":"None",
|
||||||
|
"replicatedFrom":"None",
|
||||||
|
"qualifiedName":"sales@cl1",
|
||||||
|
"displayName":"None",
|
||||||
|
"dataType":"double",
|
||||||
|
"name":"sales",
|
||||||
|
"description":"None",
|
||||||
|
"comment":"product id"
|
||||||
|
},
|
||||||
|
"guid":"6e389306-3fde-4d1b-bf1a-00940ea69414",
|
||||||
|
"isIncomplete":false,
|
||||||
|
"status":"ACTIVE",
|
||||||
|
"createdBy":"admin",
|
||||||
|
"updatedBy":"admin",
|
||||||
|
"createTime":1650276602280,
|
||||||
|
"updateTime":1650276617480,
|
||||||
|
"version":0,
|
||||||
|
"relationshipAttributes":{
|
||||||
|
"inputToProcesses":[
|
||||||
|
|
||||||
|
],
|
||||||
|
"schema":[
|
||||||
|
|
||||||
|
],
|
||||||
|
"meanings":[
|
||||||
|
|
||||||
|
],
|
||||||
|
"table":{
|
||||||
|
"guid":"84135ba4-03ee-4f2e-ae55-29aa2682a699",
|
||||||
|
"typeName":"Table",
|
||||||
|
"entityStatus":"ACTIVE",
|
||||||
|
"displayText":"sales_fact_monthly_mv",
|
||||||
|
"relationshipType":"Table_Columns",
|
||||||
|
"relationshipGuid":"8642936e-6112-4814-a538-07e1dc2db00e",
|
||||||
|
"relationshipStatus":"ACTIVE",
|
||||||
|
"relationshipAttributes":{
|
||||||
|
"typeName":"Table_Columns"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"outputFromProcesses":[
|
||||||
|
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"classifications":[
|
||||||
|
{
|
||||||
|
"typeName":"Metric",
|
||||||
|
"entityGuid":"6e389306-3fde-4d1b-bf1a-00940ea69414",
|
||||||
|
"entityStatus":"ACTIVE",
|
||||||
|
"propagate":true,
|
||||||
|
"removePropagationsOnEntityDelete":false
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"labels":[
|
||||||
|
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"e23e6fe3-e772-40bf-baa5-a738e7d00cc3":{
|
||||||
|
"typeName":"StorageDesc",
|
||||||
|
"attributes":{
|
||||||
|
"owner":"None",
|
||||||
|
"replicatedTo":"None",
|
||||||
|
"userDescription":"None",
|
||||||
|
"replicatedFrom":"None",
|
||||||
|
"qualifiedName":"sd:hdfs://host:8000/apps/warehouse/sales@cl1",
|
||||||
|
"displayName":"None",
|
||||||
|
"name":"sd:hdfs://host:8000/apps/warehouse/sales",
|
||||||
|
"description":"None",
|
||||||
|
"location":"hdfs://host:8000/apps/warehouse/sales",
|
||||||
|
"compressed":true,
|
||||||
|
"inputFormat":"TextInputFormat",
|
||||||
|
"outputFormat":"TextOutputFormat"
|
||||||
|
},
|
||||||
|
"guid":"e23e6fe3-e772-40bf-baa5-a738e7d00cc3",
|
||||||
|
"isIncomplete":false,
|
||||||
|
"status":"ACTIVE",
|
||||||
|
"createdBy":"admin",
|
||||||
|
"updatedBy":"admin",
|
||||||
|
"createTime":1650276602280,
|
||||||
|
"updateTime":1650276617480,
|
||||||
|
"version":0,
|
||||||
|
"relationshipAttributes":{
|
||||||
|
"inputToProcesses":[
|
||||||
|
|
||||||
|
],
|
||||||
|
"schema":[
|
||||||
|
|
||||||
|
],
|
||||||
|
"meanings":[
|
||||||
|
|
||||||
|
],
|
||||||
|
"table":{
|
||||||
|
"guid":"fc4eae96-a7ff-4139-8bd0-3f7f8e59761b",
|
||||||
|
"typeName":"Table",
|
||||||
|
"entityStatus":"ACTIVE",
|
||||||
|
"displayText":"logging_fact_monthly_mv",
|
||||||
|
"relationshipType":"Table_StorageDesc",
|
||||||
|
"relationshipGuid":"0e5181e7-a06b-4e03-898a-93eb9cb0d1eb",
|
||||||
|
"relationshipStatus":"ACTIVE",
|
||||||
|
"relationshipAttributes":{
|
||||||
|
"typeName":"Table_StorageDesc"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"outputFromProcesses":[
|
||||||
|
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"labels":[
|
||||||
|
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"cf2425df-dd64-49b1-8b9f-b3726e9ce11d":{
|
||||||
|
"typeName":"Column",
|
||||||
|
"attributes":{
|
||||||
|
"owner":"None",
|
||||||
|
"replicatedTo":"None",
|
||||||
|
"userDescription":"None",
|
||||||
|
"replicatedFrom":"None",
|
||||||
|
"qualifiedName":"product_id@cl1",
|
||||||
|
"displayName":"None",
|
||||||
|
"dataType":"int",
|
||||||
|
"name":"product_id",
|
||||||
|
"description":"None",
|
||||||
|
"comment":"product id"
|
||||||
|
},
|
||||||
|
"guid":"cf2425df-dd64-49b1-8b9f-b3726e9ce11d",
|
||||||
|
"isIncomplete":false,
|
||||||
|
"status":"ACTIVE",
|
||||||
|
"createdBy":"admin",
|
||||||
|
"updatedBy":"admin",
|
||||||
|
"createTime":1650276602280,
|
||||||
|
"updateTime":1650276617480,
|
||||||
|
"version":0,
|
||||||
|
"relationshipAttributes":{
|
||||||
|
"inputToProcesses":[
|
||||||
|
|
||||||
|
],
|
||||||
|
"schema":[
|
||||||
|
|
||||||
|
],
|
||||||
|
"meanings":[
|
||||||
|
|
||||||
|
],
|
||||||
|
"table":{
|
||||||
|
"guid":"1697e9bc-2fb0-4d11-8294-7e0da9c74830",
|
||||||
|
"typeName":"Table",
|
||||||
|
"entityStatus":"ACTIVE",
|
||||||
|
"displayText":"product_dim",
|
||||||
|
"relationshipType":"Table_Columns",
|
||||||
|
"relationshipGuid":"fea21306-ae96-4e2e-85f0-b121e967969c",
|
||||||
|
"relationshipStatus":"ACTIVE",
|
||||||
|
"relationshipAttributes":{
|
||||||
|
"typeName":"Table_Columns"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"outputFromProcesses":[
|
||||||
|
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"labels":[
|
||||||
|
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"entities":[
|
||||||
|
{
|
||||||
|
"typeName":"Table",
|
||||||
|
"attributes":{
|
||||||
|
"owner":"Joe BI",
|
||||||
|
"temporary":false,
|
||||||
|
"lastAccessTime":1650276616988,
|
||||||
|
"replicatedTo":"None",
|
||||||
|
"userDescription":"None",
|
||||||
|
"replicatedFrom":"None",
|
||||||
|
"qualifiedName":"sales_fact_daily_mv@cl1",
|
||||||
|
"displayName":"None",
|
||||||
|
"description":"sales fact daily materialized view",
|
||||||
|
"viewExpandedText":"None",
|
||||||
|
"tableType":"Managed",
|
||||||
|
"createTime":1650276616988,
|
||||||
|
"name":"sales_fact_daily_mv",
|
||||||
|
"retention":1650276616988,
|
||||||
|
"viewOriginalText":"None"
|
||||||
|
},
|
||||||
|
"guid":"eff274ef-f84d-4c58-81d6-c309663b887d",
|
||||||
|
"isIncomplete":false,
|
||||||
|
"status":"ACTIVE",
|
||||||
|
"createdBy":"admin",
|
||||||
|
"updatedBy":"admin",
|
||||||
|
"createTime":1650276616997,
|
||||||
|
"updateTime":1650276619535,
|
||||||
|
"version":0,
|
||||||
|
"relationshipAttributes":{
|
||||||
|
"inputToProcesses":[
|
||||||
|
{
|
||||||
|
"guid":"2194856a-9c10-40a1-8e85-8460e53cf15b",
|
||||||
|
"typeName":"LoadProcess",
|
||||||
|
"entityStatus":"ACTIVE",
|
||||||
|
"displayText":"loadSalesMonthly",
|
||||||
|
"relationshipType":"dataset_process_inputs",
|
||||||
|
"relationshipGuid":"77bb8a16-61d8-48f5-a8fb-b61c493eb008",
|
||||||
|
"relationshipStatus":"ACTIVE",
|
||||||
|
"relationshipAttributes":{
|
||||||
|
"typeName":"dataset_process_inputs"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"schema":[
|
||||||
|
|
||||||
|
],
|
||||||
|
"sd":{
|
||||||
|
"guid":"e23e6fe3-e772-40bf-baa5-a738e7d00cc3",
|
||||||
|
"typeName":"StorageDesc",
|
||||||
|
"entityStatus":"ACTIVE",
|
||||||
|
"displayText":"sd:hdfs://host:8000/apps/warehouse/sales",
|
||||||
|
"relationshipType":"Table_StorageDesc",
|
||||||
|
"relationshipGuid":"8da6f6d8-9998-425a-b409-f267ebf66631",
|
||||||
|
"relationshipStatus":"DELETED",
|
||||||
|
"relationshipAttributes":{
|
||||||
|
"typeName":"Table_StorageDesc"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"columns":[
|
||||||
|
{
|
||||||
|
"guid":"e82a090a-0f21-4694-aae5-11eac90ada6d",
|
||||||
|
"typeName":"Column",
|
||||||
|
"entityStatus":"ACTIVE",
|
||||||
|
"displayText":"time_id",
|
||||||
|
"relationshipType":"Table_Columns",
|
||||||
|
"relationshipGuid":"fcc25fe7-da42-4c39-abf6-ee862c96bf39",
|
||||||
|
"relationshipStatus":"DELETED",
|
||||||
|
"relationshipAttributes":{
|
||||||
|
"typeName":"Table_Columns"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"guid":"f0f82283-d4af-413f-ad4a-eb908f184e85",
|
||||||
|
"typeName":"Column",
|
||||||
|
"entityStatus":"ACTIVE",
|
||||||
|
"displayText":"customer_id",
|
||||||
|
"relationshipType":"Table_Columns",
|
||||||
|
"relationshipGuid":"3f062158-b7db-4a8a-b92e-c2f8aec4944f",
|
||||||
|
"relationshipStatus":"DELETED",
|
||||||
|
"relationshipAttributes":{
|
||||||
|
"typeName":"Table_Columns"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"guid":"6e389306-3fde-4d1b-bf1a-00940ea69414",
|
||||||
|
"typeName":"Column",
|
||||||
|
"entityStatus":"ACTIVE",
|
||||||
|
"displayText":"sales",
|
||||||
|
"relationshipType":"Table_Columns",
|
||||||
|
"relationshipGuid":"8a2aa73f-b76e-4a71-a99f-36d6b5145c6e",
|
||||||
|
"relationshipStatus":"DELETED",
|
||||||
|
"relationshipAttributes":{
|
||||||
|
"typeName":"Table_Columns"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"guid":"cf2425df-dd64-49b1-8b9f-b3726e9ce11d",
|
||||||
|
"typeName":"Column",
|
||||||
|
"entityStatus":"ACTIVE",
|
||||||
|
"displayText":"product_id",
|
||||||
|
"relationshipType":"Table_Columns",
|
||||||
|
"relationshipGuid":"e8f45f67-b7c2-48fb-a59a-e23067c2300a",
|
||||||
|
"relationshipStatus":"DELETED",
|
||||||
|
"relationshipAttributes":{
|
||||||
|
"typeName":"Table_Columns"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"meanings":[
|
||||||
|
|
||||||
|
],
|
||||||
|
"db":{
|
||||||
|
"guid":"388f1d3f-31d4-4ee5-898d-c35e6e752b38",
|
||||||
|
"typeName":"DB",
|
||||||
|
"entityStatus":"ACTIVE",
|
||||||
|
"displayText":"Reporting",
|
||||||
|
"description": "THIS IS TEST_DESCRIPTION FOR DATABASE",
|
||||||
|
"relationshipType":"Table_DB",
|
||||||
|
"relationshipGuid":"cdbddf6a-ec6f-4263-9238-38281d73b4ea",
|
||||||
|
"relationshipStatus":"ACTIVE",
|
||||||
|
"relationshipAttributes":{
|
||||||
|
"typeName":"Table_DB"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"outputFromProcesses":[
|
||||||
|
{
|
||||||
|
"guid":"6d6ab7e5-646f-4161-937c-dc340d7c9d06",
|
||||||
|
"typeName":"LoadProcess",
|
||||||
|
"entityStatus":"ACTIVE",
|
||||||
|
"displayText":"loadSalesDaily",
|
||||||
|
"relationshipType":"process_dataset_outputs",
|
||||||
|
"relationshipGuid":"2573b9b2-82e6-4089-b3f1-8b0b1b378491",
|
||||||
|
"relationshipStatus":"ACTIVE",
|
||||||
|
"relationshipAttributes":{
|
||||||
|
"typeName":"process_dataset_outputs"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"classifications":[
|
||||||
|
{
|
||||||
|
"typeName":"Metric",
|
||||||
|
"entityGuid":"eff274ef-f84d-4c58-81d6-c309663b887d",
|
||||||
|
"entityStatus":"ACTIVE",
|
||||||
|
"propagate":true,
|
||||||
|
"removePropagationsOnEntityDelete":false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"typeName":"Fact",
|
||||||
|
"entityGuid":"b233b2ae-8a4a-44a3-b446-4027462b2cc6",
|
||||||
|
"entityStatus":"ACTIVE",
|
||||||
|
"propagate":true,
|
||||||
|
"removePropagationsOnEntityDelete":false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"typeName":"Dimension",
|
||||||
|
"attributes":{
|
||||||
|
"customers":"None"
|
||||||
|
},
|
||||||
|
"entityGuid":"67101f5b-205a-4fd8-8273-273ccb45ca74",
|
||||||
|
"entityStatus":"ACTIVE",
|
||||||
|
"propagate":true,
|
||||||
|
"removePropagationsOnEntityDelete":false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"typeName":"ETL",
|
||||||
|
"entityGuid":"6d6ab7e5-646f-4161-937c-dc340d7c9d06",
|
||||||
|
"entityStatus":"ACTIVE",
|
||||||
|
"propagate":true,
|
||||||
|
"removePropagationsOnEntityDelete":false
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"labels":[
|
||||||
|
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
506
ingestion/tests/unit/topology/metadata/test_atlas.py
Normal file
506
ingestion/tests/unit/topology/metadata/test_atlas.py
Normal file
@ -0,0 +1,506 @@
|
|||||||
|
# Copyright 2021 Collate
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
"""
|
||||||
|
Test Atlas using the topology
|
||||||
|
"""
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
from unittest import TestCase
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
from pydantic import AnyUrl
|
||||||
|
|
||||||
|
from metadata.generated.schema.api.classification.createClassification import (
|
||||||
|
CreateClassificationRequest,
|
||||||
|
)
|
||||||
|
from metadata.generated.schema.api.classification.createTag import CreateTagRequest
|
||||||
|
from metadata.generated.schema.api.data.createDatabase import CreateDatabaseRequest
|
||||||
|
from metadata.generated.schema.api.data.createDatabaseSchema import (
|
||||||
|
CreateDatabaseSchemaRequest,
|
||||||
|
)
|
||||||
|
from metadata.generated.schema.api.data.createTable import CreateTableRequest
|
||||||
|
from metadata.generated.schema.api.services.createDatabaseService import (
|
||||||
|
CreateDatabaseServiceRequest,
|
||||||
|
)
|
||||||
|
from metadata.generated.schema.entity.data.database import Database
|
||||||
|
from metadata.generated.schema.entity.data.databaseSchema import DatabaseSchema
|
||||||
|
from metadata.generated.schema.entity.data.table import Column, Table
|
||||||
|
from metadata.generated.schema.entity.services.connections.database.hiveConnection import (
|
||||||
|
HiveConnection,
|
||||||
|
)
|
||||||
|
from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import (
|
||||||
|
OpenMetadataConnection,
|
||||||
|
)
|
||||||
|
from metadata.generated.schema.entity.services.databaseService import (
|
||||||
|
DatabaseConnection,
|
||||||
|
DatabaseService,
|
||||||
|
)
|
||||||
|
from metadata.generated.schema.metadataIngestion.workflow import (
|
||||||
|
OpenMetadataWorkflowConfig,
|
||||||
|
)
|
||||||
|
from metadata.generated.schema.type.basic import Href
|
||||||
|
from metadata.generated.schema.type.entityReference import EntityReference
|
||||||
|
from metadata.generated.schema.type.tagLabel import TagLabel
|
||||||
|
from metadata.ingestion.ometa.ometa_api import OpenMetadata
|
||||||
|
from metadata.ingestion.source.metadata.atlas.client import AtlasClient
|
||||||
|
from metadata.ingestion.source.metadata.atlas.metadata import AtlasSource
|
||||||
|
|
||||||
|
mock_atlas_config = {
|
||||||
|
"source": {
|
||||||
|
"type": "Atlas",
|
||||||
|
"serviceName": "local_atlas",
|
||||||
|
"serviceConnection": {
|
||||||
|
"config": {
|
||||||
|
"type": "Atlas",
|
||||||
|
"hostPort": "http://192.168.1.8:21000",
|
||||||
|
"username": "username",
|
||||||
|
"password": "password",
|
||||||
|
"databaseServiceName": ["hive"],
|
||||||
|
"messagingServiceName": [],
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"sourceConfig": {"config": {"type": "DatabaseMetadata"}},
|
||||||
|
},
|
||||||
|
"sink": {"type": "metadata-rest", "config": {}},
|
||||||
|
"workflowConfig": {
|
||||||
|
"openMetadataServerConfig": {
|
||||||
|
"hostPort": "http://localhost:8585/api",
|
||||||
|
"authProvider": "openmetadata",
|
||||||
|
"securityConfig": {
|
||||||
|
"jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGc"
|
||||||
|
"iOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE"
|
||||||
|
"2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXB"
|
||||||
|
"iEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fN"
|
||||||
|
"r3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3u"
|
||||||
|
"d-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg"
|
||||||
|
},
|
||||||
|
}
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
mock_file_path = (
|
||||||
|
Path(__file__).parent.parent.parent / "resources/datasets/atlas_dataset.json"
|
||||||
|
)
|
||||||
|
with open(mock_file_path, encoding="UTF-8") as file:
|
||||||
|
mock_data: dict = json.load(file)
|
||||||
|
|
||||||
|
|
||||||
|
LIST_ENTITIES = [
|
||||||
|
"b233b2ae-8a4a-44a3-b446-4027462b2cc6",
|
||||||
|
"eff274ef-f84d-4c58-81d6-c309663b887d",
|
||||||
|
"1697e9bc-2fb0-4d11-8294-7e0da9c74830",
|
||||||
|
"aabdc6c0-96c1-451c-98b5-7a7e48c319be",
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def mock_get_entity(self, table): # pylint: disable=unused-argument
|
||||||
|
return mock_data
|
||||||
|
|
||||||
|
|
||||||
|
def mock_list_entities(self, entity_type): # pylint: disable=unused-argument
|
||||||
|
return LIST_ENTITIES
|
||||||
|
|
||||||
|
|
||||||
|
EXPECTED_DATABASE_DESCRIPTION = "THIS IS TEST_DESCRIPTION FOR DATABASE"
|
||||||
|
|
||||||
|
EXPTECTED_DATABASE_SCHEMA_DESCRIPTION = "THIS IS TEST_DESCRIPTION FOR DATABASE"
|
||||||
|
|
||||||
|
EXPTECTED_TABLE = Table(
|
||||||
|
id="124d078d-dcf2-43a8-b59e-33bc7953f680",
|
||||||
|
name="delta_test_table",
|
||||||
|
displayName=None,
|
||||||
|
fullyQualifiedName="hive.Reporting.Reporting.delta_test_table",
|
||||||
|
description="sales fact daily materialized view",
|
||||||
|
version=0.1,
|
||||||
|
updatedAt=1673413042524,
|
||||||
|
updatedBy="admin",
|
||||||
|
href=Href(
|
||||||
|
__root__=AnyUrl(
|
||||||
|
"http://localhost:8585/api/v1/tables/124d078d-dcf2-43a8-b59e-33bc7953f680",
|
||||||
|
scheme="http",
|
||||||
|
host="localhost",
|
||||||
|
host_type="int_domain",
|
||||||
|
port="8585",
|
||||||
|
path="/api/v1/tables/124d078d-dcf2-43a8-b59e-33bc7953f680",
|
||||||
|
)
|
||||||
|
),
|
||||||
|
tableType="Regular",
|
||||||
|
columns=[
|
||||||
|
Column(
|
||||||
|
name="col2",
|
||||||
|
displayName=None,
|
||||||
|
dataType="STRUCT",
|
||||||
|
arrayDataType=None,
|
||||||
|
dataLength=1,
|
||||||
|
precision=None,
|
||||||
|
scale=None,
|
||||||
|
dataTypeDisplay="struct<col3:string,col4:bigint>",
|
||||||
|
description="col2 nested delta column",
|
||||||
|
fullyQualifiedName="hive.Reporting.Reporting.delta_test_table.col2",
|
||||||
|
tags=None,
|
||||||
|
constraint=None,
|
||||||
|
ordinalPosition=None,
|
||||||
|
jsonSchema=None,
|
||||||
|
children=[
|
||||||
|
Column(
|
||||||
|
name="col3",
|
||||||
|
displayName=None,
|
||||||
|
dataType="STRING",
|
||||||
|
arrayDataType=None,
|
||||||
|
dataLength=None,
|
||||||
|
precision=None,
|
||||||
|
scale=None,
|
||||||
|
dataTypeDisplay="string",
|
||||||
|
description=None,
|
||||||
|
fullyQualifiedName="hive.Reporting.Reporting.delta_test_table.col2.col3",
|
||||||
|
tags=None,
|
||||||
|
constraint=None,
|
||||||
|
ordinalPosition=None,
|
||||||
|
jsonSchema=None,
|
||||||
|
children=None,
|
||||||
|
customMetrics=None,
|
||||||
|
profile=None,
|
||||||
|
),
|
||||||
|
Column(
|
||||||
|
name="col4",
|
||||||
|
displayName=None,
|
||||||
|
dataType="BIGINT",
|
||||||
|
arrayDataType=None,
|
||||||
|
dataLength=None,
|
||||||
|
precision=None,
|
||||||
|
scale=None,
|
||||||
|
dataTypeDisplay="bigint",
|
||||||
|
description=None,
|
||||||
|
fullyQualifiedName="hive.Reporting.Reporting.delta_test_table.col2.col4",
|
||||||
|
tags=None,
|
||||||
|
constraint=None,
|
||||||
|
ordinalPosition=None,
|
||||||
|
jsonSchema=None,
|
||||||
|
children=None,
|
||||||
|
customMetrics=None,
|
||||||
|
profile=None,
|
||||||
|
),
|
||||||
|
],
|
||||||
|
customMetrics=None,
|
||||||
|
profile=None,
|
||||||
|
),
|
||||||
|
Column(
|
||||||
|
name="col1",
|
||||||
|
displayName=None,
|
||||||
|
dataType="BIGINT",
|
||||||
|
arrayDataType=None,
|
||||||
|
dataLength=1,
|
||||||
|
precision=None,
|
||||||
|
scale=None,
|
||||||
|
dataTypeDisplay="bigint",
|
||||||
|
description="col1 description",
|
||||||
|
fullyQualifiedName="hive.Reporting.Reporting.delta_test_table.col1",
|
||||||
|
tags=None,
|
||||||
|
constraint=None,
|
||||||
|
ordinalPosition=None,
|
||||||
|
jsonSchema=None,
|
||||||
|
children=None,
|
||||||
|
customMetrics=None,
|
||||||
|
profile=None,
|
||||||
|
),
|
||||||
|
],
|
||||||
|
tableConstraints=None,
|
||||||
|
tablePartition=None,
|
||||||
|
owner=None,
|
||||||
|
databaseSchema=EntityReference(
|
||||||
|
id="4cf6ee7e-9d24-4153-9318-82aa1167259b",
|
||||||
|
type="databaseSchema",
|
||||||
|
name="Reporting",
|
||||||
|
fullyQualifiedName="hive.Reporting.Reporting",
|
||||||
|
description="THIS IS TEST_DESCRIPTION FOR DATABASE",
|
||||||
|
displayName=None,
|
||||||
|
deleted=False,
|
||||||
|
href=Href(
|
||||||
|
__root__=AnyUrl(
|
||||||
|
"http://localhost:8585/api/v1/databaseSchemas/4cf6ee7e-9d24-4153-9318-82aa1167259b",
|
||||||
|
scheme="http",
|
||||||
|
host="localhost",
|
||||||
|
host_type="int_domain",
|
||||||
|
port="8585",
|
||||||
|
path="/api/v1/databaseSchemas/4cf6ee7e-9d24-4153-9318-82aa1167259b",
|
||||||
|
)
|
||||||
|
),
|
||||||
|
),
|
||||||
|
database=EntityReference(
|
||||||
|
id="367f53b5-d6c2-44be-bf5d-a0a1dc98a9dd",
|
||||||
|
type="database",
|
||||||
|
name="Reporting",
|
||||||
|
fullyQualifiedName="hive.Reporting",
|
||||||
|
description="THIS IS TEST_DESCRIPTION FOR DATABASE",
|
||||||
|
displayName=None,
|
||||||
|
deleted=False,
|
||||||
|
href=Href(
|
||||||
|
__root__=AnyUrl(
|
||||||
|
"http://localhost:8585/api/v1/databases/367f53b5-d6c2-44be-bf5d-a0a1dc98a9dd",
|
||||||
|
scheme="http",
|
||||||
|
host="localhost",
|
||||||
|
host_type="int_domain",
|
||||||
|
port="8585",
|
||||||
|
path="/api/v1/databases/367f53b5-d6c2-44be-bf5d-a0a1dc98a9dd",
|
||||||
|
)
|
||||||
|
),
|
||||||
|
),
|
||||||
|
service=EntityReference(
|
||||||
|
id="f2ab0e7a-5224-4acb-a189-74158851733f",
|
||||||
|
type="databaseService",
|
||||||
|
name="hive",
|
||||||
|
fullyQualifiedName="hive",
|
||||||
|
description=None,
|
||||||
|
displayName=None,
|
||||||
|
deleted=False,
|
||||||
|
href=Href(
|
||||||
|
__root__=AnyUrl(
|
||||||
|
"http://localhost:8585/api/v1/services/databaseServices/f2ab0e7a-5224-4acb-a189-74158851733f",
|
||||||
|
scheme="http",
|
||||||
|
host="localhost",
|
||||||
|
host_type="int_domain",
|
||||||
|
port="8585",
|
||||||
|
path="/api/v1/services/databaseServices/f2ab0e7a-5224-4acb-a189-74158851733f",
|
||||||
|
)
|
||||||
|
),
|
||||||
|
),
|
||||||
|
serviceType="Hive",
|
||||||
|
location=None,
|
||||||
|
viewDefinition=None,
|
||||||
|
tags=[
|
||||||
|
TagLabel(
|
||||||
|
tagFQN="AtlasMetadata.atlas_table",
|
||||||
|
description="test tag",
|
||||||
|
source="Tag",
|
||||||
|
labelType="Automated",
|
||||||
|
state="Confirmed",
|
||||||
|
href=None,
|
||||||
|
)
|
||||||
|
],
|
||||||
|
usageSummary=None,
|
||||||
|
followers=None,
|
||||||
|
joins=None,
|
||||||
|
sampleData=None,
|
||||||
|
tableProfilerConfig=None,
|
||||||
|
profile=None,
|
||||||
|
tableQueries=None,
|
||||||
|
dataModel=None,
|
||||||
|
changeDescription=None,
|
||||||
|
deleted=False,
|
||||||
|
extension=None,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class AtlasUnitTest(TestCase):
|
||||||
|
"""
|
||||||
|
Implements the necessary methods to extract
|
||||||
|
Atlas Metadata Unit Test
|
||||||
|
"""
|
||||||
|
|
||||||
|
@patch(
|
||||||
|
"metadata.ingestion.source.pipeline.pipeline_service.PipelineServiceSource.test_connection"
|
||||||
|
)
|
||||||
|
def __init__(self, methodName, test_connection) -> None:
|
||||||
|
super().__init__(methodName)
|
||||||
|
test_connection.return_value = False
|
||||||
|
self.config = OpenMetadataWorkflowConfig.parse_obj(mock_atlas_config)
|
||||||
|
self.atlas_source = AtlasSource.create(
|
||||||
|
mock_atlas_config["source"],
|
||||||
|
self.config.workflowConfig.openMetadataServerConfig,
|
||||||
|
)
|
||||||
|
self.metadata = OpenMetadata(
|
||||||
|
OpenMetadataConnection.parse_obj(
|
||||||
|
mock_atlas_config["workflowConfig"]["openMetadataServerConfig"]
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
self.database_service = (
|
||||||
|
mock_database_service_object
|
||||||
|
) = self.metadata.create_or_update(
|
||||||
|
CreateDatabaseServiceRequest(
|
||||||
|
name="hive",
|
||||||
|
serviceType="Hive",
|
||||||
|
connection=DatabaseConnection(
|
||||||
|
config=HiveConnection(
|
||||||
|
type="Hive",
|
||||||
|
scheme="hive",
|
||||||
|
username=None,
|
||||||
|
password=None,
|
||||||
|
hostPort="http://nohost:6000",
|
||||||
|
)
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
mock_database_object = self.metadata.create_or_update(
|
||||||
|
CreateDatabaseRequest(
|
||||||
|
name="Reporting",
|
||||||
|
displayName=None,
|
||||||
|
description=None,
|
||||||
|
tags=None,
|
||||||
|
owner=None,
|
||||||
|
service=EntityReference(
|
||||||
|
id=mock_database_service_object.id,
|
||||||
|
type="databaseService",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
mock_database_schema_object = self.metadata.create_or_update(
|
||||||
|
CreateDatabaseSchemaRequest(
|
||||||
|
name="Reporting",
|
||||||
|
database=EntityReference(id=mock_database_object.id, type="database"),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
_ = self.metadata.create_or_update(
|
||||||
|
CreateTableRequest(
|
||||||
|
name="sales_fact_daily_mv",
|
||||||
|
tableType="Regular",
|
||||||
|
columns=[
|
||||||
|
Column(
|
||||||
|
name="col2",
|
||||||
|
displayName=None,
|
||||||
|
dataType="STRUCT",
|
||||||
|
arrayDataType=None,
|
||||||
|
dataLength=1,
|
||||||
|
precision=None,
|
||||||
|
scale=None,
|
||||||
|
dataTypeDisplay="struct<col3:string,col4:bigint>",
|
||||||
|
description="col2 nested delta column",
|
||||||
|
fullyQualifiedName="delta.default.test_schema.delta_test_table.col2",
|
||||||
|
tags=None,
|
||||||
|
constraint=None,
|
||||||
|
ordinalPosition=None,
|
||||||
|
jsonSchema=None,
|
||||||
|
children=[
|
||||||
|
Column(
|
||||||
|
name="col3",
|
||||||
|
displayName=None,
|
||||||
|
dataType="STRING",
|
||||||
|
arrayDataType=None,
|
||||||
|
dataLength=None,
|
||||||
|
precision=None,
|
||||||
|
scale=None,
|
||||||
|
dataTypeDisplay="string",
|
||||||
|
description=None,
|
||||||
|
fullyQualifiedName="delta.default.test_schema.delta_test_table.col2.col3",
|
||||||
|
tags=None,
|
||||||
|
constraint=None,
|
||||||
|
ordinalPosition=None,
|
||||||
|
jsonSchema=None,
|
||||||
|
children=None,
|
||||||
|
customMetrics=None,
|
||||||
|
profile=None,
|
||||||
|
),
|
||||||
|
Column(
|
||||||
|
name="col4",
|
||||||
|
displayName=None,
|
||||||
|
dataType="BIGINT",
|
||||||
|
arrayDataType=None,
|
||||||
|
dataLength=None,
|
||||||
|
precision=None,
|
||||||
|
scale=None,
|
||||||
|
dataTypeDisplay="bigint",
|
||||||
|
description=None,
|
||||||
|
fullyQualifiedName="delta.default.test_schema.delta_test_table.col2.col4",
|
||||||
|
tags=None,
|
||||||
|
constraint=None,
|
||||||
|
ordinalPosition=None,
|
||||||
|
jsonSchema=None,
|
||||||
|
children=None,
|
||||||
|
customMetrics=None,
|
||||||
|
profile=None,
|
||||||
|
),
|
||||||
|
],
|
||||||
|
customMetrics=None,
|
||||||
|
profile=None,
|
||||||
|
),
|
||||||
|
Column(
|
||||||
|
name="col1",
|
||||||
|
displayName=None,
|
||||||
|
dataType="BIGINT",
|
||||||
|
arrayDataType=None,
|
||||||
|
dataLength=1,
|
||||||
|
precision=None,
|
||||||
|
scale=None,
|
||||||
|
dataTypeDisplay="bigint",
|
||||||
|
description="col1 description",
|
||||||
|
fullyQualifiedName="delta.default.test_schema.delta_test_table.col1",
|
||||||
|
tags=None,
|
||||||
|
constraint=None,
|
||||||
|
ordinalPosition=None,
|
||||||
|
jsonSchema=None,
|
||||||
|
children=None,
|
||||||
|
customMetrics=None,
|
||||||
|
profile=None,
|
||||||
|
),
|
||||||
|
],
|
||||||
|
databaseSchema=EntityReference(
|
||||||
|
id=mock_database_schema_object.id, type="databaseSchema"
|
||||||
|
),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
def mock_ingest_lineage(self, source_guid, name): # pylint: disable=unused-argument
|
||||||
|
return []
|
||||||
|
|
||||||
|
def mock_create_tag(self):
|
||||||
|
classification = CreateClassificationRequest(
|
||||||
|
description="test tag", name="AtlasMetadata"
|
||||||
|
)
|
||||||
|
|
||||||
|
self.metadata.create_or_update(classification)
|
||||||
|
self.metadata.create_or_update(
|
||||||
|
CreateTagRequest(
|
||||||
|
classification="AtlasMetadata",
|
||||||
|
name="atlas_table",
|
||||||
|
description="test tag",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
@patch.object(AtlasClient, "list_entities", mock_list_entities)
|
||||||
|
@patch.object(AtlasClient, "get_entity", mock_get_entity)
|
||||||
|
@patch.object(AtlasSource, "ingest_lineage", mock_ingest_lineage)
|
||||||
|
@patch.object(AtlasSource, "create_tag", mock_create_tag)
|
||||||
|
def test_description(self):
|
||||||
|
"""
|
||||||
|
Testing description updated for database, databaseSchema, table
|
||||||
|
"""
|
||||||
|
_ = list(self.atlas_source.next_record())
|
||||||
|
updated_database = self.metadata.get_by_name(
|
||||||
|
entity=Database, fqn="hive.Reporting"
|
||||||
|
)
|
||||||
|
assert updated_database.description.__root__ == EXPECTED_DATABASE_DESCRIPTION
|
||||||
|
|
||||||
|
updated_database_schema = self.metadata.get_by_name(
|
||||||
|
entity=DatabaseSchema, fqn="hive.Reporting.Reporting"
|
||||||
|
)
|
||||||
|
assert (
|
||||||
|
updated_database_schema.description.__root__
|
||||||
|
== EXPTECTED_DATABASE_SCHEMA_DESCRIPTION
|
||||||
|
)
|
||||||
|
|
||||||
|
updated_table = self.metadata.get_by_name(
|
||||||
|
entity=Table,
|
||||||
|
fqn="hive.Reporting.Reporting.sales_fact_daily_mv",
|
||||||
|
fields=["tags"],
|
||||||
|
)
|
||||||
|
|
||||||
|
assert updated_table.description == EXPTECTED_TABLE.description
|
||||||
|
|
||||||
|
assert updated_table.tags == EXPTECTED_TABLE.tags
|
||||||
|
|
||||||
|
self.metadata.delete(
|
||||||
|
entity=DatabaseService,
|
||||||
|
entity_id=self.atlas_source.service.id,
|
||||||
|
recursive=True,
|
||||||
|
hard_delete=True,
|
||||||
|
)
|
||||||
Loading…
x
Reference in New Issue
Block a user