yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-12-24 05:58:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix #15719: Improve unit test to increase coverage. (#15905)

Browse Source 
* issue-15719: unit test for superset db source

* issue-15719: use testcontainers for superset_api client test

* issue-15719: superset-api yield data changes

* fix failed test cases due to testcontainer version

* issue-15719: postgres container version fix

* issue-15719: setup & teardown with testcontainers

* issue-15719: remove more patch code
This commit is contained in:
harshsoni2024 2024-04-29 11:30:39 +05:30 committed by GitHub
parent d98a8c5cf1
commit 68e036418c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 343 additions and 288 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

127
ingestion/tests/unit/resources/datasets/superset_dataset.json
View File

@ -1,9 +1,9 @@
{
"dashboard": {
"count": 3,
"count": 1,
"description_columns": {},
"ids": [
14
1
],
"label_columns": {
"certification_details": "Certification Details",
@ -11,9 +11,7 @@
"changed_by.first_name": "Changed By First Name",
"changed_by.id": "Changed By Id",
"changed_by.last_name": "Changed By Last Name",
"changed_by.username": "Changed By Username",
"changed_by_name": "Changed By Name",
"changed_by_url": "Changed By Url",
"changed_on_delta_humanized": "Changed On Delta Humanized",
"changed_on_utc": "Changed On Utc",
"created_by.first_name": "Created By First Name",
@ -25,17 +23,18 @@
"id": "Id",
"is_managed_externally": "Is Managed Externally",
"json_metadata": "Json Metadata",
"owners.email": "Owners Email",
"owners.first_name": "Owners First Name",
"owners.id": "Owners Id",
"owners.last_name": "Owners Last Name",
"owners.username": "Owners Username",
"position_json": "Position Json",
"published": "Published",
"roles.id": "Roles Id",
"roles.name": "Roles Name",
"slug": "Slug",
"status": "Status",
"tags.id": "Tags Id",
"tags.name": "Tags Name",
"tags.type": "Tags Type",
"thumbnail_url": "Thumbnail Url",
"url": "Url"
},
@ -53,10 +52,8 @@
"certification_details",
"changed_by.first_name",
"changed_by.last_name",
"changed_by.username",
"changed_by.id",
"changed_by_name",
"changed_by_url",
"changed_on_utc",
"changed_on_delta_humanized",
"created_on_delta_humanized",
@ -65,13 +62,14 @@
"created_by.last_name",
"dashboard_title",
"owners.id",
"owners.username",
"owners.first_name",
"owners.last_name",
"owners.email",
"roles.id",
"roles.name",
"is_managed_externally"
"is_managed_externally",
"tags.id",
"tags.name",
"tags.type"
],
"list_title": "List Dashboard",
"order_columns": [
@ -84,86 +82,28 @@
],
"result": [
{
"certification_details": "",
"certified_by": "",
"changed_by": {
"first_name": "Superset",
"id": 1,
"last_name": "Admin",
"username": "admin"
},
"changed_by_name": "Superset Admin",
"changed_by_url": "/superset/profile/admin",
"changed_on_delta_humanized": "2 days ago",
"changed_on_utc": "2023-01-30T10:53:25.572977+0000",
"created_by": {
"first_name": "Superset",
"id": 1,
"last_name": "Admin"
},
"created_on_delta_humanized": "2 days ago",
"css": "",
"dashboard_title": "My DASH",
"id": 14,
"certification_details": null,
"certified_by": null,
"changed_by": null,
"changed_by_name": "",
"changed_on_delta_humanized": "10 minutes ago",
"changed_on_utc": "2024-04-18T13:23:17.562330+0000",
"created_by": null,
"created_on_delta_humanized": "10 minutes ago",
"css": null,
"dashboard_title": "Unicode Test",
"id": 10,
"is_managed_externally": false,
"json_metadata": "{\"show_native_filters\": true, \"color_scheme\": \"\", \"refresh_frequency\": 0, \"shared_label_colors\": {}, \"color_scheme_domain\": [], \"expanded_slices\": {}, \"label_colors\": {}, \"timed_refresh_immune_slices\": [], \"default_filters\": \"{}\", \"chart_configuration\": {}}",
"owners": [
{
"email": "admin@openmetadata.org",
"first_name": "Superset",
"id": 1,
"last_name": "Admin",
"username": "admin"
}
],
"position_json": "{\"CHART-dwSXo_0t5X\":{\"children\":[],\"id\":\"CHART-dwSXo_0t5X\",\"meta\":{\"chartId\":37,\"height\":50,\"sliceName\":\"% Rural\",\"uuid\":\"8f663401-854a-4da7-8e50-4b8e4ebb4f22\",\"width\":4},\"parents\":[\"ROOT_ID\",\"GRID_ID\",\"ROW-z_7odBWenK\"],\"type\":\"CHART\"},\"DASHBOARD_VERSION_KEY\":\"v2\",\"GRID_ID\":{\"children\":[\"ROW-z_7odBWenK\"],\"id\":\"GRID_ID\",\"parents\":[\"ROOT_ID\"],\"type\":\"GRID\"},\"HEADER_ID\":{\"id\":\"HEADER_ID\",\"meta\":{\"text\":\"My DASH\"},\"type\":\"HEADER\"},\"ROOT_ID\":{\"children\":[\"GRID_ID\"],\"id\":\"ROOT_ID\",\"type\":\"ROOT\"},\"ROW-z_7odBWenK\":{\"children\":[\"CHART-dwSXo_0t5X\"],\"id\":\"ROW-z_7odBWenK\",\"meta\":{\"background\":\"BACKGROUND_TRANSPARENT\"},\"parents\":[\"ROOT_ID\",\"GRID_ID\"],\"type\":\"ROW\"}}",
"json_metadata": "{}",
"owners": [],
"position_json": "{\"CHART-Hkx6154FEm\": {\"children\": [], \"id\": \"CHART-Hkx6154FEm\", \"meta\": {\"chartId\": 69, \"height\": 30, \"sliceName\": \"slice 1\", \"width\": 4, \"uuid\": \"609e26d8-8e1e-4097-9751-931708e24ee4\"}, \"type\": \"CHART\"}, \"GRID_ID\": {\"children\": [\"ROW-SyT19EFEQ\"], \"id\": \"GRID_ID\", \"type\": \"GRID\"}, \"ROOT_ID\": {\"children\": [\"GRID_ID\"], \"id\": \"ROOT_ID\", \"type\": \"ROOT\"}, \"ROW-SyT19EFEQ\": {\"children\": [\"CHART-Hkx6154FEm\"], \"id\": \"ROW-SyT19EFEQ\", \"meta\": {\"background\": \"BACKGROUND_TRANSPARENT\"}, \"type\": \"ROW\"}, \"DASHBOARD_VERSION_KEY\": \"v2\"}",
"published": true,
"roles": [],
"slug": null,
"slug": "unicode-test",
"status": "published",
"thumbnail_url": "/api/v1/dashboard/14/thumbnail/0088b55d1a7c34b5aa121f11252c11d5/",
"url": "/superset/dashboard/14/"
},
{
"certification_details": "",
"certified_by": "",
"changed_by": {
"first_name": "Superset",
"id": 1,
"last_name": "Admin",
"username": "admin"
},
"changed_by_name": "Superset Admin",
"changed_by_url": "/superset/profile/admin",
"changed_on_delta_humanized": "2 days ago",
"changed_on_utc": "2024-02-30T10:53:25.572977+0000",
"created_by": {
"first_name": "Superset",
"id": 1,
"last_name": "Admin"
},
"created_on_delta_humanized": "2 days ago",
"css": "",
"dashboard_title": "My DRAFT DASH",
"id": 15,
"is_managed_externally": false,
"json_metadata": "{\"show_native_filters\": true, \"color_scheme\": \"\", \"refresh_frequency\": 0, \"shared_label_colors\": {}, \"color_scheme_domain\": [], \"expanded_slices\": {}, \"label_colors\": {}, \"timed_refresh_immune_slices\": [], \"default_filters\": \"{}\", \"chart_configuration\": {}}",
"owners": [
{
"email": "admin@openmetadata.org",
"first_name": "Superset",
"id": 1,
"last_name": "Admin",
"username": "admin"
}
],
"position_json": "{\"CHART-dwSXo_0t5X\":{\"children\":[],\"id\":\"CHART-dwSXo_0t5X\",\"meta\":{\"chartId\":37,\"height\":50,\"sliceName\":\"% Rural\",\"uuid\":\"8f663401-854a-4da7-8e50-4b8e4ebb4f22\",\"width\":4},\"parents\":[\"ROOT_ID\",\"GRID_ID\",\"ROW-z_7odBWenK\"],\"type\":\"CHART\"},\"DASHBOARD_VERSION_KEY\":\"v2\",\"GRID_ID\":{\"children\":[\"ROW-z_7odBWenK\"],\"id\":\"GRID_ID\",\"parents\":[\"ROOT_ID\"],\"type\":\"GRID\"},\"HEADER_ID\":{\"id\":\"HEADER_ID\",\"meta\":{\"text\":\"My DASH\"},\"type\":\"HEADER\"},\"ROOT_ID\":{\"children\":[\"GRID_ID\"],\"id\":\"ROOT_ID\",\"type\":\"ROOT\"},\"ROW-z_7odBWenK\":{\"children\":[\"CHART-dwSXo_0t5X\"],\"id\":\"ROW-z_7odBWenK\",\"meta\":{\"background\":\"BACKGROUND_TRANSPARENT\"},\"parents\":[\"ROOT_ID\",\"GRID_ID\"],\"type\":\"ROW\"}}",
"published": false,
"roles": [],
"slug": null,
"status": "draft",
"thumbnail_url": "/api/v1/dashboard/15/thumbnail/0088b55d1a7c34b5aa121f11252c11d5/",
"url": "/superset/dashboard/15/"
"tags": [],
"thumbnail_url": "/api/v1/dashboard/10/thumbnail/874aa9f8f41b6956d5cf9740cf43aca4/",
"url": "/superset/dashboard/unicode-test/"
}
]
},
@ -661,7 +601,7 @@
"dashboard-db": {
"id": 14,
"dashboard_title": "My DASH",
"position_json":"{\"CHART-dwSXo_0t5X\":{\"children\":[],\"id\":\"CHART-dwSXo_0t5X\",\"meta\":{\"chartId\":37,\"height\":50,\"sliceName\":\"% Rural\",\"uuid\":\"8f663401-854a-4da7-8e50-4b8e4ebb4f22\",\"width\":4},\"parents\":[\"ROOT_ID\",\"GRID_ID\",\"ROW-z_7odBWenK\"],\"type\":\"CHART\"},\"DASHBOARD_VERSION_KEY\":\"v2\",\"GRID_ID\":{\"children\":[\"ROW-z_7odBWenK\"],\"id\":\"GRID_ID\",\"parents\":[\"ROOT_ID\"],\"type\":\"GRID\"},\"HEADER_ID\":{\"id\":\"HEADER_ID\",\"meta\":{\"text\":\"My DASH\"},\"type\":\"HEADER\"},\"ROOT_ID\":{\"children\":[\"GRID_ID\"],\"id\":\"ROOT_ID\",\"type\":\"ROOT\"},\"ROW-z_7odBWenK\":{\"children\":[\"CHART-dwSXo_0t5X\"],\"id\":\"ROW-z_7odBWenK\",\"meta\":{\"background\":\"BACKGROUND_TRANSPARENT\"},\"parents\":[\"ROOT_ID\",\"GRID_ID\"],\"type\":\"ROW\"}}",
"position_json":"{\"CHART-dwSXo_0t5X\":{\"children\":[],\"id\":\"CHART-dwSXo_0t5X\",\"meta\":{\"chartId\":1,\"height\":50,\"sliceName\":\"% Rural\",\"uuid\":\"8f663401-854a-4da7-8e50-4b8e4ebb4f22\",\"width\":4},\"parents\":[\"ROOT_ID\",\"GRID_ID\",\"ROW-z_7odBWenK\"],\"type\":\"CHART\"},\"DASHBOARD_VERSION_KEY\":\"v2\",\"GRID_ID\":{\"children\":[\"ROW-z_7odBWenK\"],\"id\":\"GRID_ID\",\"parents\":[\"ROOT_ID\"],\"type\":\"GRID\"},\"HEADER_ID\":{\"id\":\"HEADER_ID\",\"meta\":{\"text\":\"My DASH\"},\"type\":\"HEADER\"},\"ROOT_ID\":{\"children\":[\"GRID_ID\"],\"id\":\"ROOT_ID\",\"type\":\"ROOT\"},\"ROW-z_7odBWenK\":{\"children\":[\"CHART-dwSXo_0t5X\"],\"id\":\"ROW-z_7odBWenK\",\"meta\":{\"background\":\"BACKGROUND_TRANSPARENT\"},\"parents\":[\"ROOT_ID\",\"GRID_ID\"],\"type\":\"ROW\"}}",
"email" : "admin@openmetadata.org"
},
"chart-db": [
@ -673,6 +613,17 @@
"schema": "main",
"database_name": "test",
"sqlalchemy_uri": "postgres://user:pass@localhost:5432/examples"
},
{
"id":1,
"slice_name":"Rural",
"description":"desc",
"table_name":"sample_table",
"schema":"main",
"database_name":"test_db",
"sqlalchemy_uri":"postgres://user:pass@localhost:5432/examples",
"viz_type":"bar_chart",
"datasource_id":99
}]
}

504
ingestion/tests/unit/topology/dashboard/test_superset.py
View File

@ -16,13 +16,15 @@ import json
import uuid
from pathlib import Path
from unittest import TestCase
from unittest.mock import patch
from sqlalchemy.engine import Engine
import sqlalchemy
from testcontainers.core.generic import DockerContainer
from testcontainers.postgres import PostgresContainer
from metadata.generated.schema.api.data.createChart import CreateChartRequest
from metadata.generated.schema.api.data.createDashboard import CreateDashboardRequest
from metadata.generated.schema.entity.data.chart import Chart, ChartType
from metadata.generated.schema.entity.data.dashboard import DashboardType
from metadata.generated.schema.entity.services.connections.database.common.basicAuth import (
BasicAuth,
)
@ -45,23 +47,22 @@ from metadata.generated.schema.entity.services.databaseService import (
from metadata.generated.schema.metadataIngestion.workflow import (
OpenMetadataWorkflowConfig,
)
from metadata.generated.schema.type.basic import FullyQualifiedEntityName
from metadata.generated.schema.type.basic import (
EntityName,
FullyQualifiedEntityName,
SourceUrl,
)
from metadata.generated.schema.type.entityReference import EntityReference
from metadata.ingestion.api.steps import InvalidSourceException
from metadata.ingestion.ometa.mixins.server_mixin import OMetaServerMixin
from metadata.ingestion.ometa.ometa_api import OpenMetadata
from metadata.ingestion.source.dashboard.dashboard_service import DashboardServiceSource
from metadata.ingestion.source.dashboard.superset.api_source import SupersetAPISource
from metadata.ingestion.source.dashboard.superset.client import SupersetAPIClient
from metadata.ingestion.source.dashboard.superset.db_source import SupersetDBSource
from metadata.ingestion.source.dashboard.superset.metadata import SupersetSource
from metadata.ingestion.source.dashboard.superset.models import (
FetchChart,
FetchDashboard,
ListDatabaseResult,
SupersetChart,
SupersetDashboardCount,
SupersetDatasource,
)
mock_file_path = (
@ -72,79 +73,15 @@ with open(mock_file_path, encoding="UTF-8") as file:
MOCK_DASHBOARD_RESP = SupersetDashboardCount(**mock_data["dashboard"])
MOCK_DASHBOARD = MOCK_DASHBOARD_RESP.result[0]
PUBLISHED_DASHBOARD_COUNT = 1
PUBLISHED_DASHBOARD_NAME = "My DASH"
PUBLISHED_DASHBOARD_COUNT = 9
PUBLISHED_DASHBOARD_NAME = "Unicode Test"
MOCK_CHART_RESP = SupersetChart(**mock_data["chart"])
MOCK_CHART = MOCK_CHART_RESP.result[0]
MOCK_CHART_DB = FetchChart(**mock_data["chart-db"][0])
MOCK_CHART_DB_2 = FetchChart(**mock_data["chart-db"][1])
MOCK_DASHBOARD_DB = FetchDashboard(**mock_data["dashboard-db"])
MOCK_SUPERSET_API_CONFIG = {
"source": {
"type": "superset",
"serviceName": "test_supserset",
"serviceConnection": {
"config": {
"hostPort": "https://my-superset.com",
"type": "Superset",
"connection": {
"username": "admin",
"password": "admin",
"provider": "db",
},
}
},
"sourceConfig": {
"config": {"type": "DashboardMetadata", "includeDraftDashboard": False}
},
},
"sink": {"type": "metadata-rest", "config": {}},
"workflowConfig": {
"openMetadataServerConfig": {
"hostPort": "http://localhost:8585/api",
"authProvider": "openmetadata",
"securityConfig": {"jwtToken": "token"},
},
},
}
MOCK_SUPERSET_DB_CONFIG = {
"source": {
"type": "superset",
"serviceName": "test_supserset",
"serviceConnection": {
"config": {
"hostPort": "https://my-superset.com",
"type": "Superset",
"connection": {
"type": "Postgres",
"username": "superset",
"authType": {
"password": "superset",
},
"hostPort": "localhost:5432",
"database": "superset",
},
}
},
"sourceConfig": {
"config": {
"type": "DashboardMetadata",
}
},
},
"sink": {"type": "metadata-rest", "config": {}},
"workflowConfig": {
"openMetadataServerConfig": {
"hostPort": "http://localhost:8585/api",
"authProvider": "openmetadata",
"securityConfig": {"jwtToken": "token"},
},
},
}
EXPECTED_DASH_SERVICE = DashboardService(
id="c3eb265f-5445-4ad3-ba5e-797d3a3071bb",
fullyQualifiedName=FullyQualifiedEntityName(__root__="test_supserset"),
@ -152,7 +89,7 @@ EXPECTED_DASH_SERVICE = DashboardService(
connection=DashboardConnection(),
serviceType=DashboardServiceType.Superset,
)
EXPECTED_USER = EntityReference(id=uuid.uuid4(), type="user")
EXPECTED_USER = EntityReference(id="81af89aa-1bab-41aa-a567-5e68f78acdc0", type="user")
MOCK_DB_MYSQL_SERVICE_1 = DatabaseService(
id="c3eb265f-5445-4ad3-ba5e-797d3a307122",
@ -182,6 +119,16 @@ MOCK_DB_MYSQL_SERVICE_2 = DatabaseService(
),
serviceType=DatabaseServiceType.Mysql,
)
MOCK_DASHBOARD_INPUT = {
"certification_details": "sample certificate details",
"certified_by": "certified by unknown",
"css": "css",
"dashboard_title": "Top trades",
"external_url": "external url",
"slug": "top-trades",
"published": True,
"position_json": '{"CHART-dwSXo_0t5X":{"children":[],"id":"CHART-dwSXo_0t5X","meta":{"chartId":37,"height":50,"sliceName":"% Rural","uuid":"8f663401-854a-4da7-8e50-4b8e4ebb4f22","width":4},"parents":["ROOT_ID","GRID_ID","ROW-z_7odBWenK"],"type":"CHART"},"DASHBOARD_VERSION_KEY":"v2","GRID_ID":{"children":["ROW-z_7odBWenK"],"id":"GRID_ID","parents":["ROOT_ID"],"type":"GRID"},"HEADER_ID":{"id":"HEADER_ID","meta":{"text":"My DASH"},"type":"HEADER"},"ROOT_ID":{"children":["GRID_ID"],"id":"ROOT_ID","type":"ROOT"},"ROW-z_7odBWenK":{"children":["CHART-dwSXo_0t5X"],"id":"ROW-z_7odBWenK","meta":{"background":"BACKGROUND_TRANSPARENT"},"parents":["ROOT_ID","GRID_ID"],"type":"ROW"}}',
}
MOCK_DB_POSTGRES_SERVICE = DatabaseService(
id="c3eb265f-5445-4ad3-ba5e-797d3a307122",
@ -218,73 +165,265 @@ EXPECTED_DASH = CreateDashboardRequest(
owner=EXPECTED_USER,
)
EXPECTED_CHART = CreateChartRequest(
name=37,
displayName="% Rural",
description="TEST DESCRIPTION",
chartType=ChartType.Other.value,
sourceUrl="https://my-superset.com/explore/?slice_id=37",
service=EXPECTED_DASH_SERVICE.fullyQualifiedName,
EXPECTED_API_DASHBOARD = CreateDashboardRequest(
name=EntityName(__root__="10"),
displayName="Unicode Test",
description=None,
dashboardType=DashboardType.Dashboard.value,
sourceUrl=SourceUrl(
__root__="http://localhost:54510/superset/dashboard/unicode-test/"
),
project=None,
charts=[],
dataModels=None,
tags=None,
owner=None,
service=FullyQualifiedEntityName(__root__="test_supserset"),
extension=None,
domain=None,
dataProducts=None,
lifeCycle=None,
sourceHash=None,
)
EXPECTED_ALL_CHARTS = {37: MOCK_CHART}
EXPECTED_ALL_CHARTS_DB = {37: MOCK_CHART_DB}
EXPECTED_CHART = CreateChartRequest(
name=1,
displayName="Rural",
description="desc",
chartType=ChartType.Other.value,
sourceUrl="https://my-superset.com/explore/?slice_id=1",
service=EXPECTED_DASH_SERVICE.fullyQualifiedName,
)
EXPECTED_CHART_2 = CreateChartRequest(
name=EntityName(__root__="69"),
displayName="Unicode Cloud",
description=None,
chartType=ChartType.Other.value,
sourceUrl=SourceUrl(__root__="http://localhost:54510/explore/?slice_id=69"),
tags=None,
owner=None,
service=FullyQualifiedEntityName(__root__="test_supserset"),
domain=None,
dataProducts=None,
lifeCycle=None,
sourceHash=None,
)
# EXPECTED_ALL_CHARTS = {37: MOCK_CHART}
# EXPECTED_ALL_CHARTS_DB = {37: MOCK_CHART_DB}
EXPECTED_ALL_CHARTS_DB = {1: MOCK_CHART_DB_2}
NOT_FOUND_RESP = {"message": "Not found"}
EXPECTED_API_DATASET_FQN = None
EXPECTED_DATASET_FQN = "test_postgres.examples.main.wb_health_population"
def setup_sample_data(postgres_container):
engine = sqlalchemy.create_engine(postgres_container.get_connection_url())
with engine.begin() as connection:
CREATE_TABLE_AB_USER = """
CREATE TABLE ab_user (
id INT PRIMARY KEY,
username VARCHAR(50));
"""
CREATE_TABLE_DASHBOARDS = """
CREATE TABLE dashboards (
id INT PRIMARY KEY,
created_by_fk INT,
FOREIGN KEY (created_by_fk) REFERENCES ab_user(id));
"""
INSERT_AB_USER_DATA = """
INSERT INTO ab_user (id, username)
VALUES (1, 'test_user');
"""
INSERT_DASHBOARDS_DATA = """
INSERT INTO dashboards (id, created_by_fk)
VALUES (1, 1);
"""
CREATE_SLICES_TABLE = """
CREATE TABLE slices (
id INTEGER PRIMARY KEY,
slice_name VARCHAR(255),
description TEXT,
datasource_id INTEGER,
viz_type VARCHAR(255),
datasource_type VARCHAR(255)
)
"""
INSERT_SLICES_DATA = """
INSERT INTO slices(id, slice_name, description, datasource_id, viz_type, datasource_type)
VALUES (1, 'Rural', 'desc', 99, 'bar_chart', 'table');
"""
CREATE_DBS_TABLE = """
CREATE TABLE dbs (
id INTEGER PRIMARY KEY,
database_name VARCHAR(255),
sqlalchemy_uri TEXT
)
"""
INSERT_DBS_DATA = """
INSERT INTO dbs(id, database_name, sqlalchemy_uri)
VALUES (5, 'test_db', 'postgres://user:pass@localhost:5432/examples');
"""
CREATE_TABLES_TABLE = """
CREATE TABLE tables (
id INTEGER PRIMARY KEY,
table_name VARCHAR(255),
schema VARCHAR(255),
database_id INTEGER
);
"""
INSERT_TABLES_DATA = """
INSERT INTO tables(id, table_name, schema, database_id)
VALUES (99, 'sample_table', 'main', 5);
"""
connection.execute(sqlalchemy.text(CREATE_TABLE_AB_USER))
connection.execute(sqlalchemy.text(INSERT_AB_USER_DATA))
connection.execute(sqlalchemy.text(CREATE_TABLE_DASHBOARDS))
connection.execute(sqlalchemy.text(INSERT_DASHBOARDS_DATA))
connection.execute(sqlalchemy.text(CREATE_SLICES_TABLE))
connection.execute(sqlalchemy.text(INSERT_SLICES_DATA))
connection.execute(sqlalchemy.text(CREATE_DBS_TABLE))
connection.execute(sqlalchemy.text(INSERT_DBS_DATA))
connection.execute(sqlalchemy.text(CREATE_TABLES_TABLE))
connection.execute(sqlalchemy.text(INSERT_TABLES_DATA))
INITIAL_SETUP = True
superset_container = postgres_container = None
def set_testcontainers():
global INITIAL_SETUP, superset_container, postgres_container
if INITIAL_SETUP:
# postgres test container
postgres_container = PostgresContainer("postgres:16-alpine")
postgres_container.start()
setup_sample_data(postgres_container)
# superset testcontainer
superset_container = DockerContainer(image="apache/superset")
superset_container.with_env("SUPERSET_SECRET_KEY", "&3brfbcf192T!)$sabqbie")
superset_container.with_env("WTF_CSRF_ENABLED", False)
superset_container.with_exposed_ports(8088)
superset_container.start()
superset_container.exec(
"superset fab create-admin --username admin --firstname Superset --lastname Admin --email admin@superset.com --password admin"
)
superset_container.exec("superset db upgrade")
superset_container.exec("superset init")
superset_container.exec("superset load-examples")
INITIAL_SETUP = False
return superset_container, postgres_container
class SupersetUnitTest(TestCase):
"""
Validate how we work with Superset metadata
"""
@classmethod
def teardown_class(cls):
"""Teardown class"""
# stop containers
superset_container.stop()
postgres_container.stop()
def __init__(self, methodName) -> None:
super().__init__(methodName)
superset_container, postgres_container = set_testcontainers()
MOCK_SUPERSET_API_CONFIG = {
"source": {
"type": "superset",
"serviceName": "test_supserset",
"serviceConnection": {
"config": {
"hostPort": f"http://{superset_container.get_container_host_ip()}:{superset_container.get_exposed_port(8088)}",
"type": "Superset",
"connection": {
"username": "admin",
"password": "admin",
"provider": "db",
},
}
},
"sourceConfig": {
"config": {
"type": "DashboardMetadata",
"includeDraftDashboard": False,
}
},
},
"sink": {"type": "metadata-rest", "config": {}},
"workflowConfig": {
"openMetadataServerConfig": {
"hostPort": "http://localhost:8585/api",
"authProvider": "openmetadata",
"securityConfig": {
"jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg"
},
},
},
}
MOCK_SUPERSET_DB_CONFIG = {
"source": {
"type": "superset",
"serviceName": "test_supserset",
"serviceConnection": {
"config": {
"hostPort": f"http://{superset_container.get_container_host_ip()}:{superset_container.get_exposed_port(8088)}",
"type": "Superset",
"connection": {
"type": "Postgres",
"hostPort": f"{postgres_container.get_container_host_ip()}:{postgres_container.get_exposed_port(5432)}",
"username": postgres_container.env.get("POSTGRES_USER"),
"authType": {
"password": postgres_container.env.get(
"POSTGRES_PASSWORD"
)
},
"database": postgres_container.env.get("POSTGRES_DB"),
},
}
},
"sourceConfig": {
"config": {
"type": "DashboardMetadata",
}
},
},
"sink": {"type": "metadata-rest", "config": {}},
"workflowConfig": {
"openMetadataServerConfig": {
"hostPort": "http://localhost:8585/api",
"authProvider": "openmetadata",
"securityConfig": {"jwtToken": "token"},
},
},
}
self.config = OpenMetadataWorkflowConfig.parse_obj(MOCK_SUPERSET_API_CONFIG)
with patch.object(
DashboardServiceSource, "test_connection", return_value=False
), patch.object(OMetaServerMixin, "validate_versions", return_value=True):
# This already validates that the source can be initialized
self.superset_api: SupersetSource = SupersetSource.create(
MOCK_SUPERSET_API_CONFIG["source"],
OpenMetadata(self.config.workflowConfig.openMetadataServerConfig),
)
self.superset_api: SupersetSource = SupersetSource.create(
MOCK_SUPERSET_API_CONFIG["source"],
OpenMetadata(self.config.workflowConfig.openMetadataServerConfig),
)
self.assertEqual(type(self.superset_api), SupersetAPISource)
self.superset_api.context.get().__dict__[
"dashboard_service"
] = EXPECTED_DASH_SERVICE.fullyQualifiedName.__root__
self.assertEqual(type(self.superset_api), SupersetAPISource)
self.superset_api.context.get().__dict__[
"dashboard_service"
] = EXPECTED_DASH_SERVICE.fullyQualifiedName.__root__
with patch.object(
SupersetAPIClient, "fetch_total_charts", return_value=1
), patch.object(
SupersetAPIClient, "fetch_charts", return_value=MOCK_CHART_RESP
):
self.superset_api.prepare()
self.assertEqual(EXPECTED_ALL_CHARTS, self.superset_api.all_charts)
with patch.object(
DashboardServiceSource, "test_connection", return_value=False
), patch.object(OMetaServerMixin, "validate_versions", return_value=True):
# This already validates that the source can be initialized
self.superset_db: SupersetSource = SupersetSource.create(
MOCK_SUPERSET_DB_CONFIG["source"],
OpenMetadata(self.config.workflowConfig.openMetadataServerConfig),
)
self.assertEqual(type(self.superset_db), SupersetDBSource)
self.superset_db.context.get().__dict__[
"dashboard_service"
] = EXPECTED_DASH_SERVICE.fullyQualifiedName.__root__
with patch.object(Engine, "execute", return_value=mock_data["chart-db"]):
self.superset_db.prepare()
self.assertEqual(EXPECTED_ALL_CHARTS_DB, self.superset_db.all_charts)
self.superset_db: SupersetSource = SupersetSource.create(
MOCK_SUPERSET_DB_CONFIG["source"],
OpenMetadata(self.config.workflowConfig.openMetadataServerConfig),
)
self.assertEqual(type(self.superset_db), SupersetDBSource)
self.superset_db.context.get().__dict__[
"dashboard_service"
] = EXPECTED_DASH_SERVICE.fullyQualifiedName.__root__
def test_create(self):
"""
@ -318,36 +457,12 @@ class SupersetUnitTest(TestCase):
self.config.workflowConfig.openMetadataServerConfig,
)
def test_api_perpare(self):
pass
def test_api_get_dashboards_list(self):
"""
Mock the client and check that we get a list
"""
with patch.object(
SupersetAPIClient, "fetch_total_dashboards", return_value=1
), patch.object(
SupersetAPIClient, "fetch_dashboards", return_value=MOCK_DASHBOARD_RESP
):
dashboard_list = self.superset_api.get_dashboards_list()
self.assertEqual(list(dashboard_list), [MOCK_DASHBOARD])
def test_api_get_published_dashboards_list(self):
"""
Mock the client and check that we get only published dashboards list
"""
with patch.object(
SupersetAPIClient, "fetch_total_dashboards", return_value=1
), patch.object(
SupersetAPIClient, "fetch_dashboards", return_value=MOCK_DASHBOARD_RESP
):
dashboard_list = list(self.superset_api.get_dashboards_list())
self.assertEqual(len(dashboard_list), PUBLISHED_DASHBOARD_COUNT)
self.assertEqual(
dashboard_list[0].dashboard_title, PUBLISHED_DASHBOARD_NAME
)
dashboard_list = list(self.superset_api.get_dashboards_list())
self.assertEqual(len(dashboard_list), PUBLISHED_DASHBOARD_COUNT)
def test_charts_of_dashboard(self):
"""
@ -356,7 +471,14 @@ class SupersetUnitTest(TestCase):
result = self.superset_api._get_charts_of_dashboard( # pylint: disable=protected-access
MOCK_DASHBOARD
)
self.assertEqual(result, [37])
self.assertEqual(result, [69])
def test_fetch_chart_db(self):
"""
test fetch chart method of db source
"""
self.superset_db.prepare()
self.assertEqual(EXPECTED_ALL_CHARTS_DB, self.superset_db.all_charts)
def test_dashboard_name(self):
dashboard_name = self.superset_api.get_dashboard_name(MOCK_DASHBOARD)
@ -364,71 +486,53 @@ class SupersetUnitTest(TestCase):
def test_yield_dashboard(self):
# TEST API SOURCE
with patch.object(
SupersetAPISource, "_get_user_by_email", return_value=EXPECTED_USER
):
self.superset_api.context.get().__dict__["charts"] = [
chart.name.__root__ for chart in EXPECTED_CHART_ENTITY
]
dashboard = next(self.superset_api.yield_dashboard(MOCK_DASHBOARD)).right
self.assertEqual(dashboard, EXPECTED_DASH)
dashboard = next(self.superset_api.yield_dashboard(MOCK_DASHBOARD)).right
EXPECTED_API_DASHBOARD.sourceUrl = SourceUrl(
__root__=f"http://{superset_container.get_container_host_ip()}:{superset_container.get_exposed_port(8088)}{MOCK_DASHBOARD.url}"
)
self.assertEqual(dashboard, EXPECTED_API_DASHBOARD)
# TEST DB SOURCE
with patch.object(
SupersetDBSource, "_get_user_by_email", return_value=EXPECTED_USER
):
self.superset_db.context.get().__dict__["charts"] = [
chart.name.__root__ for chart in EXPECTED_CHART_ENTITY
]
dashboard = next(self.superset_db.yield_dashboard(MOCK_DASHBOARD_DB)).right
self.assertEqual(dashboard, EXPECTED_DASH)
self.superset_db.context.get().__dict__["charts"] = [
chart.name.__root__ for chart in EXPECTED_CHART_ENTITY
]
dashboard = next(self.superset_db.yield_dashboard(MOCK_DASHBOARD_DB)).right
EXPECTED_DASH.sourceUrl = SourceUrl(
__root__=f"http://{superset_container.get_container_host_ip()}:{superset_container.get_exposed_port(8088)}/superset/dashboard/14/"
)
EXPECTED_DASH.owner = dashboard.owner
self.assertEqual(dashboard, EXPECTED_DASH)
def test_yield_dashboard_chart(self):
# TEST API SOURCE
dashboard_charts = next(
self.superset_api.prepare()
dashboard_chart = next(
self.superset_api.yield_dashboard_chart(MOCK_DASHBOARD)
).right
self.assertEqual(dashboard_charts, EXPECTED_CHART)
EXPECTED_CHART_2.sourceUrl = SourceUrl(
__root__=f"http://{superset_container.get_container_host_ip()}:{superset_container.get_exposed_port(8088)}/explore/?slice_id=69"
)
EXPECTED_CHART_2.displayName = dashboard_chart.displayName
self.assertEqual(dashboard_chart, EXPECTED_CHART_2)
# TEST DB SOURCE
self.superset_db.prepare()
dashboard_charts = next(
self.superset_db.yield_dashboard_chart(MOCK_DASHBOARD_DB)
).right
EXPECTED_CHART.sourceUrl = SourceUrl(
__root__=f"http://{superset_container.get_container_host_ip()}:{superset_container.get_exposed_port(8088)}/explore/?slice_id=1"
)
self.assertEqual(dashboard_charts, EXPECTED_CHART)
def test_api_get_datasource_fqn(self):
"""
Test generated datasource fqn for api source
"""
with patch.object(
OpenMetadata, "es_search_from_fqn", return_value=None
), patch.object(
SupersetAPIClient,
"fetch_datasource",
return_value=SupersetDatasource(**mock_data["datasource"]),
), patch.object(
SupersetAPIClient,
"fetch_database",
return_value=ListDatabaseResult(**mock_data["database"]),
):
fqn = self.superset_api._get_datasource_fqn( # pylint: disable=protected-access
1, MOCK_DB_POSTGRES_SERVICE
)
self.assertEqual(fqn, EXPECTED_DATASET_FQN)
with patch.object(
OpenMetadata, "es_search_from_fqn", return_value=None
), patch.object(
SupersetAPIClient,
"fetch_datasource",
return_value=SupersetDatasource(**mock_data["datasource"]),
), patch.object(
SupersetAPIClient, "fetch_database", return_value=ListDatabaseResult()
):
fqn = self.superset_api._get_datasource_fqn( # pylint: disable=protected-access
1, MOCK_DB_POSTGRES_SERVICE
)
self.assertEqual(fqn, None)
fqn = self.superset_api._get_datasource_fqn( # pylint: disable=protected-access
1, MOCK_DB_POSTGRES_SERVICE
)
self.assertEqual(fqn, EXPECTED_API_DATASET_FQN)
def test_db_get_datasource_fqn_for_lineage(self):
fqn = self.superset_db._get_datasource_fqn_for_lineage( # pylint: disable=protected-access