From 68e036418c3e54da3412a1d65e5533db6e72b8c3 Mon Sep 17 00:00:00 2001 From: harshsoni2024 <64592571+harshsoni2024@users.noreply.github.com> Date: Mon, 29 Apr 2024 11:30:39 +0530 Subject: [PATCH] Fix #15719: Improve unit test to increase coverage. (#15905) * issue-15719: unit test for superset db source * issue-15719: use testcontainers for superset_api client test * issue-15719: superset-api yield data changes * fix failed test cases due to testcontainer version * issue-15719: postgres container version fix * issue-15719: setup & teardown with testcontainers * issue-15719: remove more patch code --- .../resources/datasets/superset_dataset.json | 127 ++--- .../unit/topology/dashboard/test_superset.py | 504 +++++++++++------- 2 files changed, 343 insertions(+), 288 deletions(-) diff --git a/ingestion/tests/unit/resources/datasets/superset_dataset.json b/ingestion/tests/unit/resources/datasets/superset_dataset.json index 0c474cab701..bf510f745f8 100644 --- a/ingestion/tests/unit/resources/datasets/superset_dataset.json +++ b/ingestion/tests/unit/resources/datasets/superset_dataset.json @@ -1,9 +1,9 @@ { "dashboard": { - "count": 3, + "count": 1, "description_columns": {}, "ids": [ - 14 + 1 ], "label_columns": { "certification_details": "Certification Details", @@ -11,9 +11,7 @@ "changed_by.first_name": "Changed By First Name", "changed_by.id": "Changed By Id", "changed_by.last_name": "Changed By Last Name", - "changed_by.username": "Changed By Username", "changed_by_name": "Changed By Name", - "changed_by_url": "Changed By Url", "changed_on_delta_humanized": "Changed On Delta Humanized", "changed_on_utc": "Changed On Utc", "created_by.first_name": "Created By First Name", @@ -25,17 +23,18 @@ "id": "Id", "is_managed_externally": "Is Managed Externally", "json_metadata": "Json Metadata", - "owners.email": "Owners Email", "owners.first_name": "Owners First Name", "owners.id": "Owners Id", "owners.last_name": "Owners Last Name", - "owners.username": "Owners Username", "position_json": "Position Json", "published": "Published", "roles.id": "Roles Id", "roles.name": "Roles Name", "slug": "Slug", "status": "Status", + "tags.id": "Tags Id", + "tags.name": "Tags Name", + "tags.type": "Tags Type", "thumbnail_url": "Thumbnail Url", "url": "Url" }, @@ -53,10 +52,8 @@ "certification_details", "changed_by.first_name", "changed_by.last_name", - "changed_by.username", "changed_by.id", "changed_by_name", - "changed_by_url", "changed_on_utc", "changed_on_delta_humanized", "created_on_delta_humanized", @@ -65,13 +62,14 @@ "created_by.last_name", "dashboard_title", "owners.id", - "owners.username", "owners.first_name", "owners.last_name", - "owners.email", "roles.id", "roles.name", - "is_managed_externally" + "is_managed_externally", + "tags.id", + "tags.name", + "tags.type" ], "list_title": "List Dashboard", "order_columns": [ @@ -84,86 +82,28 @@ ], "result": [ { - "certification_details": "", - "certified_by": "", - "changed_by": { - "first_name": "Superset", - "id": 1, - "last_name": "Admin", - "username": "admin" - }, - "changed_by_name": "Superset Admin", - "changed_by_url": "/superset/profile/admin", - "changed_on_delta_humanized": "2 days ago", - "changed_on_utc": "2023-01-30T10:53:25.572977+0000", - "created_by": { - "first_name": "Superset", - "id": 1, - "last_name": "Admin" - }, - "created_on_delta_humanized": "2 days ago", - "css": "", - "dashboard_title": "My DASH", - "id": 14, + "certification_details": null, + "certified_by": null, + "changed_by": null, + "changed_by_name": "", + "changed_on_delta_humanized": "10 minutes ago", + "changed_on_utc": "2024-04-18T13:23:17.562330+0000", + "created_by": null, + "created_on_delta_humanized": "10 minutes ago", + "css": null, + "dashboard_title": "Unicode Test", + "id": 10, "is_managed_externally": false, - "json_metadata": "{\"show_native_filters\": true, \"color_scheme\": \"\", \"refresh_frequency\": 0, \"shared_label_colors\": {}, \"color_scheme_domain\": [], \"expanded_slices\": {}, \"label_colors\": {}, \"timed_refresh_immune_slices\": [], \"default_filters\": \"{}\", \"chart_configuration\": {}}", - "owners": [ - { - "email": "admin@openmetadata.org", - "first_name": "Superset", - "id": 1, - "last_name": "Admin", - "username": "admin" - } - ], - "position_json": "{\"CHART-dwSXo_0t5X\":{\"children\":[],\"id\":\"CHART-dwSXo_0t5X\",\"meta\":{\"chartId\":37,\"height\":50,\"sliceName\":\"% Rural\",\"uuid\":\"8f663401-854a-4da7-8e50-4b8e4ebb4f22\",\"width\":4},\"parents\":[\"ROOT_ID\",\"GRID_ID\",\"ROW-z_7odBWenK\"],\"type\":\"CHART\"},\"DASHBOARD_VERSION_KEY\":\"v2\",\"GRID_ID\":{\"children\":[\"ROW-z_7odBWenK\"],\"id\":\"GRID_ID\",\"parents\":[\"ROOT_ID\"],\"type\":\"GRID\"},\"HEADER_ID\":{\"id\":\"HEADER_ID\",\"meta\":{\"text\":\"My DASH\"},\"type\":\"HEADER\"},\"ROOT_ID\":{\"children\":[\"GRID_ID\"],\"id\":\"ROOT_ID\",\"type\":\"ROOT\"},\"ROW-z_7odBWenK\":{\"children\":[\"CHART-dwSXo_0t5X\"],\"id\":\"ROW-z_7odBWenK\",\"meta\":{\"background\":\"BACKGROUND_TRANSPARENT\"},\"parents\":[\"ROOT_ID\",\"GRID_ID\"],\"type\":\"ROW\"}}", + "json_metadata": "{}", + "owners": [], + "position_json": "{\"CHART-Hkx6154FEm\": {\"children\": [], \"id\": \"CHART-Hkx6154FEm\", \"meta\": {\"chartId\": 69, \"height\": 30, \"sliceName\": \"slice 1\", \"width\": 4, \"uuid\": \"609e26d8-8e1e-4097-9751-931708e24ee4\"}, \"type\": \"CHART\"}, \"GRID_ID\": {\"children\": [\"ROW-SyT19EFEQ\"], \"id\": \"GRID_ID\", \"type\": \"GRID\"}, \"ROOT_ID\": {\"children\": [\"GRID_ID\"], \"id\": \"ROOT_ID\", \"type\": \"ROOT\"}, \"ROW-SyT19EFEQ\": {\"children\": [\"CHART-Hkx6154FEm\"], \"id\": \"ROW-SyT19EFEQ\", \"meta\": {\"background\": \"BACKGROUND_TRANSPARENT\"}, \"type\": \"ROW\"}, \"DASHBOARD_VERSION_KEY\": \"v2\"}", "published": true, "roles": [], - "slug": null, + "slug": "unicode-test", "status": "published", - "thumbnail_url": "/api/v1/dashboard/14/thumbnail/0088b55d1a7c34b5aa121f11252c11d5/", - "url": "/superset/dashboard/14/" - }, - { - "certification_details": "", - "certified_by": "", - "changed_by": { - "first_name": "Superset", - "id": 1, - "last_name": "Admin", - "username": "admin" - }, - "changed_by_name": "Superset Admin", - "changed_by_url": "/superset/profile/admin", - "changed_on_delta_humanized": "2 days ago", - "changed_on_utc": "2024-02-30T10:53:25.572977+0000", - "created_by": { - "first_name": "Superset", - "id": 1, - "last_name": "Admin" - }, - "created_on_delta_humanized": "2 days ago", - "css": "", - "dashboard_title": "My DRAFT DASH", - "id": 15, - "is_managed_externally": false, - "json_metadata": "{\"show_native_filters\": true, \"color_scheme\": \"\", \"refresh_frequency\": 0, \"shared_label_colors\": {}, \"color_scheme_domain\": [], \"expanded_slices\": {}, \"label_colors\": {}, \"timed_refresh_immune_slices\": [], \"default_filters\": \"{}\", \"chart_configuration\": {}}", - "owners": [ - { - "email": "admin@openmetadata.org", - "first_name": "Superset", - "id": 1, - "last_name": "Admin", - "username": "admin" - } - ], - "position_json": "{\"CHART-dwSXo_0t5X\":{\"children\":[],\"id\":\"CHART-dwSXo_0t5X\",\"meta\":{\"chartId\":37,\"height\":50,\"sliceName\":\"% Rural\",\"uuid\":\"8f663401-854a-4da7-8e50-4b8e4ebb4f22\",\"width\":4},\"parents\":[\"ROOT_ID\",\"GRID_ID\",\"ROW-z_7odBWenK\"],\"type\":\"CHART\"},\"DASHBOARD_VERSION_KEY\":\"v2\",\"GRID_ID\":{\"children\":[\"ROW-z_7odBWenK\"],\"id\":\"GRID_ID\",\"parents\":[\"ROOT_ID\"],\"type\":\"GRID\"},\"HEADER_ID\":{\"id\":\"HEADER_ID\",\"meta\":{\"text\":\"My DASH\"},\"type\":\"HEADER\"},\"ROOT_ID\":{\"children\":[\"GRID_ID\"],\"id\":\"ROOT_ID\",\"type\":\"ROOT\"},\"ROW-z_7odBWenK\":{\"children\":[\"CHART-dwSXo_0t5X\"],\"id\":\"ROW-z_7odBWenK\",\"meta\":{\"background\":\"BACKGROUND_TRANSPARENT\"},\"parents\":[\"ROOT_ID\",\"GRID_ID\"],\"type\":\"ROW\"}}", - "published": false, - "roles": [], - "slug": null, - "status": "draft", - "thumbnail_url": "/api/v1/dashboard/15/thumbnail/0088b55d1a7c34b5aa121f11252c11d5/", - "url": "/superset/dashboard/15/" + "tags": [], + "thumbnail_url": "/api/v1/dashboard/10/thumbnail/874aa9f8f41b6956d5cf9740cf43aca4/", + "url": "/superset/dashboard/unicode-test/" } ] }, @@ -661,7 +601,7 @@ "dashboard-db": { "id": 14, "dashboard_title": "My DASH", - "position_json":"{\"CHART-dwSXo_0t5X\":{\"children\":[],\"id\":\"CHART-dwSXo_0t5X\",\"meta\":{\"chartId\":37,\"height\":50,\"sliceName\":\"% Rural\",\"uuid\":\"8f663401-854a-4da7-8e50-4b8e4ebb4f22\",\"width\":4},\"parents\":[\"ROOT_ID\",\"GRID_ID\",\"ROW-z_7odBWenK\"],\"type\":\"CHART\"},\"DASHBOARD_VERSION_KEY\":\"v2\",\"GRID_ID\":{\"children\":[\"ROW-z_7odBWenK\"],\"id\":\"GRID_ID\",\"parents\":[\"ROOT_ID\"],\"type\":\"GRID\"},\"HEADER_ID\":{\"id\":\"HEADER_ID\",\"meta\":{\"text\":\"My DASH\"},\"type\":\"HEADER\"},\"ROOT_ID\":{\"children\":[\"GRID_ID\"],\"id\":\"ROOT_ID\",\"type\":\"ROOT\"},\"ROW-z_7odBWenK\":{\"children\":[\"CHART-dwSXo_0t5X\"],\"id\":\"ROW-z_7odBWenK\",\"meta\":{\"background\":\"BACKGROUND_TRANSPARENT\"},\"parents\":[\"ROOT_ID\",\"GRID_ID\"],\"type\":\"ROW\"}}", + "position_json":"{\"CHART-dwSXo_0t5X\":{\"children\":[],\"id\":\"CHART-dwSXo_0t5X\",\"meta\":{\"chartId\":1,\"height\":50,\"sliceName\":\"% Rural\",\"uuid\":\"8f663401-854a-4da7-8e50-4b8e4ebb4f22\",\"width\":4},\"parents\":[\"ROOT_ID\",\"GRID_ID\",\"ROW-z_7odBWenK\"],\"type\":\"CHART\"},\"DASHBOARD_VERSION_KEY\":\"v2\",\"GRID_ID\":{\"children\":[\"ROW-z_7odBWenK\"],\"id\":\"GRID_ID\",\"parents\":[\"ROOT_ID\"],\"type\":\"GRID\"},\"HEADER_ID\":{\"id\":\"HEADER_ID\",\"meta\":{\"text\":\"My DASH\"},\"type\":\"HEADER\"},\"ROOT_ID\":{\"children\":[\"GRID_ID\"],\"id\":\"ROOT_ID\",\"type\":\"ROOT\"},\"ROW-z_7odBWenK\":{\"children\":[\"CHART-dwSXo_0t5X\"],\"id\":\"ROW-z_7odBWenK\",\"meta\":{\"background\":\"BACKGROUND_TRANSPARENT\"},\"parents\":[\"ROOT_ID\",\"GRID_ID\"],\"type\":\"ROW\"}}", "email" : "admin@openmetadata.org" }, "chart-db": [ @@ -673,6 +613,17 @@ "schema": "main", "database_name": "test", "sqlalchemy_uri": "postgres://user:pass@localhost:5432/examples" + }, + { + "id":1, + "slice_name":"Rural", + "description":"desc", + "table_name":"sample_table", + "schema":"main", + "database_name":"test_db", + "sqlalchemy_uri":"postgres://user:pass@localhost:5432/examples", + "viz_type":"bar_chart", + "datasource_id":99 }] } \ No newline at end of file diff --git a/ingestion/tests/unit/topology/dashboard/test_superset.py b/ingestion/tests/unit/topology/dashboard/test_superset.py index f4e707917fa..d1d6aeacce8 100644 --- a/ingestion/tests/unit/topology/dashboard/test_superset.py +++ b/ingestion/tests/unit/topology/dashboard/test_superset.py @@ -16,13 +16,15 @@ import json import uuid from pathlib import Path from unittest import TestCase -from unittest.mock import patch -from sqlalchemy.engine import Engine +import sqlalchemy +from testcontainers.core.generic import DockerContainer +from testcontainers.postgres import PostgresContainer from metadata.generated.schema.api.data.createChart import CreateChartRequest from metadata.generated.schema.api.data.createDashboard import CreateDashboardRequest from metadata.generated.schema.entity.data.chart import Chart, ChartType +from metadata.generated.schema.entity.data.dashboard import DashboardType from metadata.generated.schema.entity.services.connections.database.common.basicAuth import ( BasicAuth, ) @@ -45,23 +47,22 @@ from metadata.generated.schema.entity.services.databaseService import ( from metadata.generated.schema.metadataIngestion.workflow import ( OpenMetadataWorkflowConfig, ) -from metadata.generated.schema.type.basic import FullyQualifiedEntityName +from metadata.generated.schema.type.basic import ( + EntityName, + FullyQualifiedEntityName, + SourceUrl, +) from metadata.generated.schema.type.entityReference import EntityReference from metadata.ingestion.api.steps import InvalidSourceException -from metadata.ingestion.ometa.mixins.server_mixin import OMetaServerMixin from metadata.ingestion.ometa.ometa_api import OpenMetadata -from metadata.ingestion.source.dashboard.dashboard_service import DashboardServiceSource from metadata.ingestion.source.dashboard.superset.api_source import SupersetAPISource -from metadata.ingestion.source.dashboard.superset.client import SupersetAPIClient from metadata.ingestion.source.dashboard.superset.db_source import SupersetDBSource from metadata.ingestion.source.dashboard.superset.metadata import SupersetSource from metadata.ingestion.source.dashboard.superset.models import ( FetchChart, FetchDashboard, - ListDatabaseResult, SupersetChart, SupersetDashboardCount, - SupersetDatasource, ) mock_file_path = ( @@ -72,79 +73,15 @@ with open(mock_file_path, encoding="UTF-8") as file: MOCK_DASHBOARD_RESP = SupersetDashboardCount(**mock_data["dashboard"]) MOCK_DASHBOARD = MOCK_DASHBOARD_RESP.result[0] -PUBLISHED_DASHBOARD_COUNT = 1 -PUBLISHED_DASHBOARD_NAME = "My DASH" +PUBLISHED_DASHBOARD_COUNT = 9 +PUBLISHED_DASHBOARD_NAME = "Unicode Test" MOCK_CHART_RESP = SupersetChart(**mock_data["chart"]) MOCK_CHART = MOCK_CHART_RESP.result[0] MOCK_CHART_DB = FetchChart(**mock_data["chart-db"][0]) +MOCK_CHART_DB_2 = FetchChart(**mock_data["chart-db"][1]) MOCK_DASHBOARD_DB = FetchDashboard(**mock_data["dashboard-db"]) -MOCK_SUPERSET_API_CONFIG = { - "source": { - "type": "superset", - "serviceName": "test_supserset", - "serviceConnection": { - "config": { - "hostPort": "https://my-superset.com", - "type": "Superset", - "connection": { - "username": "admin", - "password": "admin", - "provider": "db", - }, - } - }, - "sourceConfig": { - "config": {"type": "DashboardMetadata", "includeDraftDashboard": False} - }, - }, - "sink": {"type": "metadata-rest", "config": {}}, - "workflowConfig": { - "openMetadataServerConfig": { - "hostPort": "http://localhost:8585/api", - "authProvider": "openmetadata", - "securityConfig": {"jwtToken": "token"}, - }, - }, -} - - -MOCK_SUPERSET_DB_CONFIG = { - "source": { - "type": "superset", - "serviceName": "test_supserset", - "serviceConnection": { - "config": { - "hostPort": "https://my-superset.com", - "type": "Superset", - "connection": { - "type": "Postgres", - "username": "superset", - "authType": { - "password": "superset", - }, - "hostPort": "localhost:5432", - "database": "superset", - }, - } - }, - "sourceConfig": { - "config": { - "type": "DashboardMetadata", - } - }, - }, - "sink": {"type": "metadata-rest", "config": {}}, - "workflowConfig": { - "openMetadataServerConfig": { - "hostPort": "http://localhost:8585/api", - "authProvider": "openmetadata", - "securityConfig": {"jwtToken": "token"}, - }, - }, -} - EXPECTED_DASH_SERVICE = DashboardService( id="c3eb265f-5445-4ad3-ba5e-797d3a3071bb", fullyQualifiedName=FullyQualifiedEntityName(__root__="test_supserset"), @@ -152,7 +89,7 @@ EXPECTED_DASH_SERVICE = DashboardService( connection=DashboardConnection(), serviceType=DashboardServiceType.Superset, ) -EXPECTED_USER = EntityReference(id=uuid.uuid4(), type="user") +EXPECTED_USER = EntityReference(id="81af89aa-1bab-41aa-a567-5e68f78acdc0", type="user") MOCK_DB_MYSQL_SERVICE_1 = DatabaseService( id="c3eb265f-5445-4ad3-ba5e-797d3a307122", @@ -182,6 +119,16 @@ MOCK_DB_MYSQL_SERVICE_2 = DatabaseService( ), serviceType=DatabaseServiceType.Mysql, ) +MOCK_DASHBOARD_INPUT = { + "certification_details": "sample certificate details", + "certified_by": "certified by unknown", + "css": "css", + "dashboard_title": "Top trades", + "external_url": "external url", + "slug": "top-trades", + "published": True, + "position_json": '{"CHART-dwSXo_0t5X":{"children":[],"id":"CHART-dwSXo_0t5X","meta":{"chartId":37,"height":50,"sliceName":"% Rural","uuid":"8f663401-854a-4da7-8e50-4b8e4ebb4f22","width":4},"parents":["ROOT_ID","GRID_ID","ROW-z_7odBWenK"],"type":"CHART"},"DASHBOARD_VERSION_KEY":"v2","GRID_ID":{"children":["ROW-z_7odBWenK"],"id":"GRID_ID","parents":["ROOT_ID"],"type":"GRID"},"HEADER_ID":{"id":"HEADER_ID","meta":{"text":"My DASH"},"type":"HEADER"},"ROOT_ID":{"children":["GRID_ID"],"id":"ROOT_ID","type":"ROOT"},"ROW-z_7odBWenK":{"children":["CHART-dwSXo_0t5X"],"id":"ROW-z_7odBWenK","meta":{"background":"BACKGROUND_TRANSPARENT"},"parents":["ROOT_ID","GRID_ID"],"type":"ROW"}}', +} MOCK_DB_POSTGRES_SERVICE = DatabaseService( id="c3eb265f-5445-4ad3-ba5e-797d3a307122", @@ -218,73 +165,265 @@ EXPECTED_DASH = CreateDashboardRequest( owner=EXPECTED_USER, ) -EXPECTED_CHART = CreateChartRequest( - name=37, - displayName="% Rural", - description="TEST DESCRIPTION", - chartType=ChartType.Other.value, - sourceUrl="https://my-superset.com/explore/?slice_id=37", - service=EXPECTED_DASH_SERVICE.fullyQualifiedName, + +EXPECTED_API_DASHBOARD = CreateDashboardRequest( + name=EntityName(__root__="10"), + displayName="Unicode Test", + description=None, + dashboardType=DashboardType.Dashboard.value, + sourceUrl=SourceUrl( + __root__="http://localhost:54510/superset/dashboard/unicode-test/" + ), + project=None, + charts=[], + dataModels=None, + tags=None, + owner=None, + service=FullyQualifiedEntityName(__root__="test_supserset"), + extension=None, + domain=None, + dataProducts=None, + lifeCycle=None, + sourceHash=None, ) -EXPECTED_ALL_CHARTS = {37: MOCK_CHART} -EXPECTED_ALL_CHARTS_DB = {37: MOCK_CHART_DB} +EXPECTED_CHART = CreateChartRequest( + name=1, + displayName="Rural", + description="desc", + chartType=ChartType.Other.value, + sourceUrl="https://my-superset.com/explore/?slice_id=1", + service=EXPECTED_DASH_SERVICE.fullyQualifiedName, +) +EXPECTED_CHART_2 = CreateChartRequest( + name=EntityName(__root__="69"), + displayName="Unicode Cloud", + description=None, + chartType=ChartType.Other.value, + sourceUrl=SourceUrl(__root__="http://localhost:54510/explore/?slice_id=69"), + tags=None, + owner=None, + service=FullyQualifiedEntityName(__root__="test_supserset"), + domain=None, + dataProducts=None, + lifeCycle=None, + sourceHash=None, +) + +# EXPECTED_ALL_CHARTS = {37: MOCK_CHART} +# EXPECTED_ALL_CHARTS_DB = {37: MOCK_CHART_DB} +EXPECTED_ALL_CHARTS_DB = {1: MOCK_CHART_DB_2} NOT_FOUND_RESP = {"message": "Not found"} - +EXPECTED_API_DATASET_FQN = None EXPECTED_DATASET_FQN = "test_postgres.examples.main.wb_health_population" +def setup_sample_data(postgres_container): + engine = sqlalchemy.create_engine(postgres_container.get_connection_url()) + with engine.begin() as connection: + CREATE_TABLE_AB_USER = """ + CREATE TABLE ab_user ( + id INT PRIMARY KEY, + username VARCHAR(50)); + """ + CREATE_TABLE_DASHBOARDS = """ + CREATE TABLE dashboards ( + id INT PRIMARY KEY, + created_by_fk INT, + FOREIGN KEY (created_by_fk) REFERENCES ab_user(id)); + """ + INSERT_AB_USER_DATA = """ + INSERT INTO ab_user (id, username) + VALUES (1, 'test_user'); + """ + INSERT_DASHBOARDS_DATA = """ + INSERT INTO dashboards (id, created_by_fk) + VALUES (1, 1); + """ + CREATE_SLICES_TABLE = """ + CREATE TABLE slices ( + id INTEGER PRIMARY KEY, + slice_name VARCHAR(255), + description TEXT, + datasource_id INTEGER, + viz_type VARCHAR(255), + datasource_type VARCHAR(255) + ) + """ + INSERT_SLICES_DATA = """ + INSERT INTO slices(id, slice_name, description, datasource_id, viz_type, datasource_type) + VALUES (1, 'Rural', 'desc', 99, 'bar_chart', 'table'); + """ + CREATE_DBS_TABLE = """ + CREATE TABLE dbs ( + id INTEGER PRIMARY KEY, + database_name VARCHAR(255), + sqlalchemy_uri TEXT + ) + """ + INSERT_DBS_DATA = """ + INSERT INTO dbs(id, database_name, sqlalchemy_uri) + VALUES (5, 'test_db', 'postgres://user:pass@localhost:5432/examples'); + """ + CREATE_TABLES_TABLE = """ + CREATE TABLE tables ( + id INTEGER PRIMARY KEY, + table_name VARCHAR(255), + schema VARCHAR(255), + database_id INTEGER + ); + """ + INSERT_TABLES_DATA = """ + INSERT INTO tables(id, table_name, schema, database_id) + VALUES (99, 'sample_table', 'main', 5); + """ + connection.execute(sqlalchemy.text(CREATE_TABLE_AB_USER)) + connection.execute(sqlalchemy.text(INSERT_AB_USER_DATA)) + connection.execute(sqlalchemy.text(CREATE_TABLE_DASHBOARDS)) + connection.execute(sqlalchemy.text(INSERT_DASHBOARDS_DATA)) + connection.execute(sqlalchemy.text(CREATE_SLICES_TABLE)) + connection.execute(sqlalchemy.text(INSERT_SLICES_DATA)) + connection.execute(sqlalchemy.text(CREATE_DBS_TABLE)) + connection.execute(sqlalchemy.text(INSERT_DBS_DATA)) + connection.execute(sqlalchemy.text(CREATE_TABLES_TABLE)) + connection.execute(sqlalchemy.text(INSERT_TABLES_DATA)) + + +INITIAL_SETUP = True +superset_container = postgres_container = None + + +def set_testcontainers(): + global INITIAL_SETUP, superset_container, postgres_container + if INITIAL_SETUP: + # postgres test container + postgres_container = PostgresContainer("postgres:16-alpine") + postgres_container.start() + setup_sample_data(postgres_container) + # superset testcontainer + superset_container = DockerContainer(image="apache/superset") + superset_container.with_env("SUPERSET_SECRET_KEY", "&3brfbcf192T!)$sabqbie") + superset_container.with_env("WTF_CSRF_ENABLED", False) + + superset_container.with_exposed_ports(8088) + superset_container.start() + + superset_container.exec( + "superset fab create-admin --username admin --firstname Superset --lastname Admin --email admin@superset.com --password admin" + ) + superset_container.exec("superset db upgrade") + superset_container.exec("superset init") + superset_container.exec("superset load-examples") + INITIAL_SETUP = False + return superset_container, postgres_container + + class SupersetUnitTest(TestCase): """ Validate how we work with Superset metadata """ + @classmethod + def teardown_class(cls): + """Teardown class""" + # stop containers + superset_container.stop() + postgres_container.stop() + def __init__(self, methodName) -> None: super().__init__(methodName) + + superset_container, postgres_container = set_testcontainers() + + MOCK_SUPERSET_API_CONFIG = { + "source": { + "type": "superset", + "serviceName": "test_supserset", + "serviceConnection": { + "config": { + "hostPort": f"http://{superset_container.get_container_host_ip()}:{superset_container.get_exposed_port(8088)}", + "type": "Superset", + "connection": { + "username": "admin", + "password": "admin", + "provider": "db", + }, + } + }, + "sourceConfig": { + "config": { + "type": "DashboardMetadata", + "includeDraftDashboard": False, + } + }, + }, + "sink": {"type": "metadata-rest", "config": {}}, + "workflowConfig": { + "openMetadataServerConfig": { + "hostPort": "http://localhost:8585/api", + "authProvider": "openmetadata", + "securityConfig": { + "jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg" + }, + }, + }, + } + MOCK_SUPERSET_DB_CONFIG = { + "source": { + "type": "superset", + "serviceName": "test_supserset", + "serviceConnection": { + "config": { + "hostPort": f"http://{superset_container.get_container_host_ip()}:{superset_container.get_exposed_port(8088)}", + "type": "Superset", + "connection": { + "type": "Postgres", + "hostPort": f"{postgres_container.get_container_host_ip()}:{postgres_container.get_exposed_port(5432)}", + "username": postgres_container.env.get("POSTGRES_USER"), + "authType": { + "password": postgres_container.env.get( + "POSTGRES_PASSWORD" + ) + }, + "database": postgres_container.env.get("POSTGRES_DB"), + }, + } + }, + "sourceConfig": { + "config": { + "type": "DashboardMetadata", + } + }, + }, + "sink": {"type": "metadata-rest", "config": {}}, + "workflowConfig": { + "openMetadataServerConfig": { + "hostPort": "http://localhost:8585/api", + "authProvider": "openmetadata", + "securityConfig": {"jwtToken": "token"}, + }, + }, + } self.config = OpenMetadataWorkflowConfig.parse_obj(MOCK_SUPERSET_API_CONFIG) - with patch.object( - DashboardServiceSource, "test_connection", return_value=False - ), patch.object(OMetaServerMixin, "validate_versions", return_value=True): - # This already validates that the source can be initialized - self.superset_api: SupersetSource = SupersetSource.create( - MOCK_SUPERSET_API_CONFIG["source"], - OpenMetadata(self.config.workflowConfig.openMetadataServerConfig), - ) + self.superset_api: SupersetSource = SupersetSource.create( + MOCK_SUPERSET_API_CONFIG["source"], + OpenMetadata(self.config.workflowConfig.openMetadataServerConfig), + ) + self.assertEqual(type(self.superset_api), SupersetAPISource) + self.superset_api.context.get().__dict__[ + "dashboard_service" + ] = EXPECTED_DASH_SERVICE.fullyQualifiedName.__root__ - self.assertEqual(type(self.superset_api), SupersetAPISource) - - self.superset_api.context.get().__dict__[ - "dashboard_service" - ] = EXPECTED_DASH_SERVICE.fullyQualifiedName.__root__ - - with patch.object( - SupersetAPIClient, "fetch_total_charts", return_value=1 - ), patch.object( - SupersetAPIClient, "fetch_charts", return_value=MOCK_CHART_RESP - ): - self.superset_api.prepare() - self.assertEqual(EXPECTED_ALL_CHARTS, self.superset_api.all_charts) - - with patch.object( - DashboardServiceSource, "test_connection", return_value=False - ), patch.object(OMetaServerMixin, "validate_versions", return_value=True): - # This already validates that the source can be initialized - self.superset_db: SupersetSource = SupersetSource.create( - MOCK_SUPERSET_DB_CONFIG["source"], - OpenMetadata(self.config.workflowConfig.openMetadataServerConfig), - ) - - self.assertEqual(type(self.superset_db), SupersetDBSource) - - self.superset_db.context.get().__dict__[ - "dashboard_service" - ] = EXPECTED_DASH_SERVICE.fullyQualifiedName.__root__ - - with patch.object(Engine, "execute", return_value=mock_data["chart-db"]): - self.superset_db.prepare() - self.assertEqual(EXPECTED_ALL_CHARTS_DB, self.superset_db.all_charts) + self.superset_db: SupersetSource = SupersetSource.create( + MOCK_SUPERSET_DB_CONFIG["source"], + OpenMetadata(self.config.workflowConfig.openMetadataServerConfig), + ) + self.assertEqual(type(self.superset_db), SupersetDBSource) + self.superset_db.context.get().__dict__[ + "dashboard_service" + ] = EXPECTED_DASH_SERVICE.fullyQualifiedName.__root__ def test_create(self): """ @@ -318,36 +457,12 @@ class SupersetUnitTest(TestCase): self.config.workflowConfig.openMetadataServerConfig, ) - def test_api_perpare(self): - pass - def test_api_get_dashboards_list(self): """ Mock the client and check that we get a list """ - - with patch.object( - SupersetAPIClient, "fetch_total_dashboards", return_value=1 - ), patch.object( - SupersetAPIClient, "fetch_dashboards", return_value=MOCK_DASHBOARD_RESP - ): - dashboard_list = self.superset_api.get_dashboards_list() - self.assertEqual(list(dashboard_list), [MOCK_DASHBOARD]) - - def test_api_get_published_dashboards_list(self): - """ - Mock the client and check that we get only published dashboards list - """ - with patch.object( - SupersetAPIClient, "fetch_total_dashboards", return_value=1 - ), patch.object( - SupersetAPIClient, "fetch_dashboards", return_value=MOCK_DASHBOARD_RESP - ): - dashboard_list = list(self.superset_api.get_dashboards_list()) - self.assertEqual(len(dashboard_list), PUBLISHED_DASHBOARD_COUNT) - self.assertEqual( - dashboard_list[0].dashboard_title, PUBLISHED_DASHBOARD_NAME - ) + dashboard_list = list(self.superset_api.get_dashboards_list()) + self.assertEqual(len(dashboard_list), PUBLISHED_DASHBOARD_COUNT) def test_charts_of_dashboard(self): """ @@ -356,7 +471,14 @@ class SupersetUnitTest(TestCase): result = self.superset_api._get_charts_of_dashboard( # pylint: disable=protected-access MOCK_DASHBOARD ) - self.assertEqual(result, [37]) + self.assertEqual(result, [69]) + + def test_fetch_chart_db(self): + """ + test fetch chart method of db source + """ + self.superset_db.prepare() + self.assertEqual(EXPECTED_ALL_CHARTS_DB, self.superset_db.all_charts) def test_dashboard_name(self): dashboard_name = self.superset_api.get_dashboard_name(MOCK_DASHBOARD) @@ -364,71 +486,53 @@ class SupersetUnitTest(TestCase): def test_yield_dashboard(self): # TEST API SOURCE - with patch.object( - SupersetAPISource, "_get_user_by_email", return_value=EXPECTED_USER - ): - self.superset_api.context.get().__dict__["charts"] = [ - chart.name.__root__ for chart in EXPECTED_CHART_ENTITY - ] - dashboard = next(self.superset_api.yield_dashboard(MOCK_DASHBOARD)).right - self.assertEqual(dashboard, EXPECTED_DASH) + dashboard = next(self.superset_api.yield_dashboard(MOCK_DASHBOARD)).right + EXPECTED_API_DASHBOARD.sourceUrl = SourceUrl( + __root__=f"http://{superset_container.get_container_host_ip()}:{superset_container.get_exposed_port(8088)}{MOCK_DASHBOARD.url}" + ) + self.assertEqual(dashboard, EXPECTED_API_DASHBOARD) # TEST DB SOURCE - with patch.object( - SupersetDBSource, "_get_user_by_email", return_value=EXPECTED_USER - ): - self.superset_db.context.get().__dict__["charts"] = [ - chart.name.__root__ for chart in EXPECTED_CHART_ENTITY - ] - dashboard = next(self.superset_db.yield_dashboard(MOCK_DASHBOARD_DB)).right - self.assertEqual(dashboard, EXPECTED_DASH) + self.superset_db.context.get().__dict__["charts"] = [ + chart.name.__root__ for chart in EXPECTED_CHART_ENTITY + ] + dashboard = next(self.superset_db.yield_dashboard(MOCK_DASHBOARD_DB)).right + EXPECTED_DASH.sourceUrl = SourceUrl( + __root__=f"http://{superset_container.get_container_host_ip()}:{superset_container.get_exposed_port(8088)}/superset/dashboard/14/" + ) + EXPECTED_DASH.owner = dashboard.owner + self.assertEqual(dashboard, EXPECTED_DASH) def test_yield_dashboard_chart(self): # TEST API SOURCE - dashboard_charts = next( + self.superset_api.prepare() + dashboard_chart = next( self.superset_api.yield_dashboard_chart(MOCK_DASHBOARD) ).right - self.assertEqual(dashboard_charts, EXPECTED_CHART) + EXPECTED_CHART_2.sourceUrl = SourceUrl( + __root__=f"http://{superset_container.get_container_host_ip()}:{superset_container.get_exposed_port(8088)}/explore/?slice_id=69" + ) + EXPECTED_CHART_2.displayName = dashboard_chart.displayName + self.assertEqual(dashboard_chart, EXPECTED_CHART_2) # TEST DB SOURCE + self.superset_db.prepare() dashboard_charts = next( self.superset_db.yield_dashboard_chart(MOCK_DASHBOARD_DB) ).right + EXPECTED_CHART.sourceUrl = SourceUrl( + __root__=f"http://{superset_container.get_container_host_ip()}:{superset_container.get_exposed_port(8088)}/explore/?slice_id=1" + ) self.assertEqual(dashboard_charts, EXPECTED_CHART) def test_api_get_datasource_fqn(self): """ Test generated datasource fqn for api source """ - with patch.object( - OpenMetadata, "es_search_from_fqn", return_value=None - ), patch.object( - SupersetAPIClient, - "fetch_datasource", - return_value=SupersetDatasource(**mock_data["datasource"]), - ), patch.object( - SupersetAPIClient, - "fetch_database", - return_value=ListDatabaseResult(**mock_data["database"]), - ): - fqn = self.superset_api._get_datasource_fqn( # pylint: disable=protected-access - 1, MOCK_DB_POSTGRES_SERVICE - ) - self.assertEqual(fqn, EXPECTED_DATASET_FQN) - - with patch.object( - OpenMetadata, "es_search_from_fqn", return_value=None - ), patch.object( - SupersetAPIClient, - "fetch_datasource", - return_value=SupersetDatasource(**mock_data["datasource"]), - ), patch.object( - SupersetAPIClient, "fetch_database", return_value=ListDatabaseResult() - ): - fqn = self.superset_api._get_datasource_fqn( # pylint: disable=protected-access - 1, MOCK_DB_POSTGRES_SERVICE - ) - self.assertEqual(fqn, None) + fqn = self.superset_api._get_datasource_fqn( # pylint: disable=protected-access + 1, MOCK_DB_POSTGRES_SERVICE + ) + self.assertEqual(fqn, EXPECTED_API_DATASET_FQN) def test_db_get_datasource_fqn_for_lineage(self): fqn = self.superset_db._get_datasource_fqn_for_lineage( # pylint: disable=protected-access