mirror of
https://github.com/open-metadata/OpenMetadata.git
synced 2025-12-28 16:08:23 +00:00
* Added option to exclude drafts: superset ingestion * Updated supserset yaml docs * Added tests for exlcude draft dashboards * Added tests for exlcude draft dashboards * Formatted queries.py
This commit is contained in:
Suman Maharana
GitHub
parent
daae3c48e3
commit
16eaf925e9
@ -77,6 +77,11 @@ class SupersetAPISource(SupersetSourceMixin):
|
||||
dashboards = self.client.fetch_dashboards(current_page, page_size)
|
||||
current_page += 1
|
||||
for dashboard in dashboards.result:
|
||||
if (
|
||||
not self.source_config.includeDraftDashboard
|
||||
and not dashboard.published
|
||||
):
|
||||
continue
|
||||
yield dashboard
|
||||
|
||||
def yield_dashboard(
|
||||
|
||||
@ -45,6 +45,7 @@ from metadata.ingestion.source.dashboard.superset.queries import (
|
||||
FETCH_ALL_CHARTS,
|
||||
FETCH_COLUMN,
|
||||
FETCH_DASHBOARDS,
|
||||
FETCH_PUBLISHED_DASHBOARDS,
|
||||
)
|
||||
from metadata.utils import fqn
|
||||
from metadata.utils.filters import filter_by_datamodel
|
||||
@ -100,7 +101,12 @@ class SupersetDBSource(SupersetSourceMixin):
|
||||
"""
|
||||
Get List of all dashboards
|
||||
"""
|
||||
dashboards = self.engine.execute(FETCH_DASHBOARDS)
|
||||
query = (
|
||||
FETCH_DASHBOARDS
|
||||
if self.source_config.includeDraftDashboard
|
||||
else FETCH_PUBLISHED_DASHBOARDS
|
||||
)
|
||||
dashboards = self.engine.execute(query)
|
||||
for dashboard in dashboards:
|
||||
yield FetchDashboard(**dashboard)
|
||||
|
||||
|
||||
@ -42,6 +42,7 @@ class DashboardResult(BaseModel):
|
||||
position_json: Optional[str]
|
||||
id: Optional[int]
|
||||
email: Optional[str]
|
||||
published: Optional[bool]
|
||||
|
||||
|
||||
class SupersetDashboardCount(BaseModel):
|
||||
@ -135,6 +136,7 @@ class FetchDashboard(BaseModel):
|
||||
id: Optional[int]
|
||||
dashboard_title: Optional[str]
|
||||
position_json: Optional[str]
|
||||
published: Optional[bool]
|
||||
email: Optional[str]
|
||||
|
||||
|
||||
|
||||
@ -18,8 +18,8 @@ select
|
||||
s.id,
|
||||
s.slice_name,
|
||||
s.description,
|
||||
s.datasource_id,
|
||||
s.viz_type,
|
||||
s.datasource_id,
|
||||
s.viz_type,
|
||||
t.table_name,
|
||||
t.schema,
|
||||
db.database_name,
|
||||
@ -37,6 +37,7 @@ select
|
||||
d.id,
|
||||
d.dashboard_title,
|
||||
d.position_json,
|
||||
d.published,
|
||||
au.email
|
||||
from
|
||||
dashboards d
|
||||
@ -46,6 +47,22 @@ ON
|
||||
d.created_by_fk = au.id
|
||||
"""
|
||||
|
||||
FETCH_PUBLISHED_DASHBOARDS = """
|
||||
select
|
||||
d.id,
|
||||
d.dashboard_title,
|
||||
d.position_json,
|
||||
d.published,
|
||||
au.email
|
||||
from
|
||||
dashboards d
|
||||
LEFT JOIN
|
||||
ab_user au
|
||||
ON
|
||||
d.created_by_fk = au.id
|
||||
where
|
||||
d.published=true
|
||||
"""
|
||||
|
||||
FETCH_ALL_CHARTS_TEST = """
|
||||
select
|
||||
@ -74,10 +91,10 @@ LIMIT 1
|
||||
FETCH_COLUMN = """
|
||||
select
|
||||
tc.id,
|
||||
table_name ,
|
||||
column_name,
|
||||
type,
|
||||
tc.description
|
||||
table_name ,
|
||||
column_name,
|
||||
type,
|
||||
tc.description
|
||||
from
|
||||
table_columns tc
|
||||
inner join
|
||||
|
||||
@ -123,6 +123,47 @@
|
||||
"status": "published",
|
||||
"thumbnail_url": "/api/v1/dashboard/14/thumbnail/0088b55d1a7c34b5aa121f11252c11d5/",
|
||||
"url": "/superset/dashboard/14/"
|
||||
},
|
||||
{
|
||||
"certification_details": "",
|
||||
"certified_by": "",
|
||||
"changed_by": {
|
||||
"first_name": "Superset",
|
||||
"id": 1,
|
||||
"last_name": "Admin",
|
||||
"username": "admin"
|
||||
},
|
||||
"changed_by_name": "Superset Admin",
|
||||
"changed_by_url": "/superset/profile/admin",
|
||||
"changed_on_delta_humanized": "2 days ago",
|
||||
"changed_on_utc": "2024-02-30T10:53:25.572977+0000",
|
||||
"created_by": {
|
||||
"first_name": "Superset",
|
||||
"id": 1,
|
||||
"last_name": "Admin"
|
||||
},
|
||||
"created_on_delta_humanized": "2 days ago",
|
||||
"css": "",
|
||||
"dashboard_title": "My DRAFT DASH",
|
||||
"id": 15,
|
||||
"is_managed_externally": false,
|
||||
"json_metadata": "{\"show_native_filters\": true, \"color_scheme\": \"\", \"refresh_frequency\": 0, \"shared_label_colors\": {}, \"color_scheme_domain\": [], \"expanded_slices\": {}, \"label_colors\": {}, \"timed_refresh_immune_slices\": [], \"default_filters\": \"{}\", \"chart_configuration\": {}}",
|
||||
"owners": [
|
||||
{
|
||||
"email": "admin@openmetadata.org",
|
||||
"first_name": "Superset",
|
||||
"id": 1,
|
||||
"last_name": "Admin",
|
||||
"username": "admin"
|
||||
}
|
||||
],
|
||||
"position_json": "{\"CHART-dwSXo_0t5X\":{\"children\":[],\"id\":\"CHART-dwSXo_0t5X\",\"meta\":{\"chartId\":37,\"height\":50,\"sliceName\":\"% Rural\",\"uuid\":\"8f663401-854a-4da7-8e50-4b8e4ebb4f22\",\"width\":4},\"parents\":[\"ROOT_ID\",\"GRID_ID\",\"ROW-z_7odBWenK\"],\"type\":\"CHART\"},\"DASHBOARD_VERSION_KEY\":\"v2\",\"GRID_ID\":{\"children\":[\"ROW-z_7odBWenK\"],\"id\":\"GRID_ID\",\"parents\":[\"ROOT_ID\"],\"type\":\"GRID\"},\"HEADER_ID\":{\"id\":\"HEADER_ID\",\"meta\":{\"text\":\"My DASH\"},\"type\":\"HEADER\"},\"ROOT_ID\":{\"children\":[\"GRID_ID\"],\"id\":\"ROOT_ID\",\"type\":\"ROOT\"},\"ROW-z_7odBWenK\":{\"children\":[\"CHART-dwSXo_0t5X\"],\"id\":\"ROW-z_7odBWenK\",\"meta\":{\"background\":\"BACKGROUND_TRANSPARENT\"},\"parents\":[\"ROOT_ID\",\"GRID_ID\"],\"type\":\"ROW\"}}",
|
||||
"published": false,
|
||||
"roles": [],
|
||||
"slug": null,
|
||||
"status": "draft",
|
||||
"thumbnail_url": "/api/v1/dashboard/15/thumbnail/0088b55d1a7c34b5aa121f11252c11d5/",
|
||||
"url": "/superset/dashboard/15/"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
@ -72,6 +72,8 @@ with open(mock_file_path, encoding="UTF-8") as file:
|
||||
|
||||
MOCK_DASHBOARD_RESP = SupersetDashboardCount(**mock_data["dashboard"])
|
||||
MOCK_DASHBOARD = MOCK_DASHBOARD_RESP.result[0]
|
||||
PUBLISHED_DASHBOARD_COUNT = 1
|
||||
PUBLISHED_DASHBOARD_NAME = "My DASH"
|
||||
MOCK_CHART_RESP = SupersetChart(**mock_data["chart"])
|
||||
MOCK_CHART = MOCK_CHART_RESP.result[0]
|
||||
|
||||
@ -94,9 +96,7 @@ MOCK_SUPERSET_API_CONFIG = {
|
||||
}
|
||||
},
|
||||
"sourceConfig": {
|
||||
"config": {
|
||||
"type": "DashboardMetadata",
|
||||
}
|
||||
"config": {"type": "DashboardMetadata", "includeDraftDashboard": False}
|
||||
},
|
||||
},
|
||||
"sink": {"type": "metadata-rest", "config": {}},
|
||||
@ -334,6 +334,21 @@ class SupersetUnitTest(TestCase):
|
||||
dashboard_list = self.superset_api.get_dashboards_list()
|
||||
self.assertEqual(list(dashboard_list), [MOCK_DASHBOARD])
|
||||
|
||||
def test_api_get_published_dashboards_list(self):
|
||||
"""
|
||||
Mock the client and check that we get only published dashboards list
|
||||
"""
|
||||
with patch.object(
|
||||
SupersetAPIClient, "fetch_total_dashboards", return_value=1
|
||||
), patch.object(
|
||||
SupersetAPIClient, "fetch_dashboards", return_value=MOCK_DASHBOARD_RESP
|
||||
):
|
||||
dashboard_list = list(self.superset_api.get_dashboards_list())
|
||||
self.assertEqual(len(dashboard_list), PUBLISHED_DASHBOARD_COUNT)
|
||||
self.assertEqual(
|
||||
dashboard_list[0].dashboard_title, PUBLISHED_DASHBOARD_NAME
|
||||
)
|
||||
|
||||
def test_charts_of_dashboard(self):
|
||||
"""
|
||||
Mock the client and check that we get a list
|
||||
|
||||
@ -157,7 +157,8 @@ source:
|
||||
# For MySQL Connection
|
||||
# type: Mysql
|
||||
# username: <username>
|
||||
# password: <password>
|
||||
# authType:
|
||||
# password: <password>
|
||||
# hostPort: <hostPort>
|
||||
# databaseSchema: superset
|
||||
|
||||
@ -166,7 +167,8 @@ source:
|
||||
# For Postgres Connection
|
||||
# type: Postgres
|
||||
# username: username
|
||||
# password: password
|
||||
# authType:
|
||||
# password: <password>
|
||||
# hostPort: localhost:5432
|
||||
# database: superset
|
||||
```
|
||||
|
||||
@ -157,7 +157,8 @@ source:
|
||||
# For MySQL Connection
|
||||
# type: Mysql
|
||||
# username: <username>
|
||||
# password: <password>
|
||||
# authType:
|
||||
# password: <password>
|
||||
# hostPort: <hostPort>
|
||||
# databaseSchema: superset
|
||||
|
||||
@ -166,7 +167,8 @@ source:
|
||||
# For Postgres Connection
|
||||
# type: Postgres
|
||||
# username: username
|
||||
# password: password
|
||||
# authType:
|
||||
# password: <password>
|
||||
# hostPort: localhost:5432
|
||||
# database: superset
|
||||
```
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user