mirror of
https://github.com/open-metadata/OpenMetadata.git
synced 2025-08-21 07:28:44 +00:00
Fix #339: Add sample dashboards & charts
This commit is contained in:
Suresh Srinivas
parent
76d5e7d0ca
commit
fd99dbdb1a
95
ingestion/examples/superset_data/charts.json
Normal file
95
ingestion/examples/superset_data/charts.json
Normal file
@ -0,0 +1,95 @@
|
||||
{
|
||||
"charts": [
|
||||
{
|
||||
"id": "2841fdb1-e378-4a2c-94f8-27c9f5d6ef8e",
|
||||
"name": "# of Games That Hit 100k in Sales By Release Year",
|
||||
"fullyQualifiedName": "local_superset.# of Games That Hit 100k in Sales By Release Year",
|
||||
"description": "",
|
||||
"chartId": "114",
|
||||
"chartType": "Area",
|
||||
"chartUrl": "http://localhost:8088/superset/explore/?form_data=%7B%22slice_id%22%3A%20114%7D",
|
||||
"href": "http://localhost:8585/api/v1/charts/2841fdb1-e378-4a2c-94f8-27c9f5d6ef8e"
|
||||
}, {
|
||||
"id": "3bcba490-9e5c-4946-a0e3-41e8ff8f4aa4",
|
||||
"name": "% Rural",
|
||||
"fullyQualifiedName": "local_superset.% Rural",
|
||||
"description": "",
|
||||
"chartId": "166",
|
||||
"chartType": "Other",
|
||||
"chartUrl": "http://localhost:8088/superset/explore/?form_data=%7B%22slice_id%22%3A%20166%7D",
|
||||
"href": "http://localhost:8585/api/v1/charts/3bcba490-9e5c-4946-a0e3-41e8ff8f4aa4"
|
||||
}, {
|
||||
"id": "22b95748-4a7b-48ad-859e-cf7c66a7f343",
|
||||
"name": "✈️ Relocation ability",
|
||||
"fullyQualifiedName": "local_superset.✈️ Relocation ability",
|
||||
"description": "",
|
||||
"chartId": "92",
|
||||
"chartType": "Other",
|
||||
"chartUrl": "http://localhost:8088/superset/explore/?form_data=%7B%22slice_id%22%3A%2092%7D",
|
||||
"href": "http://localhost:8585/api/v1/charts/22b95748-4a7b-48ad-859e-cf7c66a7f343"
|
||||
}, {
|
||||
"id": "62b31dcc-4619-46a0-99b1-0fa7cd6f93da",
|
||||
"name": "Age distribution of respondents",
|
||||
"fullyQualifiedName": "local_superset.Age distribution of respondents",
|
||||
"description": "",
|
||||
"chartId": "117",
|
||||
"chartType": "Histogram",
|
||||
"chartUrl": "http://localhost:8088/superset/explore/?form_data=%7B%22slice_id%22%3A%20117%7D",
|
||||
"href": "http://localhost:8585/api/v1/charts/62b31dcc-4619-46a0-99b1-0fa7cd6f93da"
|
||||
}, {
|
||||
"id": "57944482-e187-439a-aaae-0e8aabd2f455",
|
||||
"name": "Arcs",
|
||||
"fullyQualifiedName": "local_superset.Arcs",
|
||||
"description": "",
|
||||
"chartId": "197",
|
||||
"chartType": "Other",
|
||||
"chartUrl": "http://localhost:8088/superset/explore/?form_data=%7B%22slice_id%22%3A%20197%7D",
|
||||
"href": "http://localhost:8585/api/v1/charts/57944482-e187-439a-aaae-0e8aabd2f455"
|
||||
}, {
|
||||
"id": "d88e2056-c74a-410d-829e-eb31b040c132",
|
||||
"name": "Are you an ethnic minority in your city?",
|
||||
"fullyQualifiedName": "local_superset.Are you an ethnic minority in your city?",
|
||||
"description": "",
|
||||
"chartId": "127",
|
||||
"chartType": "Other",
|
||||
"chartUrl": "http://localhost:8088/superset/explore/?form_data=%7B%22slice_id%22%3A%20127%7D",
|
||||
"href": "http://localhost:8585/api/v1/charts/d88e2056-c74a-410d-829e-eb31b040c132"
|
||||
}, {
|
||||
"id": "c1d3e156-4628-414e-8d6e-a6bdd486128f",
|
||||
"name": "Average and Sum Trends",
|
||||
"fullyQualifiedName": "local_superset.Average and Sum Trends",
|
||||
"description": "",
|
||||
"chartId": "183",
|
||||
"chartType": "Line",
|
||||
"chartUrl": "http://localhost:8088/superset/explore/?form_data=%7B%22slice_id%22%3A%20183%7D",
|
||||
"href": "http://localhost:8585/api/v1/charts/c1d3e156-4628-414e-8d6e-a6bdd486128f"
|
||||
}, {
|
||||
"id": "bfc57519-8cef-47e6-a423-375d5b89a6a4",
|
||||
"name": "Birth in France by department in 2016",
|
||||
"fullyQualifiedName": "local_superset.Birth in France by department in 2016",
|
||||
"description": "",
|
||||
"chartId": "161",
|
||||
"chartType": "Other",
|
||||
"chartUrl": "http://localhost:8088/superset/explore/?form_data=%7B%22slice_id%22%3A%20161%7D",
|
||||
"href": "http://localhost:8585/api/v1/charts/bfc57519-8cef-47e6-a423-375d5b89a6a4"
|
||||
}, {
|
||||
"id": "bf2eeac4-7226-46c6-bbef-918569c137a0",
|
||||
"name": "Box plot",
|
||||
"fullyQualifiedName": "local_superset.Box plot",
|
||||
"description": "",
|
||||
"chartId": "170",
|
||||
"chartType": "Bar",
|
||||
"chartUrl": "http://localhost:8088/superset/explore/?form_data=%7B%22slice_id%22%3A%20170%7D",
|
||||
"href": "http://localhost:8585/api/v1/charts/bf2eeac4-7226-46c6-bbef-918569c137a0"
|
||||
}, {
|
||||
"id": "167fd63b-42f1-4d7e-a37d-893fd8173b44",
|
||||
"name": "Boy Name Cloud",
|
||||
"fullyQualifiedName": "local_superset.Boy Name Cloud",
|
||||
"description": "",
|
||||
"chartId": "180",
|
||||
"chartType": "Other",
|
||||
"chartUrl": "http://localhost:8088/superset/explore/?form_data=%7B%22slice_id%22%3A%20180%7D",
|
||||
"href": "http://localhost:8585/api/v1/charts/167fd63b-42f1-4d7e-a37d-893fd8173b44"
|
||||
}
|
||||
]
|
||||
}
|
||||
94
ingestion/examples/superset_data/dashboards.json
Normal file
94
ingestion/examples/superset_data/dashboards.json
Normal file
@ -0,0 +1,94 @@
|
||||
{
|
||||
"dashboards": [
|
||||
{
|
||||
"id": "d4dc7baf-1b17-45f8-acd5-a15b78cc7c5f",
|
||||
"name": "[ untitled dashboard ]",
|
||||
"fullyQualifiedName": "local_superset.[ untitled dashboard ]",
|
||||
"description": "",
|
||||
"dashboardUrl": "http://localhost:808/superset/dashboard/1/",
|
||||
"charts": [183, 170, 197],
|
||||
"href": "http://localhost:8585/api/v1/dashboards/d4dc7baf-1b17-45f8-acd5-a15b78cc7c5f"
|
||||
},
|
||||
{
|
||||
"id": "063cd787-8630-4809-9702-34d3992c7248",
|
||||
"name": "COVID Vaccine Dashboard",
|
||||
"fullyQualifiedName": "local_superset.COVID Vaccine Dashboard",
|
||||
"description": "",
|
||||
"dashboardUrl": "http://localhost:808/superset/dashboard/8/",
|
||||
"charts": [117, 197],
|
||||
"href": "http://localhost:8585/api/v1/dashboards/063cd787-8630-4809-9702-34d3992c7248"
|
||||
},
|
||||
{
|
||||
"id": "df6c698e-066a-4440-be0a-121025573b73",
|
||||
"name": "deck.gl Demo",
|
||||
"fullyQualifiedName": "local_superset.deck.gl Demo",
|
||||
"description": "",
|
||||
"dashboardUrl": "http://localhost:808/superset/dashboard/deck/",
|
||||
"charts": [127, 166, 114],
|
||||
"href": "http://localhost:8585/api/v1/dashboards/df6c698e-066a-4440-be0a-121025573b73"
|
||||
},
|
||||
{
|
||||
"id": "98b38a49-b5c6-431b-b61f-690e39f8ead2",
|
||||
"name": "FCC New Coder Survey 2018",
|
||||
"fullyQualifiedName": "local_superset.FCC New Coder Survey 2018",
|
||||
"description": "",
|
||||
"dashboardUrl": "http://localhost:808/superset/dashboard/7/",
|
||||
"charts": [183, 197, 170, 180],
|
||||
"href": "http://localhost:8585/api/v1/dashboards/98b38a49-b5c6-431b-b61f-690e39f8ead2"
|
||||
},
|
||||
{
|
||||
"id": "dffcf9b2-4f43-4881-a5f5-10109655bf50",
|
||||
"name": "Misc Charts",
|
||||
"fullyQualifiedName": "local_superset.Misc Charts",
|
||||
"description": "",
|
||||
"dashboardUrl": "http://localhost:808/superset/dashboard/misc_charts/",
|
||||
"charts": [127, 197],
|
||||
"href": "http://localhost:8585/api/v1/dashboards/dffcf9b2-4f43-4881-a5f5-10109655bf50"
|
||||
},
|
||||
{
|
||||
"id": "2583737d-6236-421e-ba0f-cd0b79adb216",
|
||||
"name": "Sales Dashboard",
|
||||
"fullyQualifiedName": "local_superset.Sales Dashboard",
|
||||
"description": "",
|
||||
"dashboardUrl": "http://localhost:808/superset/dashboard/6/",
|
||||
"charts": [92,117,166],
|
||||
"href": "http://localhost:8585/api/v1/dashboards/2583737d-6236-421e-ba0f-cd0b79adb216"
|
||||
},
|
||||
{
|
||||
"id": "6bf9bfcb-4e80-4af0-9f0c-13e47bbc27a2",
|
||||
"name": "Slack Dashboard",
|
||||
"fullyQualifiedName": "local_superset.Slack Dashboard",
|
||||
"description": "",
|
||||
"dashboardUrl": "http://localhost:808/superset/dashboard/10/",
|
||||
"charts": [114, 92, 127],
|
||||
"href": "http://localhost:8585/api/v1/dashboards/6bf9bfcb-4e80-4af0-9f0c-13e47bbc27a2"
|
||||
},
|
||||
{
|
||||
"id": "1f02caf2-c5e5-442d-bda3-b8ce3e757b45",
|
||||
"name": "Unicode Test",
|
||||
"fullyQualifiedName": "local_superset.Unicode Test",
|
||||
"description": "",
|
||||
"dashboardUrl": "http://localhost:808/superset/dashboard/unicode-test/",
|
||||
"charts": [161, 170, 180],
|
||||
"href": "http://localhost:8585/api/v1/dashboards/1f02caf2-c5e5-442d-bda3-b8ce3e757b45"
|
||||
},
|
||||
{
|
||||
"id": "a3ace318-ee37-4da1-974a-62eddbd77d20",
|
||||
"name": "USA Births Names",
|
||||
"fullyQualifiedName": "local_superset.USA Births Names",
|
||||
"description": "",
|
||||
"dashboardUrl": "http://localhost:808/superset/dashboard/births/",
|
||||
"charts": [180],
|
||||
"href": "http://localhost:8585/api/v1/dashboards/a3ace318-ee37-4da1-974a-62eddbd77d20"
|
||||
},
|
||||
{
|
||||
"id": "e6e21717-1164-403f-8807-d12be277aec6",
|
||||
"name": "Video Game Sales",
|
||||
"fullyQualifiedName": "local_superset.Video Game Sales",
|
||||
"description": "",
|
||||
"dashboardUrl": "http://localhost:808/superset/dashboard/11/",
|
||||
"charts": [127, 183],
|
||||
"href": "http://localhost:8585/api/v1/dashboards/e6e21717-1164-403f-8807-d12be277aec6"
|
||||
}
|
||||
]
|
||||
}
|
||||
9
ingestion/examples/superset_data/service.json
Normal file
9
ingestion/examples/superset_data/service.json
Normal file
@ -0,0 +1,9 @@
|
||||
{
|
||||
"id": "a6fb4f54-ba3d-4a16-97f0-766713199141",
|
||||
"name": "sample_superset",
|
||||
"serviceType": "Superset",
|
||||
"description": "Supset Service",
|
||||
"dashboardUrl": "http://localhost:8088",
|
||||
"username": "admin",
|
||||
"password": "admin"
|
||||
}
|
||||
28
ingestion/pipelines/sample_dashboards.json
Normal file
28
ingestion/pipelines/sample_dashboards.json
Normal file
@ -0,0 +1,28 @@
|
||||
{
|
||||
"source": {
|
||||
"type": "sample-dashboards",
|
||||
"config": {
|
||||
"service_name": "sample_superset",
|
||||
"service_type": "Superset",
|
||||
"sample_dashboard_folder": "./examples/superset_data/"
|
||||
}
|
||||
},
|
||||
"sink": {
|
||||
"type": "metadata-rest-dashboards",
|
||||
"config": {}
|
||||
},
|
||||
"metadata_server": {
|
||||
"type": "metadata-server",
|
||||
"config": {
|
||||
"api_endpoint": "http://localhost:8585/api",
|
||||
"auth_provider_type": "no-auth"
|
||||
}
|
||||
},
|
||||
"cron": {
|
||||
"minute": "*/5",
|
||||
"hour": null,
|
||||
"day": null,
|
||||
"month": null,
|
||||
"day_of_week": null
|
||||
}
|
||||
}
|
||||
@ -223,11 +223,11 @@ class Chart(BaseModel):
|
||||
description: str
|
||||
chart_type: str
|
||||
url: str
|
||||
owners: List[DashboardOwner]
|
||||
lastModified: int
|
||||
datasource_fqn: str
|
||||
owners: List[DashboardOwner] = None
|
||||
lastModified: int = None
|
||||
datasource_fqn: str = None
|
||||
service: EntityReference
|
||||
custom_props: Dict[Any, Any]
|
||||
custom_props: Dict[Any, Any] = None
|
||||
|
||||
|
||||
class Dashboard(BaseModel):
|
||||
@ -235,7 +235,7 @@ class Dashboard(BaseModel):
|
||||
name: str
|
||||
description: str
|
||||
url: str
|
||||
owners: List
|
||||
owners: List = None
|
||||
charts: List
|
||||
service: EntityReference
|
||||
lastModified: int
|
||||
lastModified: int = None
|
||||
|
||||
@ -72,7 +72,6 @@ class MetadataRestDashboardsSink(Sink):
|
||||
return cls(ctx, config, metadata_config)
|
||||
|
||||
def write_record(self, record: Record) -> None:
|
||||
print(type(record))
|
||||
if isinstance(record, Chart):
|
||||
self._ingest_charts(record)
|
||||
elif isinstance(record, Dashboard):
|
||||
@ -118,13 +117,10 @@ class MetadataRestDashboardsSink(Sink):
|
||||
service=dashboard.service
|
||||
)
|
||||
created_dashboard = self.client.create_or_update_dashboard(dashboard_request)
|
||||
logger.info(
|
||||
'Successfully ingested {}'.format(created_dashboard.name))
|
||||
self.status.records_written(
|
||||
'{}'.format(created_dashboard.name))
|
||||
logger.info('Successfully ingested {}'.format(created_dashboard.name))
|
||||
self.status.records_written('{}'.format(created_dashboard.name))
|
||||
except (APIError, ValidationError) as err:
|
||||
logger.error(
|
||||
"Failed to ingest chart {}".format(dashboard.name))
|
||||
logger.error("Failed to ingest chart {}".format(dashboard.name))
|
||||
logger.error(err)
|
||||
self.status.failure(dashboard.name)
|
||||
|
||||
|
||||
111
ingestion/src/metadata/ingestion/source/sample_dashboards.py
Normal file
111
ingestion/src/metadata/ingestion/source/sample_dashboards.py
Normal file
@ -0,0 +1,111 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
|
||||
import json
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Iterable, List
|
||||
|
||||
from pydantic import ValidationError
|
||||
|
||||
from metadata.config.common import ConfigModel
|
||||
from metadata.generated.schema.api.services.createDashboardService import CreateDashboardServiceEntityRequest
|
||||
from metadata.generated.schema.entity.services.dashboardService import DashboardService
|
||||
from metadata.generated.schema.type.entityReference import EntityReference
|
||||
from metadata.ingestion.api.common import Record
|
||||
from metadata.ingestion.api.source import SourceStatus, Source
|
||||
from metadata.ingestion.models.table_metadata import Chart, Dashboard
|
||||
from metadata.ingestion.ometa.openmetadata_rest import OpenMetadataAPIClient, MetadataServerConfig
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
def get_service_or_create(service_json, metadata_config) -> DashboardService:
|
||||
client = OpenMetadataAPIClient(metadata_config)
|
||||
service = client.get_dashboard_service(service_json['name'])
|
||||
if service is not None:
|
||||
return service
|
||||
else:
|
||||
created_service = client.create_dashboard_service(CreateDashboardServiceEntityRequest(**service_json))
|
||||
return created_service
|
||||
|
||||
|
||||
class SampleDashboardSourceConfig(ConfigModel):
|
||||
sample_dashboard_folder: str
|
||||
service_name: str
|
||||
service_type: str = "Superset"
|
||||
|
||||
def get_sample_dashboard_folder(self):
|
||||
return self.sample_dashboard_folder
|
||||
|
||||
|
||||
@dataclass
|
||||
class SampleDashboardSourceStatus(SourceStatus):
|
||||
dashboards_scanned: List[str] = field(default_factory=list)
|
||||
|
||||
def report_dashboard_scanned(self, dashboard_name: str) -> None:
|
||||
self.dashboards_scanned.append(dashboard_name)
|
||||
|
||||
|
||||
class SampleDashboardsSource(Source):
|
||||
|
||||
def __init__(self, config: SampleDashboardSourceConfig, metadata_config: MetadataServerConfig, ctx):
|
||||
super().__init__(ctx)
|
||||
self.status = SampleDashboardSourceStatus()
|
||||
self.config = config
|
||||
self.metadata_config = metadata_config
|
||||
self.client = OpenMetadataAPIClient(metadata_config)
|
||||
self.service_json = json.load(open(config.sample_dashboard_folder + "/service.json", 'r'))
|
||||
self.charts = json.load(open(config.sample_dashboard_folder + "/charts.json", 'r'))
|
||||
self.dashboards = json.load(open(config.sample_dashboard_folder + "/dashboards.json", 'r'))
|
||||
self.service = get_service_or_create(self.service_json, metadata_config)
|
||||
|
||||
@classmethod
|
||||
def create(cls, config_dict, metadata_config_dict, ctx):
|
||||
config = SampleDashboardSourceConfig.parse_obj(config_dict)
|
||||
metadata_config = MetadataServerConfig.parse_obj(metadata_config_dict)
|
||||
return cls(config, metadata_config, ctx)
|
||||
|
||||
def prepare(self):
|
||||
pass
|
||||
|
||||
def next_record(self) -> Iterable[Record]:
|
||||
for chart in self.charts['charts']:
|
||||
try:
|
||||
chart_ev = Chart(name=chart['name'],
|
||||
description=chart['description'],
|
||||
chart_id=chart['chartId'],
|
||||
chart_type=chart['chartType'],
|
||||
url=chart['chartUrl'],
|
||||
service=EntityReference(id=self.service.id, type="dashboardService"))
|
||||
yield chart_ev
|
||||
except ValidationError as err:
|
||||
logger.error(err)
|
||||
|
||||
|
||||
for dashboard in self.dashboards['dashboards']:
|
||||
dashboard_ev = Dashboard(name=dashboard['name'],
|
||||
description=dashboard['description'],
|
||||
url=dashboard['dashboardUrl'],
|
||||
charts=dashboard['charts'],
|
||||
service=EntityReference(id=self.service.id, type="dashboardService"))
|
||||
yield dashboard_ev
|
||||
|
||||
def close(self):
|
||||
self.client.close()
|
||||
|
||||
def get_status(self):
|
||||
return self.status
|
||||
Loading…
x
Reference in New Issue
Block a user