From fd99dbdb1a33748f71d1d8df7f4e23a052e82db2 Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Mon, 30 Aug 2021 22:13:45 -0700 Subject: [PATCH] Fix #339: Add sample dashboards & charts --- ingestion/examples/superset_data/charts.json | 95 +++++++++++++++ .../examples/superset_data/dashboards.json | 94 +++++++++++++++ ingestion/examples/superset_data/service.json | 9 ++ ingestion/pipelines/sample_dashboards.json | 28 +++++ .../ingestion/models/table_metadata.py | 12 +- .../sink/metadata_rest_dashboards.py | 10 +- .../ingestion/source/sample_dashboards.py | 111 ++++++++++++++++++ 7 files changed, 346 insertions(+), 13 deletions(-) create mode 100644 ingestion/examples/superset_data/charts.json create mode 100644 ingestion/examples/superset_data/dashboards.json create mode 100644 ingestion/examples/superset_data/service.json create mode 100644 ingestion/pipelines/sample_dashboards.json create mode 100644 ingestion/src/metadata/ingestion/source/sample_dashboards.py diff --git a/ingestion/examples/superset_data/charts.json b/ingestion/examples/superset_data/charts.json new file mode 100644 index 00000000000..828cbc83c4d --- /dev/null +++ b/ingestion/examples/superset_data/charts.json @@ -0,0 +1,95 @@ +{ + "charts": [ + { + "id": "2841fdb1-e378-4a2c-94f8-27c9f5d6ef8e", + "name": "# of Games That Hit 100k in Sales By Release Year", + "fullyQualifiedName": "local_superset.# of Games That Hit 100k in Sales By Release Year", + "description": "", + "chartId": "114", + "chartType": "Area", + "chartUrl": "http://localhost:8088/superset/explore/?form_data=%7B%22slice_id%22%3A%20114%7D", + "href": "http://localhost:8585/api/v1/charts/2841fdb1-e378-4a2c-94f8-27c9f5d6ef8e" + }, { + "id": "3bcba490-9e5c-4946-a0e3-41e8ff8f4aa4", + "name": "% Rural", + "fullyQualifiedName": "local_superset.% Rural", + "description": "", + "chartId": "166", + "chartType": "Other", + "chartUrl": "http://localhost:8088/superset/explore/?form_data=%7B%22slice_id%22%3A%20166%7D", + "href": "http://localhost:8585/api/v1/charts/3bcba490-9e5c-4946-a0e3-41e8ff8f4aa4" + }, { + "id": "22b95748-4a7b-48ad-859e-cf7c66a7f343", + "name": "✈️ Relocation ability", + "fullyQualifiedName": "local_superset.✈️ Relocation ability", + "description": "", + "chartId": "92", + "chartType": "Other", + "chartUrl": "http://localhost:8088/superset/explore/?form_data=%7B%22slice_id%22%3A%2092%7D", + "href": "http://localhost:8585/api/v1/charts/22b95748-4a7b-48ad-859e-cf7c66a7f343" + }, { + "id": "62b31dcc-4619-46a0-99b1-0fa7cd6f93da", + "name": "Age distribution of respondents", + "fullyQualifiedName": "local_superset.Age distribution of respondents", + "description": "", + "chartId": "117", + "chartType": "Histogram", + "chartUrl": "http://localhost:8088/superset/explore/?form_data=%7B%22slice_id%22%3A%20117%7D", + "href": "http://localhost:8585/api/v1/charts/62b31dcc-4619-46a0-99b1-0fa7cd6f93da" + }, { + "id": "57944482-e187-439a-aaae-0e8aabd2f455", + "name": "Arcs", + "fullyQualifiedName": "local_superset.Arcs", + "description": "", + "chartId": "197", + "chartType": "Other", + "chartUrl": "http://localhost:8088/superset/explore/?form_data=%7B%22slice_id%22%3A%20197%7D", + "href": "http://localhost:8585/api/v1/charts/57944482-e187-439a-aaae-0e8aabd2f455" + }, { + "id": "d88e2056-c74a-410d-829e-eb31b040c132", + "name": "Are you an ethnic minority in your city?", + "fullyQualifiedName": "local_superset.Are you an ethnic minority in your city?", + "description": "", + "chartId": "127", + "chartType": "Other", + "chartUrl": "http://localhost:8088/superset/explore/?form_data=%7B%22slice_id%22%3A%20127%7D", + "href": "http://localhost:8585/api/v1/charts/d88e2056-c74a-410d-829e-eb31b040c132" + }, { + "id": "c1d3e156-4628-414e-8d6e-a6bdd486128f", + "name": "Average and Sum Trends", + "fullyQualifiedName": "local_superset.Average and Sum Trends", + "description": "", + "chartId": "183", + "chartType": "Line", + "chartUrl": "http://localhost:8088/superset/explore/?form_data=%7B%22slice_id%22%3A%20183%7D", + "href": "http://localhost:8585/api/v1/charts/c1d3e156-4628-414e-8d6e-a6bdd486128f" + }, { + "id": "bfc57519-8cef-47e6-a423-375d5b89a6a4", + "name": "Birth in France by department in 2016", + "fullyQualifiedName": "local_superset.Birth in France by department in 2016", + "description": "", + "chartId": "161", + "chartType": "Other", + "chartUrl": "http://localhost:8088/superset/explore/?form_data=%7B%22slice_id%22%3A%20161%7D", + "href": "http://localhost:8585/api/v1/charts/bfc57519-8cef-47e6-a423-375d5b89a6a4" + }, { + "id": "bf2eeac4-7226-46c6-bbef-918569c137a0", + "name": "Box plot", + "fullyQualifiedName": "local_superset.Box plot", + "description": "", + "chartId": "170", + "chartType": "Bar", + "chartUrl": "http://localhost:8088/superset/explore/?form_data=%7B%22slice_id%22%3A%20170%7D", + "href": "http://localhost:8585/api/v1/charts/bf2eeac4-7226-46c6-bbef-918569c137a0" + }, { + "id": "167fd63b-42f1-4d7e-a37d-893fd8173b44", + "name": "Boy Name Cloud", + "fullyQualifiedName": "local_superset.Boy Name Cloud", + "description": "", + "chartId": "180", + "chartType": "Other", + "chartUrl": "http://localhost:8088/superset/explore/?form_data=%7B%22slice_id%22%3A%20180%7D", + "href": "http://localhost:8585/api/v1/charts/167fd63b-42f1-4d7e-a37d-893fd8173b44" + } +] +} diff --git a/ingestion/examples/superset_data/dashboards.json b/ingestion/examples/superset_data/dashboards.json new file mode 100644 index 00000000000..51ee4972865 --- /dev/null +++ b/ingestion/examples/superset_data/dashboards.json @@ -0,0 +1,94 @@ +{ + "dashboards": [ + { + "id": "d4dc7baf-1b17-45f8-acd5-a15b78cc7c5f", + "name": "[ untitled dashboard ]", + "fullyQualifiedName": "local_superset.[ untitled dashboard ]", + "description": "", + "dashboardUrl": "http://localhost:808/superset/dashboard/1/", + "charts": [183, 170, 197], + "href": "http://localhost:8585/api/v1/dashboards/d4dc7baf-1b17-45f8-acd5-a15b78cc7c5f" + }, + { + "id": "063cd787-8630-4809-9702-34d3992c7248", + "name": "COVID Vaccine Dashboard", + "fullyQualifiedName": "local_superset.COVID Vaccine Dashboard", + "description": "", + "dashboardUrl": "http://localhost:808/superset/dashboard/8/", + "charts": [117, 197], + "href": "http://localhost:8585/api/v1/dashboards/063cd787-8630-4809-9702-34d3992c7248" + }, + { + "id": "df6c698e-066a-4440-be0a-121025573b73", + "name": "deck.gl Demo", + "fullyQualifiedName": "local_superset.deck.gl Demo", + "description": "", + "dashboardUrl": "http://localhost:808/superset/dashboard/deck/", + "charts": [127, 166, 114], + "href": "http://localhost:8585/api/v1/dashboards/df6c698e-066a-4440-be0a-121025573b73" + }, + { + "id": "98b38a49-b5c6-431b-b61f-690e39f8ead2", + "name": "FCC New Coder Survey 2018", + "fullyQualifiedName": "local_superset.FCC New Coder Survey 2018", + "description": "", + "dashboardUrl": "http://localhost:808/superset/dashboard/7/", + "charts": [183, 197, 170, 180], + "href": "http://localhost:8585/api/v1/dashboards/98b38a49-b5c6-431b-b61f-690e39f8ead2" + }, + { + "id": "dffcf9b2-4f43-4881-a5f5-10109655bf50", + "name": "Misc Charts", + "fullyQualifiedName": "local_superset.Misc Charts", + "description": "", + "dashboardUrl": "http://localhost:808/superset/dashboard/misc_charts/", + "charts": [127, 197], + "href": "http://localhost:8585/api/v1/dashboards/dffcf9b2-4f43-4881-a5f5-10109655bf50" + }, + { + "id": "2583737d-6236-421e-ba0f-cd0b79adb216", + "name": "Sales Dashboard", + "fullyQualifiedName": "local_superset.Sales Dashboard", + "description": "", + "dashboardUrl": "http://localhost:808/superset/dashboard/6/", + "charts": [92,117,166], + "href": "http://localhost:8585/api/v1/dashboards/2583737d-6236-421e-ba0f-cd0b79adb216" + }, + { + "id": "6bf9bfcb-4e80-4af0-9f0c-13e47bbc27a2", + "name": "Slack Dashboard", + "fullyQualifiedName": "local_superset.Slack Dashboard", + "description": "", + "dashboardUrl": "http://localhost:808/superset/dashboard/10/", + "charts": [114, 92, 127], + "href": "http://localhost:8585/api/v1/dashboards/6bf9bfcb-4e80-4af0-9f0c-13e47bbc27a2" + }, + { + "id": "1f02caf2-c5e5-442d-bda3-b8ce3e757b45", + "name": "Unicode Test", + "fullyQualifiedName": "local_superset.Unicode Test", + "description": "", + "dashboardUrl": "http://localhost:808/superset/dashboard/unicode-test/", + "charts": [161, 170, 180], + "href": "http://localhost:8585/api/v1/dashboards/1f02caf2-c5e5-442d-bda3-b8ce3e757b45" + }, + { + "id": "a3ace318-ee37-4da1-974a-62eddbd77d20", + "name": "USA Births Names", + "fullyQualifiedName": "local_superset.USA Births Names", + "description": "", + "dashboardUrl": "http://localhost:808/superset/dashboard/births/", + "charts": [180], + "href": "http://localhost:8585/api/v1/dashboards/a3ace318-ee37-4da1-974a-62eddbd77d20" + }, + { + "id": "e6e21717-1164-403f-8807-d12be277aec6", + "name": "Video Game Sales", + "fullyQualifiedName": "local_superset.Video Game Sales", + "description": "", + "dashboardUrl": "http://localhost:808/superset/dashboard/11/", + "charts": [127, 183], + "href": "http://localhost:8585/api/v1/dashboards/e6e21717-1164-403f-8807-d12be277aec6" + } + ] +} diff --git a/ingestion/examples/superset_data/service.json b/ingestion/examples/superset_data/service.json new file mode 100644 index 00000000000..2e896705025 --- /dev/null +++ b/ingestion/examples/superset_data/service.json @@ -0,0 +1,9 @@ +{ + "id": "a6fb4f54-ba3d-4a16-97f0-766713199141", + "name": "sample_superset", + "serviceType": "Superset", + "description": "Supset Service", + "dashboardUrl": "http://localhost:8088", + "username": "admin", + "password": "admin" +} diff --git a/ingestion/pipelines/sample_dashboards.json b/ingestion/pipelines/sample_dashboards.json new file mode 100644 index 00000000000..a27d3bddee0 --- /dev/null +++ b/ingestion/pipelines/sample_dashboards.json @@ -0,0 +1,28 @@ +{ + "source": { + "type": "sample-dashboards", + "config": { + "service_name": "sample_superset", + "service_type": "Superset", + "sample_dashboard_folder": "./examples/superset_data/" + } + }, + "sink": { + "type": "metadata-rest-dashboards", + "config": {} + }, + "metadata_server": { + "type": "metadata-server", + "config": { + "api_endpoint": "http://localhost:8585/api", + "auth_provider_type": "no-auth" + } + }, + "cron": { + "minute": "*/5", + "hour": null, + "day": null, + "month": null, + "day_of_week": null + } +} diff --git a/ingestion/src/metadata/ingestion/models/table_metadata.py b/ingestion/src/metadata/ingestion/models/table_metadata.py index c25fed79723..e8cc65ce64e 100644 --- a/ingestion/src/metadata/ingestion/models/table_metadata.py +++ b/ingestion/src/metadata/ingestion/models/table_metadata.py @@ -223,11 +223,11 @@ class Chart(BaseModel): description: str chart_type: str url: str - owners: List[DashboardOwner] - lastModified: int - datasource_fqn: str + owners: List[DashboardOwner] = None + lastModified: int = None + datasource_fqn: str = None service: EntityReference - custom_props: Dict[Any, Any] + custom_props: Dict[Any, Any] = None class Dashboard(BaseModel): @@ -235,7 +235,7 @@ class Dashboard(BaseModel): name: str description: str url: str - owners: List + owners: List = None charts: List service: EntityReference - lastModified: int + lastModified: int = None diff --git a/ingestion/src/metadata/ingestion/sink/metadata_rest_dashboards.py b/ingestion/src/metadata/ingestion/sink/metadata_rest_dashboards.py index ca08f04c601..0dd3d714502 100644 --- a/ingestion/src/metadata/ingestion/sink/metadata_rest_dashboards.py +++ b/ingestion/src/metadata/ingestion/sink/metadata_rest_dashboards.py @@ -72,7 +72,6 @@ class MetadataRestDashboardsSink(Sink): return cls(ctx, config, metadata_config) def write_record(self, record: Record) -> None: - print(type(record)) if isinstance(record, Chart): self._ingest_charts(record) elif isinstance(record, Dashboard): @@ -118,13 +117,10 @@ class MetadataRestDashboardsSink(Sink): service=dashboard.service ) created_dashboard = self.client.create_or_update_dashboard(dashboard_request) - logger.info( - 'Successfully ingested {}'.format(created_dashboard.name)) - self.status.records_written( - '{}'.format(created_dashboard.name)) + logger.info('Successfully ingested {}'.format(created_dashboard.name)) + self.status.records_written('{}'.format(created_dashboard.name)) except (APIError, ValidationError) as err: - logger.error( - "Failed to ingest chart {}".format(dashboard.name)) + logger.error("Failed to ingest chart {}".format(dashboard.name)) logger.error(err) self.status.failure(dashboard.name) diff --git a/ingestion/src/metadata/ingestion/source/sample_dashboards.py b/ingestion/src/metadata/ingestion/source/sample_dashboards.py new file mode 100644 index 00000000000..64b9edd9d32 --- /dev/null +++ b/ingestion/src/metadata/ingestion/source/sample_dashboards.py @@ -0,0 +1,111 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import json +import logging +from dataclasses import dataclass, field +from typing import Iterable, List + +from pydantic import ValidationError + +from metadata.config.common import ConfigModel +from metadata.generated.schema.api.services.createDashboardService import CreateDashboardServiceEntityRequest +from metadata.generated.schema.entity.services.dashboardService import DashboardService +from metadata.generated.schema.type.entityReference import EntityReference +from metadata.ingestion.api.common import Record +from metadata.ingestion.api.source import SourceStatus, Source +from metadata.ingestion.models.table_metadata import Chart, Dashboard +from metadata.ingestion.ometa.openmetadata_rest import OpenMetadataAPIClient, MetadataServerConfig + + +logger = logging.getLogger(__name__) + +def get_service_or_create(service_json, metadata_config) -> DashboardService: + client = OpenMetadataAPIClient(metadata_config) + service = client.get_dashboard_service(service_json['name']) + if service is not None: + return service + else: + created_service = client.create_dashboard_service(CreateDashboardServiceEntityRequest(**service_json)) + return created_service + + +class SampleDashboardSourceConfig(ConfigModel): + sample_dashboard_folder: str + service_name: str + service_type: str = "Superset" + + def get_sample_dashboard_folder(self): + return self.sample_dashboard_folder + + +@dataclass +class SampleDashboardSourceStatus(SourceStatus): + dashboards_scanned: List[str] = field(default_factory=list) + + def report_dashboard_scanned(self, dashboard_name: str) -> None: + self.dashboards_scanned.append(dashboard_name) + + +class SampleDashboardsSource(Source): + + def __init__(self, config: SampleDashboardSourceConfig, metadata_config: MetadataServerConfig, ctx): + super().__init__(ctx) + self.status = SampleDashboardSourceStatus() + self.config = config + self.metadata_config = metadata_config + self.client = OpenMetadataAPIClient(metadata_config) + self.service_json = json.load(open(config.sample_dashboard_folder + "/service.json", 'r')) + self.charts = json.load(open(config.sample_dashboard_folder + "/charts.json", 'r')) + self.dashboards = json.load(open(config.sample_dashboard_folder + "/dashboards.json", 'r')) + self.service = get_service_or_create(self.service_json, metadata_config) + + @classmethod + def create(cls, config_dict, metadata_config_dict, ctx): + config = SampleDashboardSourceConfig.parse_obj(config_dict) + metadata_config = MetadataServerConfig.parse_obj(metadata_config_dict) + return cls(config, metadata_config, ctx) + + def prepare(self): + pass + + def next_record(self) -> Iterable[Record]: + for chart in self.charts['charts']: + try: + chart_ev = Chart(name=chart['name'], + description=chart['description'], + chart_id=chart['chartId'], + chart_type=chart['chartType'], + url=chart['chartUrl'], + service=EntityReference(id=self.service.id, type="dashboardService")) + yield chart_ev + except ValidationError as err: + logger.error(err) + + + for dashboard in self.dashboards['dashboards']: + dashboard_ev = Dashboard(name=dashboard['name'], + description=dashboard['description'], + url=dashboard['dashboardUrl'], + charts=dashboard['charts'], + service=EntityReference(id=self.service.id, type="dashboardService")) + yield dashboard_ev + + def close(self): + self.client.close() + + def get_status(self): + return self.status