mirror of
https://github.com/open-metadata/OpenMetadata.git
synced 2025-12-02 18:46:02 +00:00
Add Looker Sample Data (#11047)
This commit is contained in:
Pere Miquel Brull
GitHub
parent
c0172173ae
commit
4c9690b0c2
30
ingestion/examples/sample_data/looker/charts.json
Normal file
30
ingestion/examples/sample_data/looker/charts.json
Normal file
@ -0,0 +1,30 @@
|
||||
[
|
||||
{
|
||||
"name": "chart_1",
|
||||
"displayName": "Chart 1",
|
||||
"description": "This is a sample Chart for Looker",
|
||||
"chartType": "Text",
|
||||
"chartUrl": "http://localhost:8088/looker/explore/?form_data=%7B%22slice_id%22%3A%20114%7D"
|
||||
},
|
||||
{
|
||||
"name":"chart_2",
|
||||
"displayName": "Chart 2",
|
||||
"description": "This is a sample Chart for Looker",
|
||||
"chartType": "Bar",
|
||||
"chartUrl": "http://localhost:8088/looker/explore/?form_data=%7B%22slice_id%22%3A%20166%7D"
|
||||
},
|
||||
{
|
||||
"name": "chart_3",
|
||||
"displayName": "✈️ Chart 3",
|
||||
"description": "This is a sample Chart for Looker",
|
||||
"chartType": "Other",
|
||||
"chartUrl": "http://localhost:8088/looker/explore/?form_data=%7B%22slice_id%22%3A%2092%7D"
|
||||
},
|
||||
{
|
||||
"name": "chart_4",
|
||||
"displayName": "Chart 4",
|
||||
"description": "This is a sample Chart for Looker",
|
||||
"chartType": "Histogram",
|
||||
"chartUrl": "http://localhost:8088/looker/explore/?form_data=%7B%22slice_id%22%3A%20117%7D"
|
||||
}
|
||||
]
|
||||
163
ingestion/examples/sample_data/looker/dashboardDataModels.json
Normal file
163
ingestion/examples/sample_data/looker/dashboardDataModels.json
Normal file
@ -0,0 +1,163 @@
|
||||
[
|
||||
{
|
||||
"name": "orders",
|
||||
"displayName": "Orders",
|
||||
"description": "Orders explore from Sample Data",
|
||||
"dataModelType": "LookMlExplore",
|
||||
"serviceType": "Looker",
|
||||
"sql": "SELECT CASE\n WHEN stage_of_development = 'Pre-clinical' THEN '0. Pre-clinical'\n WHEN stage_of_development = 'Phase I' THEN '1. Phase I'\n WHEN stage_of_development = 'Phase I/II'\n or stage_of_development = 'Phase II' THEN '2. Phase II or Combined I/II'\n WHEN stage_of_development = 'Phase III' THEN '3. Phase III'\n WHEN stage_of_development = 'Authorized' THEN '4. Authorized'\n END AS clinical_stage,\n COUNT(*) AS count\nFROM covid_vaccines\nGROUP BY CASE\n WHEN stage_of_development = 'Pre-clinical' THEN '0. Pre-clinical'\n WHEN stage_of_development = 'Phase I' THEN '1. Phase I'\n WHEN stage_of_development = 'Phase I/II'\n or stage_of_development = 'Phase II' THEN '2. Phase II or Combined I/II'\n WHEN stage_of_development = 'Phase III' THEN '3. Phase III'\n WHEN stage_of_development = 'Authorized' THEN '4. Authorized'\n END\nORDER BY count DESC\nLIMIT 10000\nOFFSET 0;\n",
|
||||
"columns": [
|
||||
{
|
||||
"name": "0. Pre-clinical",
|
||||
"dataType": "NUMERIC",
|
||||
"dataTypeDisplay": "numeric",
|
||||
"description": "Vaccine Candidates in phase: 'Pre-clinical'",
|
||||
"ordinalPosition": 1
|
||||
},
|
||||
{
|
||||
"name": "2. Phase II or Combined I/II",
|
||||
"dataType": "NUMERIC",
|
||||
"dataTypeDisplay": "numeric",
|
||||
"description": "Vaccine Candidates in phase: 'Phase II or Combined I/II'",
|
||||
"ordinalPosition": 2
|
||||
},
|
||||
{
|
||||
"name": "1. Phase I",
|
||||
"dataType": "NUMERIC",
|
||||
"dataTypeDisplay": "numeric",
|
||||
"description": "Vaccine Candidates in phase: 'Phase I'",
|
||||
"ordinalPosition": 3
|
||||
},
|
||||
{
|
||||
"name": "3. Phase III",
|
||||
"dataType": "NUMERIC",
|
||||
"dataTypeDisplay": "numeric",
|
||||
"description": "Vaccine Candidates in phase: 'Phase III'",
|
||||
"ordinalPosition": 4
|
||||
},
|
||||
{
|
||||
"name": "4. Authorized",
|
||||
"dataType": "NUMERIC",
|
||||
"dataTypeDisplay": "numeric",
|
||||
"description": "Vaccine Candidates in phase: 'Authorize'",
|
||||
"ordinalPosition": 5
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "operations",
|
||||
"displayName": "Operations",
|
||||
"description": "Operations Explore from Sample Data",
|
||||
"dataModelType": "LookMlExplore",
|
||||
"serviceType": "Looker",
|
||||
"sql": "SELECT country_name AS country_name,\n COUNT(*) AS count,\n count(country_name) AS \"COUNT(Country_Name)\"\nFROM covid_vaccines\nGROUP BY country_name\nLIMIT 10000\nOFFSET 0;",
|
||||
"columns": [
|
||||
{
|
||||
"name": "country_name",
|
||||
"dataType": "VARCHAR",
|
||||
"dataTypeDisplay": "varchar",
|
||||
"dataLength": 256,
|
||||
"description": "Name of the country.",
|
||||
"tags": [],
|
||||
"ordinalPosition": 1
|
||||
},
|
||||
{
|
||||
"name": "count",
|
||||
"dataType": "NUMERIC",
|
||||
"dataTypeDisplay": "numeric",
|
||||
"description": "Total number of vaccine candidates per country.",
|
||||
"tags": [],
|
||||
"ordinalPosition": 2
|
||||
},
|
||||
{
|
||||
"name": "COUNT(Country_Name)",
|
||||
"dataType": "NUMERIC",
|
||||
"dataTypeDisplay": "numeric",
|
||||
"description": "Total number of vaccine candidates.",
|
||||
"tags": [],
|
||||
"ordinalPosition": 3
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "operations_view",
|
||||
"displayName": "Operations View",
|
||||
"description": "Operations View from Sample Data",
|
||||
"dataModelType": "LookMlView",
|
||||
"serviceType": "Looker",
|
||||
"sql": "SELECT country_name AS country_name,\n COUNT(*) AS count,\n count(country_name) AS \"COUNT(Country_Name)\"\nFROM covid_vaccines\nGROUP BY country_name\nLIMIT 10000\nOFFSET 0;",
|
||||
"columns": [
|
||||
{
|
||||
"name": "country_name",
|
||||
"dataType": "VARCHAR",
|
||||
"dataTypeDisplay": "varchar",
|
||||
"dataLength": 256,
|
||||
"description": "Name of the country.",
|
||||
"tags": [],
|
||||
"ordinalPosition": 1
|
||||
},
|
||||
{
|
||||
"name": "count",
|
||||
"dataType": "NUMERIC",
|
||||
"dataTypeDisplay": "numeric",
|
||||
"description": "Total number of vaccine candidates per country.",
|
||||
"tags": [],
|
||||
"ordinalPosition": 2
|
||||
},
|
||||
{
|
||||
"name": "COUNT(Country_Name)",
|
||||
"dataType": "NUMERIC",
|
||||
"dataTypeDisplay": "numeric",
|
||||
"description": "Total number of vaccine candidates.",
|
||||
"tags": [],
|
||||
"ordinalPosition": 3
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "orders_view",
|
||||
"displayName": "Orders View",
|
||||
"description": "Orders View from Sample Data",
|
||||
"dataModelType": "LookMlView",
|
||||
"serviceType": "Looker",
|
||||
"sql": "SELECT CASE\n WHEN stage_of_development = 'Pre-clinical' THEN '0. Pre-clinical'\n WHEN stage_of_development = 'Phase I' THEN '1. Phase I'\n WHEN stage_of_development = 'Phase I/II'\n or stage_of_development = 'Phase II' THEN '2. Phase II or Combined I/II'\n WHEN stage_of_development = 'Phase III' THEN '3. Phase III'\n WHEN stage_of_development = 'Authorized' THEN '4. Authorized'\n END AS clinical_stage,\n COUNT(*) AS count\nFROM covid_vaccines\nGROUP BY CASE\n WHEN stage_of_development = 'Pre-clinical' THEN '0. Pre-clinical'\n WHEN stage_of_development = 'Phase I' THEN '1. Phase I'\n WHEN stage_of_development = 'Phase I/II'\n or stage_of_development = 'Phase II' THEN '2. Phase II or Combined I/II'\n WHEN stage_of_development = 'Phase III' THEN '3. Phase III'\n WHEN stage_of_development = 'Authorized' THEN '4. Authorized'\n END\nORDER BY count DESC\nLIMIT 10000\nOFFSET 0;\n",
|
||||
"columns": [
|
||||
{
|
||||
"name": "0. Pre-clinical",
|
||||
"dataType": "NUMERIC",
|
||||
"dataTypeDisplay": "numeric",
|
||||
"description": "Vaccine Candidates in phase: 'Pre-clinical'",
|
||||
"ordinalPosition": 1
|
||||
},
|
||||
{
|
||||
"name": "2. Phase II or Combined I/II",
|
||||
"dataType": "NUMERIC",
|
||||
"dataTypeDisplay": "numeric",
|
||||
"description": "Vaccine Candidates in phase: 'Phase II or Combined I/II'",
|
||||
"ordinalPosition": 2
|
||||
},
|
||||
{
|
||||
"name": "1. Phase I",
|
||||
"dataType": "NUMERIC",
|
||||
"dataTypeDisplay": "numeric",
|
||||
"description": "Vaccine Candidates in phase: 'Phase I'",
|
||||
"ordinalPosition": 3
|
||||
},
|
||||
{
|
||||
"name": "3. Phase III",
|
||||
"dataType": "NUMERIC",
|
||||
"dataTypeDisplay": "numeric",
|
||||
"description": "Vaccine Candidates in phase: 'Phase III'",
|
||||
"ordinalPosition": 4
|
||||
},
|
||||
{
|
||||
"name": "4. Authorized",
|
||||
"dataType": "NUMERIC",
|
||||
"dataTypeDisplay": "numeric",
|
||||
"description": "Vaccine Candidates in phase: 'Authorize'",
|
||||
"ordinalPosition": 5
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
|
||||
17
ingestion/examples/sample_data/looker/dashboards.json
Normal file
17
ingestion/examples/sample_data/looker/dashboards.json
Normal file
@ -0,0 +1,17 @@
|
||||
[
|
||||
{
|
||||
"name": "customers",
|
||||
"displayName": "Customers dashboard",
|
||||
"description": "This is a sample Dashboard for Looker",
|
||||
"dashboardUrl": "http://localhost:808/looker/dashboard/1/",
|
||||
"charts": ["sample_looker.chart_1", "sample_looker.chart_2"]
|
||||
},
|
||||
{
|
||||
"name": "orders",
|
||||
"displayName": "Orders Dashboard",
|
||||
"description": "This is a sample Dashboard for Looker",
|
||||
"dashboardUrl": "http://localhost:808/looker/dashboard/8/",
|
||||
"charts": ["sample_looker.chart_3", "sample_looker.chart_4"],
|
||||
"dataModels": ["sample_looker.model.orders", "sample_looker.model.operations"]
|
||||
}
|
||||
]
|
||||
15
ingestion/examples/sample_data/looker/service.json
Normal file
15
ingestion/examples/sample_data/looker/service.json
Normal file
@ -0,0 +1,15 @@
|
||||
{
|
||||
"type": "looker",
|
||||
"serviceName": "sample_looker",
|
||||
"serviceConnection": {
|
||||
"config": {
|
||||
"type": "Looker",
|
||||
"hostPort": "https://looker.com",
|
||||
"clientId": "admin",
|
||||
"clientSecret": "admin"
|
||||
}
|
||||
},
|
||||
"sourceConfig": {
|
||||
"config": {}
|
||||
}
|
||||
}
|
||||
@ -11,6 +11,7 @@
|
||||
"""
|
||||
Sample Data source ingestion
|
||||
"""
|
||||
# pylint: disable=too-many-lines,too-many-statements
|
||||
import json
|
||||
import random
|
||||
import traceback
|
||||
@ -264,6 +265,37 @@ class SampleDataSource(
|
||||
entity=MessagingService, config=WorkflowSource(**self.kafka_service_json)
|
||||
)
|
||||
|
||||
with open(
|
||||
sample_data_folder + "/looker/service.json",
|
||||
"r",
|
||||
encoding=UTF_8,
|
||||
) as file:
|
||||
self.looker_service = self.metadata.get_service_or_create(
|
||||
entity=DashboardService,
|
||||
config=WorkflowSource(**json.load(file)),
|
||||
)
|
||||
|
||||
with open(
|
||||
sample_data_folder + "/looker/charts.json",
|
||||
"r",
|
||||
encoding=UTF_8,
|
||||
) as file:
|
||||
self.looker_charts = json.load(file)
|
||||
|
||||
with open(
|
||||
sample_data_folder + "/looker/dashboards.json",
|
||||
"r",
|
||||
encoding=UTF_8,
|
||||
) as file:
|
||||
self.looker_dashboards = json.load(file)
|
||||
|
||||
with open(
|
||||
sample_data_folder + "/looker/dashboardDataModels.json",
|
||||
"r",
|
||||
encoding=UTF_8,
|
||||
) as file:
|
||||
self.looker_models = json.load(file)
|
||||
|
||||
self.dashboard_service_json = json.load(
|
||||
open( # pylint: disable=consider-using-with
|
||||
sample_data_folder + "/dashboards/service.json",
|
||||
@ -436,6 +468,7 @@ class SampleDataSource(
|
||||
yield from self.ingest_charts()
|
||||
yield from self.ingest_data_models()
|
||||
yield from self.ingest_dashboards()
|
||||
yield from self.ingest_looker()
|
||||
yield from self.ingest_pipelines()
|
||||
yield from self.ingest_lineage()
|
||||
yield from self.ingest_pipeline_status()
|
||||
@ -623,6 +656,113 @@ class SampleDataSource(
|
||||
self.status.scanned(f"Topic Scanned: {create_topic.name.__root__}")
|
||||
yield create_topic
|
||||
|
||||
def ingest_looker(self) -> Iterable[Entity]:
|
||||
"""
|
||||
Looker sample data
|
||||
"""
|
||||
for data_model in self.looker_models:
|
||||
try:
|
||||
data_model_ev = CreateDashboardDataModelRequest(
|
||||
name=data_model["name"],
|
||||
displayName=data_model["displayName"],
|
||||
description=data_model["description"],
|
||||
columns=data_model["columns"],
|
||||
dataModelType=data_model["dataModelType"],
|
||||
sql=data_model["sql"],
|
||||
serviceType=data_model["serviceType"],
|
||||
service=self.looker_service.fullyQualifiedName,
|
||||
)
|
||||
self.status.scanned(
|
||||
f"Data Model Scanned: {data_model_ev.name.__root__}"
|
||||
)
|
||||
yield data_model_ev
|
||||
except ValidationError as err:
|
||||
logger.debug(traceback.format_exc())
|
||||
logger.warning(
|
||||
f"Unexpected exception ingesting chart [{data_model}]: {err}"
|
||||
)
|
||||
|
||||
for chart in self.looker_charts:
|
||||
try:
|
||||
chart_ev = CreateChartRequest(
|
||||
name=chart["name"],
|
||||
displayName=chart["displayName"],
|
||||
description=chart["description"],
|
||||
chartType=get_standard_chart_type(chart["chartType"]),
|
||||
chartUrl=chart["chartUrl"],
|
||||
service=self.looker_service.fullyQualifiedName,
|
||||
)
|
||||
self.status.scanned(f"Chart Scanned: {chart_ev.name.__root__}")
|
||||
yield chart_ev
|
||||
except ValidationError as err:
|
||||
logger.debug(traceback.format_exc())
|
||||
logger.warning(f"Unexpected exception ingesting chart [{chart}]: {err}")
|
||||
|
||||
for dashboard in self.looker_dashboards:
|
||||
try:
|
||||
dashboard_ev = CreateDashboardRequest(
|
||||
name=dashboard["name"],
|
||||
displayName=dashboard["displayName"],
|
||||
description=dashboard["description"],
|
||||
dashboardUrl=dashboard["dashboardUrl"],
|
||||
charts=dashboard["charts"],
|
||||
dataModels=dashboard.get("dataModels", None),
|
||||
service=self.looker_service.fullyQualifiedName,
|
||||
)
|
||||
self.status.scanned(f"Dashboard Scanned: {dashboard_ev.name.__root__}")
|
||||
yield dashboard_ev
|
||||
except ValidationError as err:
|
||||
logger.debug(traceback.format_exc())
|
||||
logger.warning(
|
||||
f"Unexpected exception ingesting dashboard [{dashboard}]: {err}"
|
||||
)
|
||||
|
||||
orders_view = self.metadata.get_by_name(
|
||||
entity=DashboardDataModel, fqn="sample_looker.model.orders_view"
|
||||
)
|
||||
operations_view = self.metadata.get_by_name(
|
||||
entity=DashboardDataModel, fqn="sample_looker.model.operations_view"
|
||||
)
|
||||
orders_explore = self.metadata.get_by_name(
|
||||
entity=DashboardDataModel, fqn="sample_looker.model.orders"
|
||||
)
|
||||
orders_dashboard = self.metadata.get_by_name(
|
||||
entity=Dashboard, fqn="sample_looker.orders"
|
||||
)
|
||||
|
||||
yield AddLineageRequest(
|
||||
edge=EntitiesEdge(
|
||||
fromEntity=EntityReference(
|
||||
id=orders_view.id.__root__, type="dashboardDataModel"
|
||||
),
|
||||
toEntity=EntityReference(
|
||||
id=orders_explore.id.__root__, type="dashboardDataModel"
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
yield AddLineageRequest(
|
||||
edge=EntitiesEdge(
|
||||
fromEntity=EntityReference(
|
||||
id=operations_view.id.__root__, type="dashboardDataModel"
|
||||
),
|
||||
toEntity=EntityReference(
|
||||
id=orders_explore.id.__root__, type="dashboardDataModel"
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
yield AddLineageRequest(
|
||||
edge=EntitiesEdge(
|
||||
fromEntity=EntityReference(
|
||||
id=orders_explore.id.__root__, type="dashboardDataModel"
|
||||
),
|
||||
toEntity=EntityReference(
|
||||
id=orders_dashboard.id.__root__, type="dashboard"
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
def ingest_charts(self) -> Iterable[CreateChartRequest]:
|
||||
for chart in self.charts["charts"]:
|
||||
try:
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user