diff --git a/ingestion/examples/sample_data/looker/charts.json b/ingestion/examples/sample_data/looker/charts.json new file mode 100644 index 00000000000..9652aeaa37c --- /dev/null +++ b/ingestion/examples/sample_data/looker/charts.json @@ -0,0 +1,30 @@ +[ + { + "name": "chart_1", + "displayName": "Chart 1", + "description": "This is a sample Chart for Looker", + "chartType": "Text", + "chartUrl": "http://localhost:8088/looker/explore/?form_data=%7B%22slice_id%22%3A%20114%7D" + }, + { + "name":"chart_2", + "displayName": "Chart 2", + "description": "This is a sample Chart for Looker", + "chartType": "Bar", + "chartUrl": "http://localhost:8088/looker/explore/?form_data=%7B%22slice_id%22%3A%20166%7D" + }, + { + "name": "chart_3", + "displayName": "✈️ Chart 3", + "description": "This is a sample Chart for Looker", + "chartType": "Other", + "chartUrl": "http://localhost:8088/looker/explore/?form_data=%7B%22slice_id%22%3A%2092%7D" + }, + { + "name": "chart_4", + "displayName": "Chart 4", + "description": "This is a sample Chart for Looker", + "chartType": "Histogram", + "chartUrl": "http://localhost:8088/looker/explore/?form_data=%7B%22slice_id%22%3A%20117%7D" + } + ] \ No newline at end of file diff --git a/ingestion/examples/sample_data/looker/dashboardDataModels.json b/ingestion/examples/sample_data/looker/dashboardDataModels.json new file mode 100644 index 00000000000..f19fa77b936 --- /dev/null +++ b/ingestion/examples/sample_data/looker/dashboardDataModels.json @@ -0,0 +1,163 @@ +[ + { + "name": "orders", + "displayName": "Orders", + "description": "Orders explore from Sample Data", + "dataModelType": "LookMlExplore", + "serviceType": "Looker", + "sql": "SELECT CASE\n WHEN stage_of_development = 'Pre-clinical' THEN '0. Pre-clinical'\n WHEN stage_of_development = 'Phase I' THEN '1. Phase I'\n WHEN stage_of_development = 'Phase I/II'\n or stage_of_development = 'Phase II' THEN '2. Phase II or Combined I/II'\n WHEN stage_of_development = 'Phase III' THEN '3. Phase III'\n WHEN stage_of_development = 'Authorized' THEN '4. Authorized'\n END AS clinical_stage,\n COUNT(*) AS count\nFROM covid_vaccines\nGROUP BY CASE\n WHEN stage_of_development = 'Pre-clinical' THEN '0. Pre-clinical'\n WHEN stage_of_development = 'Phase I' THEN '1. Phase I'\n WHEN stage_of_development = 'Phase I/II'\n or stage_of_development = 'Phase II' THEN '2. Phase II or Combined I/II'\n WHEN stage_of_development = 'Phase III' THEN '3. Phase III'\n WHEN stage_of_development = 'Authorized' THEN '4. Authorized'\n END\nORDER BY count DESC\nLIMIT 10000\nOFFSET 0;\n", + "columns": [ + { + "name": "0. Pre-clinical", + "dataType": "NUMERIC", + "dataTypeDisplay": "numeric", + "description": "Vaccine Candidates in phase: 'Pre-clinical'", + "ordinalPosition": 1 + }, + { + "name": "2. Phase II or Combined I/II", + "dataType": "NUMERIC", + "dataTypeDisplay": "numeric", + "description": "Vaccine Candidates in phase: 'Phase II or Combined I/II'", + "ordinalPosition": 2 + }, + { + "name": "1. Phase I", + "dataType": "NUMERIC", + "dataTypeDisplay": "numeric", + "description": "Vaccine Candidates in phase: 'Phase I'", + "ordinalPosition": 3 + }, + { + "name": "3. Phase III", + "dataType": "NUMERIC", + "dataTypeDisplay": "numeric", + "description": "Vaccine Candidates in phase: 'Phase III'", + "ordinalPosition": 4 + }, + { + "name": "4. Authorized", + "dataType": "NUMERIC", + "dataTypeDisplay": "numeric", + "description": "Vaccine Candidates in phase: 'Authorize'", + "ordinalPosition": 5 + } + ] + }, + { + "name": "operations", + "displayName": "Operations", + "description": "Operations Explore from Sample Data", + "dataModelType": "LookMlExplore", + "serviceType": "Looker", + "sql": "SELECT country_name AS country_name,\n COUNT(*) AS count,\n count(country_name) AS \"COUNT(Country_Name)\"\nFROM covid_vaccines\nGROUP BY country_name\nLIMIT 10000\nOFFSET 0;", + "columns": [ + { + "name": "country_name", + "dataType": "VARCHAR", + "dataTypeDisplay": "varchar", + "dataLength": 256, + "description": "Name of the country.", + "tags": [], + "ordinalPosition": 1 + }, + { + "name": "count", + "dataType": "NUMERIC", + "dataTypeDisplay": "numeric", + "description": "Total number of vaccine candidates per country.", + "tags": [], + "ordinalPosition": 2 + }, + { + "name": "COUNT(Country_Name)", + "dataType": "NUMERIC", + "dataTypeDisplay": "numeric", + "description": "Total number of vaccine candidates.", + "tags": [], + "ordinalPosition": 3 + } + ] + }, + { + "name": "operations_view", + "displayName": "Operations View", + "description": "Operations View from Sample Data", + "dataModelType": "LookMlView", + "serviceType": "Looker", + "sql": "SELECT country_name AS country_name,\n COUNT(*) AS count,\n count(country_name) AS \"COUNT(Country_Name)\"\nFROM covid_vaccines\nGROUP BY country_name\nLIMIT 10000\nOFFSET 0;", + "columns": [ + { + "name": "country_name", + "dataType": "VARCHAR", + "dataTypeDisplay": "varchar", + "dataLength": 256, + "description": "Name of the country.", + "tags": [], + "ordinalPosition": 1 + }, + { + "name": "count", + "dataType": "NUMERIC", + "dataTypeDisplay": "numeric", + "description": "Total number of vaccine candidates per country.", + "tags": [], + "ordinalPosition": 2 + }, + { + "name": "COUNT(Country_Name)", + "dataType": "NUMERIC", + "dataTypeDisplay": "numeric", + "description": "Total number of vaccine candidates.", + "tags": [], + "ordinalPosition": 3 + } + ] + }, + { + "name": "orders_view", + "displayName": "Orders View", + "description": "Orders View from Sample Data", + "dataModelType": "LookMlView", + "serviceType": "Looker", + "sql": "SELECT CASE\n WHEN stage_of_development = 'Pre-clinical' THEN '0. Pre-clinical'\n WHEN stage_of_development = 'Phase I' THEN '1. Phase I'\n WHEN stage_of_development = 'Phase I/II'\n or stage_of_development = 'Phase II' THEN '2. Phase II or Combined I/II'\n WHEN stage_of_development = 'Phase III' THEN '3. Phase III'\n WHEN stage_of_development = 'Authorized' THEN '4. Authorized'\n END AS clinical_stage,\n COUNT(*) AS count\nFROM covid_vaccines\nGROUP BY CASE\n WHEN stage_of_development = 'Pre-clinical' THEN '0. Pre-clinical'\n WHEN stage_of_development = 'Phase I' THEN '1. Phase I'\n WHEN stage_of_development = 'Phase I/II'\n or stage_of_development = 'Phase II' THEN '2. Phase II or Combined I/II'\n WHEN stage_of_development = 'Phase III' THEN '3. Phase III'\n WHEN stage_of_development = 'Authorized' THEN '4. Authorized'\n END\nORDER BY count DESC\nLIMIT 10000\nOFFSET 0;\n", + "columns": [ + { + "name": "0. Pre-clinical", + "dataType": "NUMERIC", + "dataTypeDisplay": "numeric", + "description": "Vaccine Candidates in phase: 'Pre-clinical'", + "ordinalPosition": 1 + }, + { + "name": "2. Phase II or Combined I/II", + "dataType": "NUMERIC", + "dataTypeDisplay": "numeric", + "description": "Vaccine Candidates in phase: 'Phase II or Combined I/II'", + "ordinalPosition": 2 + }, + { + "name": "1. Phase I", + "dataType": "NUMERIC", + "dataTypeDisplay": "numeric", + "description": "Vaccine Candidates in phase: 'Phase I'", + "ordinalPosition": 3 + }, + { + "name": "3. Phase III", + "dataType": "NUMERIC", + "dataTypeDisplay": "numeric", + "description": "Vaccine Candidates in phase: 'Phase III'", + "ordinalPosition": 4 + }, + { + "name": "4. Authorized", + "dataType": "NUMERIC", + "dataTypeDisplay": "numeric", + "description": "Vaccine Candidates in phase: 'Authorize'", + "ordinalPosition": 5 + } + ] + } +] + diff --git a/ingestion/examples/sample_data/looker/dashboards.json b/ingestion/examples/sample_data/looker/dashboards.json new file mode 100644 index 00000000000..3899b9ef6fa --- /dev/null +++ b/ingestion/examples/sample_data/looker/dashboards.json @@ -0,0 +1,17 @@ +[ + { + "name": "customers", + "displayName": "Customers dashboard", + "description": "This is a sample Dashboard for Looker", + "dashboardUrl": "http://localhost:808/looker/dashboard/1/", + "charts": ["sample_looker.chart_1", "sample_looker.chart_2"] + }, + { + "name": "orders", + "displayName": "Orders Dashboard", + "description": "This is a sample Dashboard for Looker", + "dashboardUrl": "http://localhost:808/looker/dashboard/8/", + "charts": ["sample_looker.chart_3", "sample_looker.chart_4"], + "dataModels": ["sample_looker.model.orders", "sample_looker.model.operations"] + } + ] \ No newline at end of file diff --git a/ingestion/examples/sample_data/looker/service.json b/ingestion/examples/sample_data/looker/service.json new file mode 100644 index 00000000000..54c476c8635 --- /dev/null +++ b/ingestion/examples/sample_data/looker/service.json @@ -0,0 +1,15 @@ +{ + "type": "looker", + "serviceName": "sample_looker", + "serviceConnection": { + "config": { + "type": "Looker", + "hostPort": "https://looker.com", + "clientId": "admin", + "clientSecret": "admin" + } + }, + "sourceConfig": { + "config": {} + } +} diff --git a/ingestion/src/metadata/ingestion/source/database/sample_data.py b/ingestion/src/metadata/ingestion/source/database/sample_data.py index 0bacdc134f1..ce08d05c62d 100644 --- a/ingestion/src/metadata/ingestion/source/database/sample_data.py +++ b/ingestion/src/metadata/ingestion/source/database/sample_data.py @@ -11,6 +11,7 @@ """ Sample Data source ingestion """ +# pylint: disable=too-many-lines,too-many-statements import json import random import traceback @@ -264,6 +265,37 @@ class SampleDataSource( entity=MessagingService, config=WorkflowSource(**self.kafka_service_json) ) + with open( + sample_data_folder + "/looker/service.json", + "r", + encoding=UTF_8, + ) as file: + self.looker_service = self.metadata.get_service_or_create( + entity=DashboardService, + config=WorkflowSource(**json.load(file)), + ) + + with open( + sample_data_folder + "/looker/charts.json", + "r", + encoding=UTF_8, + ) as file: + self.looker_charts = json.load(file) + + with open( + sample_data_folder + "/looker/dashboards.json", + "r", + encoding=UTF_8, + ) as file: + self.looker_dashboards = json.load(file) + + with open( + sample_data_folder + "/looker/dashboardDataModels.json", + "r", + encoding=UTF_8, + ) as file: + self.looker_models = json.load(file) + self.dashboard_service_json = json.load( open( # pylint: disable=consider-using-with sample_data_folder + "/dashboards/service.json", @@ -436,6 +468,7 @@ class SampleDataSource( yield from self.ingest_charts() yield from self.ingest_data_models() yield from self.ingest_dashboards() + yield from self.ingest_looker() yield from self.ingest_pipelines() yield from self.ingest_lineage() yield from self.ingest_pipeline_status() @@ -623,6 +656,113 @@ class SampleDataSource( self.status.scanned(f"Topic Scanned: {create_topic.name.__root__}") yield create_topic + def ingest_looker(self) -> Iterable[Entity]: + """ + Looker sample data + """ + for data_model in self.looker_models: + try: + data_model_ev = CreateDashboardDataModelRequest( + name=data_model["name"], + displayName=data_model["displayName"], + description=data_model["description"], + columns=data_model["columns"], + dataModelType=data_model["dataModelType"], + sql=data_model["sql"], + serviceType=data_model["serviceType"], + service=self.looker_service.fullyQualifiedName, + ) + self.status.scanned( + f"Data Model Scanned: {data_model_ev.name.__root__}" + ) + yield data_model_ev + except ValidationError as err: + logger.debug(traceback.format_exc()) + logger.warning( + f"Unexpected exception ingesting chart [{data_model}]: {err}" + ) + + for chart in self.looker_charts: + try: + chart_ev = CreateChartRequest( + name=chart["name"], + displayName=chart["displayName"], + description=chart["description"], + chartType=get_standard_chart_type(chart["chartType"]), + chartUrl=chart["chartUrl"], + service=self.looker_service.fullyQualifiedName, + ) + self.status.scanned(f"Chart Scanned: {chart_ev.name.__root__}") + yield chart_ev + except ValidationError as err: + logger.debug(traceback.format_exc()) + logger.warning(f"Unexpected exception ingesting chart [{chart}]: {err}") + + for dashboard in self.looker_dashboards: + try: + dashboard_ev = CreateDashboardRequest( + name=dashboard["name"], + displayName=dashboard["displayName"], + description=dashboard["description"], + dashboardUrl=dashboard["dashboardUrl"], + charts=dashboard["charts"], + dataModels=dashboard.get("dataModels", None), + service=self.looker_service.fullyQualifiedName, + ) + self.status.scanned(f"Dashboard Scanned: {dashboard_ev.name.__root__}") + yield dashboard_ev + except ValidationError as err: + logger.debug(traceback.format_exc()) + logger.warning( + f"Unexpected exception ingesting dashboard [{dashboard}]: {err}" + ) + + orders_view = self.metadata.get_by_name( + entity=DashboardDataModel, fqn="sample_looker.model.orders_view" + ) + operations_view = self.metadata.get_by_name( + entity=DashboardDataModel, fqn="sample_looker.model.operations_view" + ) + orders_explore = self.metadata.get_by_name( + entity=DashboardDataModel, fqn="sample_looker.model.orders" + ) + orders_dashboard = self.metadata.get_by_name( + entity=Dashboard, fqn="sample_looker.orders" + ) + + yield AddLineageRequest( + edge=EntitiesEdge( + fromEntity=EntityReference( + id=orders_view.id.__root__, type="dashboardDataModel" + ), + toEntity=EntityReference( + id=orders_explore.id.__root__, type="dashboardDataModel" + ), + ) + ) + + yield AddLineageRequest( + edge=EntitiesEdge( + fromEntity=EntityReference( + id=operations_view.id.__root__, type="dashboardDataModel" + ), + toEntity=EntityReference( + id=orders_explore.id.__root__, type="dashboardDataModel" + ), + ) + ) + + yield AddLineageRequest( + edge=EntitiesEdge( + fromEntity=EntityReference( + id=orders_explore.id.__root__, type="dashboardDataModel" + ), + toEntity=EntityReference( + id=orders_dashboard.id.__root__, type="dashboard" + ), + ) + ) + def ingest_charts(self) -> Iterable[CreateChartRequest]: for chart in self.charts["charts"]: try: