From bea38d720016916214d1311e072acb46a02b13d9 Mon Sep 17 00:00:00 2001 From: Nahuel Date: Tue, 28 Mar 2023 17:07:38 +0200 Subject: [PATCH] Fix#10584: Add Data Model as an entity (#10636) * Add Data Model as entity * Add sample_data + update dashboard resource and repository with data models * Fix Java style * Addess PR comments * Update bootstrap/sql/com.mysql.cj.jdbc.Driver/v009__create_db_connection_info.sql * Pylint error * Address PR comments * Address PR comments * Address PR comments * Minor change * Fix error in sample_data * Fix failing test * Add missing resource and event sub descriptors --- .../v009__create_db_connection_info.sql | 13 +- .../v009__create_db_connection_info.sql | 12 + .../sample_data/dashboards/charts.json | 24 +- .../dashboards/dashboardDataModels.json | 101 ++++ .../sample_data/dashboards/dashboards.json | 25 +- .../examples/sample_data/lineage/lineage.json | 23 +- .../src/metadata/ingestion/ometa/ometa_api.py | 12 + .../ingestion/source/database/sample_data.py | 48 +- .../java/org/openmetadata/service/Entity.java | 2 + .../service/jdbi3/CollectionDAO.java | 21 + .../service/jdbi3/ColumnUtil.java | 27 + .../jdbi3/DashboardDataModelRepository.java | 174 +++++++ .../service/jdbi3/DashboardRepository.java | 77 +-- .../service/jdbi3/LineageRepository.java | 6 +- .../service/jdbi3/QueryRepository.java | 1 - .../service/jdbi3/TableRepository.java | 53 +- .../dashboards/DashboardResource.java | 6 +- .../resources/databases/DatabaseUtil.java | 7 +- .../resources/databases/TableResource.java | 2 +- .../DashboardDataModelResource.java | 460 ++++++++++++++++++ .../openmetadata/service/util/EntityUtil.java | 19 + .../json/data/EventSubResourceDescriptor.json | 10 + .../json/data/ResourceDescriptors.json | 29 ++ .../DashboardDataModelResourceTest.java | 147 ++++++ .../DashboardServiceResourceTest.java | 8 +- .../json/schema/api/data/createDashboard.json | 8 + .../api/data/createDashboardDataModel.json | 61 +++ .../api/services/createDashboardService.json | 1 - .../json/schema/entity/data/dashboard.json | 10 +- .../entity/data/dashboardDataModel.json | 131 +++++ 30 files changed, 1381 insertions(+), 137 deletions(-) create mode 100644 ingestion/examples/sample_data/dashboards/dashboardDataModels.json create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DashboardDataModelRepository.java create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/resources/datamodels/DashboardDataModelResource.java create mode 100644 openmetadata-service/src/test/java/org/openmetadata/service/resources/datamodels/DashboardDataModelResourceTest.java create mode 100644 openmetadata-spec/src/main/resources/json/schema/api/data/createDashboardDataModel.json create mode 100644 openmetadata-spec/src/main/resources/json/schema/entity/data/dashboardDataModel.json diff --git a/bootstrap/sql/com.mysql.cj.jdbc.Driver/v009__create_db_connection_info.sql b/bootstrap/sql/com.mysql.cj.jdbc.Driver/v009__create_db_connection_info.sql index 4d829f7a0e2..051dfc92c48 100644 --- a/bootstrap/sql/com.mysql.cj.jdbc.Driver/v009__create_db_connection_info.sql +++ b/bootstrap/sql/com.mysql.cj.jdbc.Driver/v009__create_db_connection_info.sql @@ -122,4 +122,15 @@ ALTER TABLE user_tokens MODIFY COLUMN expiryDate BIGINT UNSIGNED GENERATED ALWAY DELETE FROM alert_entity; drop table alert_action_def; -ALTER TABLE alert_entity RENAME TO event_subscription_entity; \ No newline at end of file +ALTER TABLE alert_entity RENAME TO event_subscription_entity; +-- create data model table +CREATE TABLE IF NOT EXISTS dashboard_data_model_entity ( + id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL, + fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL, + json JSON NOT NULL, + updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL, + updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL, + deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'), + PRIMARY KEY (id), + UNIQUE (fullyQualifiedName) +); diff --git a/bootstrap/sql/org.postgresql.Driver/v009__create_db_connection_info.sql b/bootstrap/sql/org.postgresql.Driver/v009__create_db_connection_info.sql index 433a5a0324a..36dcd5065a7 100644 --- a/bootstrap/sql/org.postgresql.Driver/v009__create_db_connection_info.sql +++ b/bootstrap/sql/org.postgresql.Driver/v009__create_db_connection_info.sql @@ -126,3 +126,15 @@ DELETE FROM alert_entity; drop table alert_action_def; ALTER TABLE alert_entity RENAME TO event_subscription_entity; + +-- create data model table +CREATE TABLE IF NOT EXISTS dashboard_data_model_entity ( + id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL, + json JSONB NOT NULL, + updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL, + updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL, + deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED, + fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL, + PRIMARY KEY (id), + UNIQUE (fullyQualifiedName) +); diff --git a/ingestion/examples/sample_data/dashboards/charts.json b/ingestion/examples/sample_data/dashboards/charts.json index b0cc5fc50e5..8c7b8b9c41d 100644 --- a/ingestion/examples/sample_data/dashboards/charts.json +++ b/ingestion/examples/sample_data/dashboards/charts.json @@ -4,7 +4,7 @@ "id": "2841fdb1-e378-4a2c-94f8-27c9f5d6ef8e", "name": "114", "displayName": "# of Games That Hit 100k in Sales By Release Year", - "fullyQualifiedName": "local_superset.101", + "fullyQualifiedName": "sample_superset.101", "description": "", "chartId": "114", "chartType": "Area", @@ -15,7 +15,7 @@ "id": "3bcba490-9e5c-4946-a0e3-41e8ff8f4aa4", "name":"166", "displayName": "% Rural", - "fullyQualifiedName": "local_superset.110", + "fullyQualifiedName": "sample_superset.110", "description": "", "chartId": "166", "chartType": "Other", @@ -26,7 +26,7 @@ "id": "22b95748-4a7b-48ad-859e-cf7c66a7f343", "name": "92", "displayName": "✈️ Relocation ability", - "fullyQualifiedName": "local_superset.92", + "fullyQualifiedName": "sample_superset.92", "description": "", "chartId": "92", "chartType": "Other", @@ -37,7 +37,7 @@ "id": "62b31dcc-4619-46a0-99b1-0fa7cd6f93da", "name": "117", "displayName": "Age distribution of respondents", - "fullyQualifiedName": "local_superset.11", + "fullyQualifiedName": "sample_superset.11", "description": "", "chartId": "117", "chartType": "Histogram", @@ -47,7 +47,7 @@ { "id": "57944482-e187-439a-aaae-0e8aabd2f455", "displayName": "Arcs", - "fullyQualifiedName": "local_superset.197", + "fullyQualifiedName": "sample_superset.197", "description": "", "name": "197", "chartType": "Other", @@ -57,7 +57,7 @@ { "id": "d88e2056-c74a-410d-829e-eb31b040c132", "displayName": "Are you an ethnic minority in your city?", - "fullyQualifiedName": "local_superset.127", + "fullyQualifiedName": "sample_superset.127", "description": "", "name": "127", "chartType": "Other", @@ -67,7 +67,7 @@ { "id": "c1d3e156-4628-414e-8d6e-a6bdd486128f", "displayName": "Average and Sum Trends", - "fullyQualifiedName": "local_superset.183", + "fullyQualifiedName": "sample_superset.183", "description": "", "name": "183", "chartType": "Line", @@ -77,7 +77,7 @@ { "id": "bfc57519-8cef-47e6-a423-375d5b89a6a4", "displayName": "Birth in France by department in 2016", - "fullyQualifiedName": "local_superset.Birth in France by department in 2016", + "fullyQualifiedName": "sample_superset.Birth in France by department in 2016", "description": "", "name": "161", "chartType": "Other", @@ -87,7 +87,7 @@ { "id": "bf2eeac4-7226-46c6-bbef-918569c137a0", "displayName": "Box plot", - "fullyQualifiedName": "local_superset.170", + "fullyQualifiedName": "sample_superset.170", "description": "", "name": "170", "chartType": "Bar", @@ -97,7 +97,7 @@ { "id": "167fd63b-42f1-4d7e-a37d-893fd8173b44", "displayName": "Boy Name Cloud", - "fullyQualifiedName": "local_superset.180", + "fullyQualifiedName": "sample_superset.180", "description": "", "name": "180", "chartType": "Other", @@ -107,7 +107,7 @@ { "id": "8474e579-4eff-492b-8685-70ec9aa99f5f", "displayName": "ETA Predictions Accuracy", - "fullyQualifiedName": "local_superset.210", + "fullyQualifiedName": "sample_superset.210", "description": "", "name": "210", "chartType": "Line", @@ -117,7 +117,7 @@ { "id": "12345e567-4eff-492b-8685-69ec9bb88g4g", "displayName": "Sales Predictions Accuracy", - "fullyQualifiedName": "local_superset.211", + "fullyQualifiedName": "sample_superset.211", "description": "", "name": "211", "chartType": "Line", diff --git a/ingestion/examples/sample_data/dashboards/dashboardDataModels.json b/ingestion/examples/sample_data/dashboards/dashboardDataModels.json new file mode 100644 index 00000000000..650d1a07c85 --- /dev/null +++ b/ingestion/examples/sample_data/dashboards/dashboardDataModels.json @@ -0,0 +1,101 @@ +{ + "datamodels": [ + { + "id": "e093dd27-390e-4360-8efd-e4d63ec167a9", + "name": "103", + "displayName": "Vaccine Candidates per Phase", + "fullyQualifiedName": "sample_superset.model.103", + "description": "Data of Vaccine Candidates per Phase", + "version": 0.1, + "updatedAt": 1638354087591, + "dataModelType": "SupersetDataModel", + "serviceType": "Superset", + "sql": "SELECT CASE\n WHEN stage_of_development = 'Pre-clinical' THEN '0. Pre-clinical'\n WHEN stage_of_development = 'Phase I' THEN '1. Phase I'\n WHEN stage_of_development = 'Phase I/II'\n or stage_of_development = 'Phase II' THEN '2. Phase II or Combined I/II'\n WHEN stage_of_development = 'Phase III' THEN '3. Phase III'\n WHEN stage_of_development = 'Authorized' THEN '4. Authorized'\n END AS clinical_stage,\n COUNT(*) AS count\nFROM covid_vaccines\nGROUP BY CASE\n WHEN stage_of_development = 'Pre-clinical' THEN '0. Pre-clinical'\n WHEN stage_of_development = 'Phase I' THEN '1. Phase I'\n WHEN stage_of_development = 'Phase I/II'\n or stage_of_development = 'Phase II' THEN '2. Phase II or Combined I/II'\n WHEN stage_of_development = 'Phase III' THEN '3. Phase III'\n WHEN stage_of_development = 'Authorized' THEN '4. Authorized'\n END\nORDER BY count DESC\nLIMIT 10000\nOFFSET 0;\n", + "columns": [ + { + "name": "0. Pre-clinical", + "dataType": "NUMERIC", + "dataTypeDisplay": "numeric", + "description": "Vaccine Candidates in phase: 'Pre-clinical'", + "tags": [], + "ordinalPosition": 1 + }, + { + "name": "2. Phase II or Combined I/II", + "dataType": "NUMERIC", + "dataTypeDisplay": "numeric", + "description": "Vaccine Candidates in phase: 'Phase II or Combined I/II'", + "tags": [], + "ordinalPosition": 2 + }, + { + "name": "1. Phase I", + "dataType": "NUMERIC", + "dataTypeDisplay": "numeric", + "description": "Vaccine Candidates in phase: 'Phase I'", + "tags": [], + "ordinalPosition": 3 + }, + { + "name": "3. Phase III", + "dataType": "NUMERIC", + "dataTypeDisplay": "numeric", + "description": "Vaccine Candidates in phase: 'Phase III'", + "tags": [], + "ordinalPosition": 4 + }, + { + "name": "4. Authorized", + "dataType": "NUMERIC", + "dataTypeDisplay": "numeric", + "description": "Vaccine Candidates in phase: 'Authorize'", + "tags": [], + "ordinalPosition": 5 + } + ], + "tags": [], + "followers": [] + }, + { + "id": "5bab1ca3-7a22-4f21-9b34-e2c44dee1af6", + "name": "73", + "displayName": "Vaccine Candidates per Country", + "fullyQualifiedName": "sample_superset.model.73", + "description": "Data of Vaccine Candidates per Country", + "version": 0.1, + "updatedAt": 1638354087591, + "dataModelType": "SupersetDataModel", + "serviceType": "Superset", + "sql": "SELECT country_name AS country_name,\n COUNT(*) AS count,\n count(country_name) AS \"COUNT(Country_Name)\"\nFROM covid_vaccines\nGROUP BY country_name\nLIMIT 10000\nOFFSET 0;", + "columns": [ + { + "name": "country_name", + "dataType": "VARCHAR", + "dataTypeDisplay": "varchar", + "dataLength": 256, + "description": "Name of the country.", + "tags": [], + "ordinalPosition": 1 + }, + { + "name": "count", + "dataType": "NUMERIC", + "dataTypeDisplay": "numeric", + "description": "Total number of vaccine candidates per country.", + "tags": [], + "ordinalPosition": 2 + }, + { + "name": "COUNT(Country_Name)", + "dataType": "NUMERIC", + "dataTypeDisplay": "numeric", + "description": "Total number of vaccine candidates.", + "tags": [], + "ordinalPosition": 3 + } + ], + "tags": [], + "followers": [] + } + ] +} diff --git a/ingestion/examples/sample_data/dashboards/dashboards.json b/ingestion/examples/sample_data/dashboards/dashboards.json index 38976f0a8e7..64c956d8dc1 100644 --- a/ingestion/examples/sample_data/dashboards/dashboards.json +++ b/ingestion/examples/sample_data/dashboards/dashboards.json @@ -4,7 +4,7 @@ "id": "d4dc7baf-1b17-45f8-acd5-a15b78cc7c5f", "name": "8", "displayName": "Orders dashboard", - "fullyQualifiedName": "local_superset.8", + "fullyQualifiedName": "sample_superset.8", "description": "", "dashboardUrl": "http://localhost:808/superset/dashboard/1/", "charts": ["sample_superset.183", "sample_superset.170", "sample_superset.197"], @@ -14,17 +14,18 @@ "id": "063cd787-8630-4809-9702-34d3992c7248", "name": "9", "displayName": "COVID Vaccine Dashboard", - "fullyQualifiedName": "local_superset.9", + "fullyQualifiedName": "sample_superset.9", "description": "", "dashboardUrl": "http://localhost:808/superset/dashboard/8/", "charts": ["sample_superset.117", "sample_superset.197"], + "dataModels": ["sample_superset.model.103", "sample_superset.model.73"], "href": "http://localhost:8585/api/v1/dashboards/063cd787-8630-4809-9702-34d3992c7248" }, { "id": "df6c698e-066a-4440-be0a-121025573b73", "name": "10", "displayName": "deck.gl Demo", - "fullyQualifiedName": "local_superset.10", + "fullyQualifiedName": "sample_superset.10", "description": "", "dashboardUrl": "http://localhost:808/superset/dashboard/deck/", "charts": ["sample_superset.127", "sample_superset.166", "sample_superset.114"], @@ -34,7 +35,7 @@ "id": "98b38a49-b5c6-431b-b61f-690e39f8ead2", "name": "11", "displayName": "FCC New Coder Survey 2018", - "fullyQualifiedName": "local_superset.11", + "fullyQualifiedName": "sample_superset.11", "description": "", "dashboardUrl": "http://localhost:808/superset/dashboard/7/", "charts": ["sample_superset.183", "sample_superset.197", "sample_superset.170", "sample_superset.180"], @@ -44,7 +45,7 @@ "id": "dffcf9b2-4f43-4881-a5f5-10109655bf50", "name": "12", "displayName": "Misc Charts", - "fullyQualifiedName": "local_superset.12", + "fullyQualifiedName": "sample_superset.12", "description": "", "dashboardUrl": "http://localhost:808/superset/dashboard/misc_charts/", "charts": ["sample_superset.127", "sample_superset.197"], @@ -54,7 +55,7 @@ "id": "2583737d-6236-421e-ba0f-cd0b79adb216", "name": "31", "displayName": "Sales Dashboard", - "fullyQualifiedName": "local_superset.31", + "fullyQualifiedName": "sample_superset.31", "description": "", "dashboardUrl": "http://localhost:808/superset/dashboard/6/", "charts": ["sample_superset.92", "sample_superset.117", "sample_superset.166"], @@ -64,7 +65,7 @@ "id": "6bf9bfcb-4e80-4af0-9f0c-13e47bbc27a2", "name": "33", "displayName": "Slack Dashboard", - "fullyQualifiedName": "local_superset.33", + "fullyQualifiedName": "sample_superset.33", "description": "", "dashboardUrl": "http://localhost:808/superset/dashboard/10/", "charts": ["sample_superset.114", "sample_superset.92", "sample_superset.127"], @@ -74,7 +75,7 @@ "id": "1f02caf2-c5e5-442d-bda3-b8ce3e757b45", "name": "34", "displayName": "Unicode Test", - "fullyQualifiedName": "local_superset.34", + "fullyQualifiedName": "sample_superset.34", "description": "", "dashboardUrl": "http://localhost:808/superset/dashboard/unicode-test/", "charts": ["sample_superset.161", "sample_superset.170", "sample_superset.180"], @@ -84,7 +85,7 @@ "id": "a3ace318-ee37-4da1-974a-62eddbd77d20", "name": "45", "displayName": "USA Births Names", - "fullyQualifiedName": "local_superset.45", + "fullyQualifiedName": "sample_superset.45", "description": "", "dashboardUrl": "http://localhost:808/superset/dashboard/births/", "charts": ["sample_superset.180"], @@ -94,7 +95,7 @@ "id": "e6e21717-1164-403f-8807-d12be277aec6", "name": "51", "displayName": "Video Game Sales", - "fullyQualifiedName": "local_superset.51", + "fullyQualifiedName": "sample_superset.51", "description": "", "dashboardUrl": "http://localhost:808/superset/dashboard/11/", "charts": ["sample_superset.127", "sample_superset.183"], @@ -104,7 +105,7 @@ "id": "d2b0af00-f419-4905-bb43-036697ce53a5", "name": "eta_predictions_performance", "displayName": "ETA Predictions Performance", - "fullyQualifiedName": "local_superset.eta_predictions_performance", + "fullyQualifiedName": "sample_superset.eta_predictions_performance", "description": "", "dashboardUrl": "http://localhost:808/superset/dashboard/eta_predictions_performance/", "charts": ["sample_superset.210"], @@ -114,7 +115,7 @@ "id": "f5a1af99-f123-7845-cc12-0312347ce53a", "name": "forecast_sales_performance", "displayName": "ETA Predictions Performance", - "fullyQualifiedName": "local_superset.forecast_sales_performance", + "fullyQualifiedName": "sample_superset.forecast_sales_performance", "description": "", "dashboardUrl": "http://localhost:808/superset/dashboard/forecast_sales_performance/", "charts": ["sample_superset.211"], diff --git a/ingestion/examples/sample_data/lineage/lineage.json b/ingestion/examples/sample_data/lineage/lineage.json index 4ed3468fa6e..d024d1f1ebe 100644 --- a/ingestion/examples/sample_data/lineage/lineage.json +++ b/ingestion/examples/sample_data/lineage/lineage.json @@ -58,6 +58,27 @@ }, "edge_meta": { "fqn": "sample_airflow.dim_product_etl", "type": "pipeline" }, "sql_query": "create ecommerce_db.shopify.\"dim.product.variant\" as select * from sample_data.ecommerce_db.shopify.raw_customer" - + }, + { + "from": { + "fqn": "sample_superset.model.103", + "type": "dashboardDataModel" + }, + "to": { + "fqn": "sample_superset.9", + "type": "dashboard" + }, + "edge_meta": { "fqn": "", "type": "" } + }, + { + "from": { + "fqn": "sample_superset.model.73", + "type": "dashboardDataModel" + }, + "to": { + "fqn": "sample_superset.9", + "type": "dashboard" + }, + "edge_meta": { "fqn": "", "type": "" } } ] diff --git a/ingestion/src/metadata/ingestion/ometa/ometa_api.py b/ingestion/src/metadata/ingestion/ometa/ometa_api.py index 8e7e9e0906f..64c7e18854a 100644 --- a/ingestion/src/metadata/ingestion/ometa/ometa_api.py +++ b/ingestion/src/metadata/ingestion/ometa/ometa_api.py @@ -39,6 +39,7 @@ from metadata.generated.schema.entity.classification.tag import Tag from metadata.generated.schema.entity.data.chart import Chart from metadata.generated.schema.entity.data.container import Container from metadata.generated.schema.entity.data.dashboard import Dashboard +from metadata.generated.schema.entity.data.dashboardDataModel import DashboardDataModel from metadata.generated.schema.entity.data.database import Database from metadata.generated.schema.entity.data.databaseSchema import DatabaseSchema from metadata.generated.schema.entity.data.glossary import Glossary @@ -249,6 +250,16 @@ class OpenMetadata( ): return "/charts" + if issubclass( + entity, + get_args( + Union[ + DashboardDataModel, self.get_create_entity_type(DashboardDataModel) + ] + ), + ): + return "/dashboard/datamodels" + if issubclass( entity, get_args(Union[Dashboard, self.get_create_entity_type(Dashboard)]) ): @@ -529,6 +540,7 @@ class OpenMetadata( file_name = ( class_name.lower() .replace("glossaryterm", "glossaryTerm") + .replace("dashboarddatamodel", "dashboardDataModel") .replace("testsuite", "testSuite") .replace("testdefinition", "testDefinition") .replace("testcase", "testCase") diff --git a/ingestion/src/metadata/ingestion/source/database/sample_data.py b/ingestion/src/metadata/ingestion/source/database/sample_data.py index 6162b15479e..e2e52f34894 100644 --- a/ingestion/src/metadata/ingestion/source/database/sample_data.py +++ b/ingestion/src/metadata/ingestion/source/database/sample_data.py @@ -23,6 +23,9 @@ from pydantic import ValidationError from metadata.generated.schema.api.data.createChart import CreateChartRequest from metadata.generated.schema.api.data.createContainer import CreateContainerRequest from metadata.generated.schema.api.data.createDashboard import CreateDashboardRequest +from metadata.generated.schema.api.data.createDashboardDataModel import ( + CreateDashboardDataModelRequest, +) from metadata.generated.schema.api.data.createDatabase import CreateDatabaseRequest from metadata.generated.schema.api.data.createDatabaseSchema import ( CreateDatabaseSchemaRequest, @@ -46,6 +49,7 @@ from metadata.generated.schema.api.tests.createTestCase import CreateTestCaseReq from metadata.generated.schema.api.tests.createTestSuite import CreateTestSuiteRequest from metadata.generated.schema.entity.data.container import Container from metadata.generated.schema.entity.data.dashboard import Dashboard +from metadata.generated.schema.entity.data.dashboardDataModel import DashboardDataModel from metadata.generated.schema.entity.data.database import Database from metadata.generated.schema.entity.data.databaseSchema import DatabaseSchema from metadata.generated.schema.entity.data.location import Location @@ -158,6 +162,10 @@ def get_lineage_entity_ref(edge, metadata_config) -> EntityReference: dashboard = metadata.get_by_name(entity=Dashboard, fqn=edge_fqn) if dashboard: return EntityReference(id=dashboard.id, type="dashboard") + if edge["type"] == "dashboardDataModel": + data_model = metadata.get_by_name(entity=DashboardDataModel, fqn=edge_fqn) + if data_model: + return EntityReference(id=data_model.id, type="dashboardDataModel") return None @@ -328,6 +336,13 @@ class SampleDataSource( encoding="utf-8", ) ) + self.data_models = json.load( + open( # pylint: disable=consider-using-with + sample_data_folder + "/dashboards/dashboardDataModels.json", + "r", + encoding="utf-8", + ) + ) self.dashboards = json.load( open( # pylint: disable=consider-using-with sample_data_folder + "/dashboards/dashboards.json", @@ -478,6 +493,7 @@ class SampleDataSource( yield from self.ingest_tables() yield from self.ingest_topics() yield from self.ingest_charts() + yield from self.ingest_data_models() yield from self.ingest_dashboards() yield from self.ingest_pipelines() yield from self.ingest_lineage() @@ -716,7 +732,7 @@ class SampleDataSource( name=chart["name"], displayName=chart["displayName"], description=chart["description"], - chartType=get_standard_chart_type(chart["chartType"]).value, + chartType=get_standard_chart_type(chart["chartType"]), chartUrl=chart["chartUrl"], service=self.dashboard_service.fullyQualifiedName, ) @@ -726,6 +742,29 @@ class SampleDataSource( logger.debug(traceback.format_exc()) logger.warning(f"Unexpected exception ingesting chart [{chart}]: {err}") + def ingest_data_models(self) -> Iterable[CreateDashboardDataModelRequest]: + for data_model in self.data_models["datamodels"]: + try: + data_model_ev = CreateDashboardDataModelRequest( + name=data_model["name"], + displayName=data_model["displayName"], + description=data_model["description"], + columns=data_model["columns"], + dataModelType=data_model["dataModelType"], + sql=data_model["sql"], + serviceType=data_model["serviceType"], + service=self.dashboard_service.fullyQualifiedName, + ) + self.status.scanned( + f"Data Model Scanned: {data_model_ev.name.__root__}" + ) + yield data_model_ev + except ValidationError as err: + logger.debug(traceback.format_exc()) + logger.warning( + f"Unexpected exception ingesting chart [{data_model}]: {err}" + ) + def ingest_dashboards(self) -> Iterable[CreateDashboardRequest]: for dashboard in self.dashboards["dashboards"]: dashboard_ev = CreateDashboardRequest( @@ -734,6 +773,7 @@ class SampleDataSource( description=dashboard["description"], dashboardUrl=dashboard["dashboardUrl"], charts=dashboard["charts"], + dataModels=dashboard.get("dataModels", None), service=self.dashboard_service.fullyQualifiedName, ) self.status.scanned(f"Dashboard Scanned: {dashboard_ev.name.__root__}") @@ -758,8 +798,10 @@ class SampleDataSource( edge_entity_ref = get_lineage_entity_ref( edge["edge_meta"], self.metadata_config ) - lineage_details = LineageDetails( - pipeline=edge_entity_ref, sqlQuery=edge.get("sql_query") + lineage_details = ( + LineageDetails(pipeline=edge_entity_ref, sqlQuery=edge.get("sql_query")) + if edge_entity_ref + else None ) lineage = AddLineageRequest( edge=EntitiesEdge( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java b/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java index 1196f147e27..f91a9e2e07b 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java @@ -115,6 +115,8 @@ public final class Entity { public static final String WEB_ANALYTIC_EVENT = "webAnalyticEvent"; public static final String DATA_INSIGHT_CHART = "dataInsightChart"; + public static final String DASHBOARD_DATA_MODEL = "dashboardDataModel"; + // // Policy entity // diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java index 47fa91ff8fa..04711e1b51a 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java @@ -65,6 +65,7 @@ import org.openmetadata.schema.entity.classification.Tag; import org.openmetadata.schema.entity.data.Chart; import org.openmetadata.schema.entity.data.Container; import org.openmetadata.schema.entity.data.Dashboard; +import org.openmetadata.schema.entity.data.DashboardDataModel; import org.openmetadata.schema.entity.data.Database; import org.openmetadata.schema.entity.data.DatabaseSchema; import org.openmetadata.schema.entity.data.Glossary; @@ -270,6 +271,9 @@ public interface CollectionDAO { @CreateSqlObject WorkflowDAO workflowDAO(); + @CreateSqlObject + DataModelDAO dataModelDAO(); + interface DashboardDAO extends EntityDAO { @Override default String getTableName() { @@ -3666,4 +3670,21 @@ public interface CollectionDAO { @Define("nameColumn") String nameColumn, @Define("sqlCondition") String sqlCondition); } + + interface DataModelDAO extends EntityDAO { + @Override + default String getTableName() { + return "dashboard_data_model_entity"; + } + + @Override + default Class getEntityClass() { + return DashboardDataModel.class; + } + + @Override + default String getNameColumn() { + return "fullyQualifiedName"; + } + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ColumnUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ColumnUtil.java index 44fcf1f1320..1669bf6f671 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ColumnUtil.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ColumnUtil.java @@ -5,6 +5,8 @@ import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; import java.util.ArrayList; import java.util.List; import org.openmetadata.schema.type.Column; +import org.openmetadata.service.exception.CatalogExceptionMessage; +import org.openmetadata.service.util.FullyQualifiedName; public final class ColumnUtil { private ColumnUtil() {} @@ -35,4 +37,29 @@ public final class ColumnUtil { .withOrdinalPosition(column.getOrdinalPosition()) .withChildren(children); } + + public static void setColumnFQN(String parentFQN, List columns) { + columns.forEach( + c -> { + String columnFqn = FullyQualifiedName.add(parentFQN, c.getName()); + c.setFullyQualifiedName(columnFqn); + if (c.getChildren() != null) { + setColumnFQN(columnFqn, c.getChildren()); + } + }); + } + + // Validate if a given column exists in the table + public static void validateColumnFQN(List columns, String columnFQN) { + boolean validColumn = false; + for (Column column : columns) { + if (column.getFullyQualifiedName().equals(columnFQN)) { + validColumn = true; + break; + } + } + if (!validColumn) { + throw new IllegalArgumentException(CatalogExceptionMessage.invalidColumnFQN(columnFQN)); + } + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DashboardDataModelRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DashboardDataModelRepository.java new file mode 100644 index 00000000000..4de5ddcc84d --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DashboardDataModelRepository.java @@ -0,0 +1,174 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.service.jdbi3; + +import static org.openmetadata.common.utils.CommonUtil.listOrEmpty; +import static org.openmetadata.service.Entity.FIELD_FOLLOWERS; +import static org.openmetadata.service.Entity.FIELD_TAGS; + +import com.fasterxml.jackson.core.JsonProcessingException; +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.openmetadata.schema.entity.data.DashboardDataModel; +import org.openmetadata.schema.entity.services.DashboardService; +import org.openmetadata.schema.type.Column; +import org.openmetadata.schema.type.EntityReference; +import org.openmetadata.schema.type.Include; +import org.openmetadata.schema.type.Relationship; +import org.openmetadata.schema.type.TagLabel; +import org.openmetadata.service.Entity; +import org.openmetadata.service.resources.databases.DatabaseUtil; +import org.openmetadata.service.resources.datamodels.DashboardDataModelResource; +import org.openmetadata.service.util.EntityUtil; +import org.openmetadata.service.util.EntityUtil.Fields; +import org.openmetadata.service.util.FullyQualifiedName; + +@Slf4j +public class DashboardDataModelRepository extends EntityRepository { + + private static final String DATA_MODELS_FIELD = "dataModels"; + + private static final String DATA_MODEL_UPDATE_FIELDS = "owner,tags,followers"; + private static final String DATA_MODEL_PATCH_FIELDS = "owner,tags,followers"; + + public DashboardDataModelRepository(CollectionDAO dao) { + super( + DashboardDataModelResource.COLLECTION_PATH, + Entity.DASHBOARD_DATA_MODEL, + DashboardDataModel.class, + dao.dataModelDAO(), + dao, + DATA_MODEL_PATCH_FIELDS, + DATA_MODEL_UPDATE_FIELDS); + } + + @Override + public void setFullyQualifiedName(DashboardDataModel dashboardDataModel) { + dashboardDataModel.setFullyQualifiedName( + FullyQualifiedName.add(dashboardDataModel.getService().getName() + ".model", dashboardDataModel.getName())); + ColumnUtil.setColumnFQN(dashboardDataModel.getFullyQualifiedName(), dashboardDataModel.getColumns()); + } + + @Override + public void prepare(DashboardDataModel dashboardDataModel) throws IOException { + DashboardService dashboardService = Entity.getEntity(dashboardDataModel.getService(), "", Include.ALL); + dashboardDataModel.setService(dashboardService.getEntityReference()); + dashboardDataModel.setServiceType(dashboardService.getServiceType()); + } + + @Override + public void storeEntity(DashboardDataModel dashboardDataModel, boolean update) throws JsonProcessingException { + // Relationships and fields such as href are derived and not stored as part of json + EntityReference owner = dashboardDataModel.getOwner(); + List tags = dashboardDataModel.getTags(); + List dataModels = dashboardDataModel.getDataModels(); + EntityReference service = dashboardDataModel.getService(); + + // Don't store owner, database, href and tags as JSON. Build it on the fly based on relationships + dashboardDataModel.withOwner(null).withService(null).withHref(null).withTags(null).withDataModels(null); + + store(dashboardDataModel, update); + + // Restore the relationships + dashboardDataModel.withOwner(owner).withService(service).withTags(tags).withDataModels(dataModels); + } + + @Override + @SneakyThrows + public void storeRelationships(DashboardDataModel dashboardDataModel) { + EntityReference service = dashboardDataModel.getService(); + addRelationship( + service.getId(), + dashboardDataModel.getId(), + service.getType(), + Entity.DASHBOARD_DATA_MODEL, + Relationship.CONTAINS); + storeOwner(dashboardDataModel, dashboardDataModel.getOwner()); + applyTags(dashboardDataModel); + } + + @Override + public DashboardDataModel setFields(DashboardDataModel dashboardDataModel, Fields fields) throws IOException { + getColumnTags(fields.contains(FIELD_TAGS), dashboardDataModel.getColumns()); + return dashboardDataModel + .withService(getContainer(dashboardDataModel.getId())) + .withFollowers(fields.contains(FIELD_FOLLOWERS) ? getFollowers(dashboardDataModel) : null) + .withTags(fields.contains(FIELD_TAGS) ? getTags(dashboardDataModel.getFullyQualifiedName()) : null) + .withDataModels(fields.contains(DATA_MODELS_FIELD) ? getDataModels(dashboardDataModel) : null); + } + + @Override + public void restorePatchAttributes(DashboardDataModel original, DashboardDataModel updated) { + // Patch can't make changes to following fields. Ignore the changes + updated + .withFullyQualifiedName(original.getFullyQualifiedName()) + .withName(original.getName()) + .withService(original.getService()) + .withId(original.getId()); + } + + protected List getDataModels(DashboardDataModel dashboardDataModel) throws IOException { + if (dashboardDataModel == null) { + return Collections.emptyList(); + } + List tableIds = + findTo(dashboardDataModel.getId(), entityType, Relationship.USES, Entity.DASHBOARD_DATA_MODEL); + return EntityUtil.populateEntityReferences(tableIds, Entity.TABLE); + } + + private void getColumnTags(boolean setTags, List columns) { + for (Column c : listOrEmpty(columns)) { + c.setTags(setTags ? getTags(c.getFullyQualifiedName()) : null); + getColumnTags(setTags, c.getChildren()); + } + } + + private void applyTags(List columns) { + // Add column level tags by adding tag to column relationship + for (Column column : columns) { + applyTags(column.getTags(), column.getFullyQualifiedName()); + if (column.getChildren() != null) { + applyTags(column.getChildren()); + } + } + } + + @Override + public void applyTags(DashboardDataModel dashboardDataModel) { + // Add table level tags by adding tag to table relationship + super.applyTags(dashboardDataModel); + applyTags(dashboardDataModel.getColumns()); + } + + @Override + public EntityUpdater getUpdater(DashboardDataModel original, DashboardDataModel updated, Operation operation) { + return new DataModelUpdater(original, updated, operation); + } + + public class DataModelUpdater extends ColumnEntityUpdater { + + public DataModelUpdater(DashboardDataModel original, DashboardDataModel updated, Operation operation) { + super(original, updated, operation); + } + + @Override + public void entitySpecificUpdate() throws IOException { + DatabaseUtil.validateColumns(original.getColumns()); + updateColumns("columns", original.getColumns(), updated.getColumns(), EntityUtil.columnMatch); + } + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DashboardRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DashboardRepository.java index 4b944de03b9..ef9e0d0b201 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DashboardRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DashboardRepository.java @@ -14,7 +14,6 @@ package org.openmetadata.service.jdbi3; import static org.openmetadata.common.utils.CommonUtil.listOrEmpty; -import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; import static org.openmetadata.service.Entity.FIELD_FOLLOWERS; import com.fasterxml.jackson.core.JsonProcessingException; @@ -35,8 +34,8 @@ import org.openmetadata.service.util.EntityUtil.Fields; import org.openmetadata.service.util.FullyQualifiedName; public class DashboardRepository extends EntityRepository { - private static final String DASHBOARD_UPDATE_FIELDS = "owner,tags,charts,extension,followers"; - private static final String DASHBOARD_PATCH_FIELDS = "owner,tags,charts,extension,followers"; + private static final String DASHBOARD_UPDATE_FIELDS = "owner,tags,charts,extension,followers,dataModels"; + private static final String DASHBOARD_PATCH_FIELDS = "owner,tags,charts,extension,followers,dataModels"; public DashboardRepository(CollectionDAO dao) { super( @@ -58,7 +57,9 @@ public class DashboardRepository extends EntityRepository { public Dashboard setFields(Dashboard dashboard, Fields fields) throws IOException { dashboard.setService(getContainer(dashboard.getId())); dashboard.setFollowers(fields.contains(FIELD_FOLLOWERS) ? getFollowers(dashboard) : null); - dashboard.setCharts(fields.contains("charts") ? getCharts(dashboard) : null); + dashboard.setCharts(fields.contains("charts") ? getRelatedEntities(dashboard, Entity.CHART) : null); + dashboard.setDataModels( + fields.contains("dataModels") ? getRelatedEntities(dashboard, Entity.DASHBOARD_DATA_MODEL) : null); return dashboard.withUsageSummary( fields.contains("usageSummary") ? EntityUtil.getLatestUsage(daoCollection.usageDAO(), dashboard.getId()) @@ -92,16 +93,20 @@ public class DashboardRepository extends EntityRepository { @Override public void prepare(Dashboard dashboard) throws IOException { populateService(dashboard); - dashboard.setCharts(getCharts(dashboard.getCharts())); + dashboard.setCharts(EntityUtil.getEntityReferences(dashboard.getCharts(), Include.NON_DELETED)); + dashboard.setDataModels(EntityUtil.getEntityReferences(dashboard.getDataModels(), Include.NON_DELETED)); } @Override public void storeEntity(Dashboard dashboard, boolean update) throws JsonProcessingException { // Relationships and fields such as service are not stored as part of json EntityReference service = dashboard.getService(); - dashboard.withService(null); + List charts = dashboard.getCharts(); + List dataModels = dashboard.getDataModels(); + + dashboard.withService(null).withCharts(null).withDataModels(null); store(dashboard, update); - dashboard.withService(service); + dashboard.withService(service).withCharts(charts).withDataModels(dataModels); } @Override @@ -114,6 +119,15 @@ public class DashboardRepository extends EntityRepository { addRelationship(dashboard.getId(), chart.getId(), Entity.DASHBOARD, Entity.CHART, Relationship.HAS); } } + + // Add relationship from dashboard to data models + if (dashboard.getDataModels() != null) { + for (EntityReference dataModel : dashboard.getDataModels()) { + addRelationship( + dashboard.getId(), dataModel.getId(), Entity.DASHBOARD, Entity.DASHBOARD_DATA_MODEL, Relationship.HAS); + } + } + // Add owner relationship storeOwner(dashboard, dashboard.getOwner()); @@ -126,30 +140,12 @@ public class DashboardRepository extends EntityRepository { return new DashboardUpdater(original, updated, operation); } - private List getCharts(Dashboard dashboard) throws IOException { + private List getRelatedEntities(Dashboard dashboard, String entityType) throws IOException { if (dashboard == null) { return Collections.emptyList(); } - List chartIds = - findTo(dashboard.getId(), Entity.DASHBOARD, Relationship.HAS, Entity.CHART); - return EntityUtil.populateEntityReferences(chartIds, Entity.CHART); - } - - /** - * This method is used to populate the dashboard entity with all details of Chart EntityReference Users/Tools can send - * minimum details required to set relationship as id, type are the only required fields in entity reference, whereas - * we need to send fully populated object such that ElasticSearch index has all the details. - */ - private List getCharts(List charts) throws IOException { - if (nullOrEmpty(charts)) { - return Collections.emptyList(); - } - List chartRefs = new ArrayList<>(); - for (EntityReference chart : charts) { - EntityReference chartRef = Entity.getEntityReference(chart, Include.NON_DELETED); - chartRefs.add(chartRef); - } - return chartRefs.isEmpty() ? null : chartRefs; + List ids = findTo(dashboard.getId(), Entity.DASHBOARD, Relationship.HAS, entityType); + return EntityUtil.populateEntityReferences(ids, entityType); } /** Handles entity updated from PUT and POST operation. */ @@ -160,23 +156,28 @@ public class DashboardRepository extends EntityRepository { @Override public void entitySpecificUpdate() throws IOException { - updateCharts(); + update(Entity.CHART, "charts", listOrEmpty(updated.getCharts()), listOrEmpty(original.getCharts())); + update( + Entity.DASHBOARD_DATA_MODEL, + "dataModels", + listOrEmpty(updated.getDataModels()), + listOrEmpty(original.getDataModels())); } - private void updateCharts() throws JsonProcessingException { - // Remove all charts associated with this dashboard - deleteFrom(updated.getId(), Entity.DASHBOARD, Relationship.HAS, Entity.CHART); + private void update( + String entityType, String field, List updEntities, List oriEntities) + throws JsonProcessingException { + // Remove all entity type associated with this dashboard + deleteFrom(updated.getId(), Entity.DASHBOARD, Relationship.HAS, entityType); - // Add relationship from dashboard to chart - List updatedCharts = listOrEmpty(updated.getCharts()); - List origCharts = listOrEmpty(original.getCharts()); - for (EntityReference chart : updatedCharts) { - addRelationship(updated.getId(), chart.getId(), Entity.DASHBOARD, Entity.CHART, Relationship.HAS); + // Add relationship from dashboard to entity type + for (EntityReference entity : updEntities) { + addRelationship(updated.getId(), entity.getId(), Entity.DASHBOARD, entityType, Relationship.HAS); } List added = new ArrayList<>(); List deleted = new ArrayList<>(); - recordListChange("charts", origCharts, updatedCharts, added, deleted, EntityUtil.entityReferenceMatch); + recordListChange(field, oriEntities, updEntities, added, deleted, EntityUtil.entityReferenceMatch); } } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/LineageRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/LineageRepository.java index 7d2ab1a0037..340069b1cf7 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/LineageRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/LineageRepository.java @@ -100,13 +100,13 @@ public class LineageRepository { for (String fromColumn : columnLineage.getFromColumns()) { // From column belongs to the fromNode if (fromColumn.startsWith(fromTable.getFullyQualifiedName())) { - TableRepository.validateColumnFQN(fromTable, fromColumn); + ColumnUtil.validateColumnFQN(fromTable.getColumns(), fromColumn); } else { Table otherTable = dao.tableDAO().findEntityByName(FullyQualifiedName.getTableFQN(fromColumn)); - TableRepository.validateColumnFQN(otherTable, fromColumn); + ColumnUtil.validateColumnFQN(otherTable.getColumns(), fromColumn); } } - TableRepository.validateColumnFQN(toTable, columnLineage.getToColumn()); + ColumnUtil.validateColumnFQN(toTable.getColumns(), columnLineage.getToColumn()); } } return JsonUtils.pojoToJson(details); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/QueryRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/QueryRepository.java index 101882dd5cc..292747e452a 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/QueryRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/QueryRepository.java @@ -76,7 +76,6 @@ public class QueryRepository extends EntityRepository { entity.setChecksum(checkSum); entity.setName(checkSum); } - entity.setUsers(EntityUtil.populateEntityReferences(entity.getUsers())); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java index ebb52c33334..f77b52622b9 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java @@ -68,7 +68,6 @@ import org.openmetadata.schema.type.TableProfile; import org.openmetadata.schema.type.TableProfilerConfig; import org.openmetadata.schema.type.TagLabel; import org.openmetadata.service.Entity; -import org.openmetadata.service.exception.CatalogExceptionMessage; import org.openmetadata.service.exception.EntityNotFoundException; import org.openmetadata.service.resources.databases.DatabaseUtil; import org.openmetadata.service.resources.databases.TableResource; @@ -146,7 +145,7 @@ public class TableRepository extends EntityRepository { public void setFullyQualifiedName(Table table) { table.setFullyQualifiedName( FullyQualifiedName.add(table.getDatabaseSchema().getFullyQualifiedName(), table.getName())); - setColumnFQN(table.getFullyQualifiedName(), table.getColumns()); + ColumnUtil.setColumnFQN(table.getFullyQualifiedName(), table.getColumns()); } @Transaction @@ -577,17 +576,6 @@ public class TableRepository extends EntityRepository
{ deleteFrom(tableId, TABLE, Relationship.HAS, LOCATION); } - private void setColumnFQN(String parentFQN, List columns) { - columns.forEach( - c -> { - String columnFqn = FullyQualifiedName.add(parentFQN, c.getName()); - c.setFullyQualifiedName(columnFqn); - if (c.getChildren() != null) { - setColumnFQN(columnFqn, c.getChildren()); - } - }); - } - private void addDerivedColumnTags(List columns) { if (nullOrEmpty(columns)) { return; @@ -697,19 +685,6 @@ public class TableRepository extends EntityRepository
{ } } - private void getColumnProfile(boolean setProfile, List columns) throws IOException { - if (setProfile) { - for (Column c : listOrEmpty(columns)) { - c.setProfile( - JsonUtils.readValue( - daoCollection - .entityExtensionTimeSeriesDao() - .getLatestExtension(c.getFullyQualifiedName(), TABLE_COLUMN_PROFILE_EXTENSION), - ColumnProfile.class)); - } - } - } - private void validateTableFQN(String fqn) { try { dao.existsByName(fqn); @@ -726,20 +701,6 @@ public class TableRepository extends EntityRepository
{ } } - // Validate if a given column exists in the table - public static void validateColumnFQN(Table table, String columnFQN) { - boolean validColumn = false; - for (Column column : table.getColumns()) { - if (column.getFullyQualifiedName().equals(columnFQN)) { - validColumn = true; - break; - } - } - if (!validColumn) { - throw new IllegalArgumentException(CatalogExceptionMessage.invalidColumnFQN(columnFQN)); - } - } - private void validateColumnFQNs(List joinedWithList) { for (JoinedWith joinedWith : joinedWithList) { // Validate table @@ -747,7 +708,7 @@ public class TableRepository extends EntityRepository
{ Table joinedWithTable = dao.findEntityByName(tableFQN); // Validate column - validateColumnFQN(joinedWithTable, joinedWith.getFullyQualifiedName()); + ColumnUtil.validateColumnFQN(joinedWithTable.getColumns(), joinedWith.getFullyQualifiedName()); } } @@ -918,14 +879,6 @@ public class TableRepository extends EntityRepository
{ return dc -> CommonUtil.dateInRange(RestUtil.DATE_FORMAT, dc.getDate(), 0, 30); } - private TableProfile getTableProfile(Table table) throws IOException { - return JsonUtils.readValue( - daoCollection - .entityExtensionTimeSeriesDao() - .getLatestExtension(table.getFullyQualifiedName(), TABLE_PROFILE_EXTENSION), - TableProfile.class); - } - private List getCustomMetrics(Table table, String columnName) throws IOException { String extension = TABLE_COLUMN_EXTENSION + columnName + CUSTOM_METRICS_EXTENSION; return JsonUtils.readObjects( @@ -950,7 +903,7 @@ public class TableRepository extends EntityRepository
{ public void entitySpecificUpdate() throws IOException { Table origTable = original; Table updatedTable = updated; - DatabaseUtil.validateColumns(updatedTable); + DatabaseUtil.validateColumns(updatedTable.getColumns()); recordChange("tableType", origTable.getTableType(), updatedTable.getTableType()); updateConstraints(origTable, updatedTable); updateColumns("columns", origTable.getColumns(), updated.getColumns(), EntityUtil.columnMatch); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dashboards/DashboardResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dashboards/DashboardResource.java index e9c10f4be7f..61b38e10fce 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dashboards/DashboardResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dashboards/DashboardResource.java @@ -66,12 +66,15 @@ import org.openmetadata.service.util.ResultList; public class DashboardResource extends EntityResource { public static final String COLLECTION_PATH = "v1/dashboards/"; + protected static final String FIELDS = "owner,charts,followers,tags,usageSummary,extension,dataModels"; + @Override public Dashboard addHref(UriInfo uriInfo, Dashboard dashboard) { Entity.withHref(uriInfo, dashboard.getOwner()); Entity.withHref(uriInfo, dashboard.getService()); Entity.withHref(uriInfo, dashboard.getCharts()); Entity.withHref(uriInfo, dashboard.getFollowers()); + Entity.withHref(uriInfo, dashboard.getDataModels()); return dashboard; } @@ -86,8 +89,6 @@ public class DashboardResource extends EntityResource columns) { + validateColumnNames(columns); + for (Column c : columns) { validateColumnDataTypeDisplay(c); validateColumnDataLength(c); validateArrayColumn(c); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/TableResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/TableResource.java index 3f271dec638..03454106b25 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/TableResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/TableResource.java @@ -980,7 +980,7 @@ public class TableResource extends EntityResource { DatabaseUtil.validateConstraints(table.getColumns(), table.getTableConstraints()); DatabaseUtil.validateTablePartition(table.getColumns(), table.getTablePartition()); DatabaseUtil.validateViewDefinition(table.getTableType(), table.getViewDefinition()); - DatabaseUtil.validateColumns(table); + DatabaseUtil.validateColumns(table.getColumns()); return table; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/datamodels/DashboardDataModelResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/datamodels/DashboardDataModelResource.java new file mode 100644 index 00000000000..b187246eb55 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/datamodels/DashboardDataModelResource.java @@ -0,0 +1,460 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.service.resources.datamodels; + +import io.swagger.annotations.Api; +import io.swagger.v3.oas.annotations.ExternalDocumentation; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.ExampleObject; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.parameters.RequestBody; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import java.io.IOException; +import java.util.UUID; +import javax.json.JsonPatch; +import javax.validation.Valid; +import javax.validation.constraints.Max; +import javax.validation.constraints.Min; +import javax.ws.rs.Consumes; +import javax.ws.rs.DELETE; +import javax.ws.rs.DefaultValue; +import javax.ws.rs.GET; +import javax.ws.rs.PATCH; +import javax.ws.rs.POST; +import javax.ws.rs.PUT; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.SecurityContext; +import javax.ws.rs.core.UriInfo; +import org.openmetadata.schema.api.data.CreateDashboardDataModel; +import org.openmetadata.schema.api.data.RestoreEntity; +import org.openmetadata.schema.entity.data.DashboardDataModel; +import org.openmetadata.schema.type.EntityHistory; +import org.openmetadata.schema.type.Include; +import org.openmetadata.service.Entity; +import org.openmetadata.service.jdbi3.CollectionDAO; +import org.openmetadata.service.jdbi3.DashboardDataModelRepository; +import org.openmetadata.service.jdbi3.ListFilter; +import org.openmetadata.service.resources.Collection; +import org.openmetadata.service.resources.EntityResource; +import org.openmetadata.service.resources.databases.DatabaseUtil; +import org.openmetadata.service.security.Authorizer; +import org.openmetadata.service.util.EntityUtil; +import org.openmetadata.service.util.RestUtil; +import org.openmetadata.service.util.ResultList; + +@Path("/v1/dashboard/datamodels") +@Api(value = "Data Model data asset collection", tags = "Data Model data asset collection") +@Produces(MediaType.APPLICATION_JSON) +@Consumes(MediaType.APPLICATION_JSON) +@Collection(name = "datamodels") +public class DashboardDataModelResource extends EntityResource { + public static final String COLLECTION_PATH = "/v1/dashboard/datamodels"; + protected static final String FIELDS = "owner,tags,followers"; + + @Override + public DashboardDataModel addHref(UriInfo uriInfo, DashboardDataModel dashboardDataModel) { + dashboardDataModel.setHref(RestUtil.getHref(uriInfo, COLLECTION_PATH, dashboardDataModel.getId())); + Entity.withHref(uriInfo, dashboardDataModel.getOwner()); + Entity.withHref(uriInfo, dashboardDataModel.getService()); + Entity.withHref(uriInfo, dashboardDataModel.getFollowers()); + return dashboardDataModel; + } + + public DashboardDataModelResource(CollectionDAO dao, Authorizer authorizer) { + super(DashboardDataModel.class, new DashboardDataModelRepository(dao), authorizer); + } + + public static class DashboardDataModelList extends ResultList { + @SuppressWarnings("unused") + DashboardDataModelList() { + // Empty constructor needed for deserialization + } + } + + @GET + @Operation( + operationId = "listDashboardDataModels", + summary = "List Dashboard Data Models", + tags = "dashboardDataModel", + description = + "Get a list of dashboard datamodels, optionally filtered by `service` it belongs to. Use `fields` " + + "parameter to get only necessary fields. Use cursor-based pagination to limit the number " + + "entries in the list using `limit` and `before` or `after` query params.", + responses = { + @ApiResponse( + responseCode = "200", + description = "List of dashboard datamodels", + content = + @Content( + mediaType = "application/json", + schema = @Schema(implementation = DashboardDataModelList.class))) + }) + public ResultList list( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter( + description = "Fields requested in the returned resource", + schema = @Schema(type = "string", example = FIELDS)) + @QueryParam("fields") + String fieldsParam, + @Parameter( + description = "Filter dashboardDataModel by service name", + schema = @Schema(type = "string", example = "superset")) + @QueryParam("service") + String serviceParam, + @Parameter(description = "Limit the number dashboardDataModel returned. (1 to 1000000, default = 10)") + @DefaultValue("10") + @QueryParam("limit") + @Min(0) + @Max(1000000) + int limitParam, + @Parameter( + description = "Returns list of dashboardDataModel before this cursor", + schema = @Schema(type = "string")) + @QueryParam("before") + String before, + @Parameter( + description = "Returns list of dashboardDataModel after this cursor", + schema = @Schema(type = "string")) + @QueryParam("after") + String after, + @Parameter( + description = "Include all, deleted, or non-deleted entities.", + schema = @Schema(implementation = Include.class)) + @QueryParam("include") + @DefaultValue("non-deleted") + Include include) + throws IOException { + ListFilter filter = new ListFilter(include).addQueryParam("service", serviceParam); + return super.listInternal(uriInfo, securityContext, fieldsParam, filter, limitParam, before, after); + } + + @GET + @Path("/{id}/versions") + @Operation( + operationId = "listAllDataModelVersions", + summary = "List dashboard datamodel versions", + tags = "dashboardDataModel", + description = "Get a list of all the versions of a dashboard datamodel identified by `id`", + responses = { + @ApiResponse( + responseCode = "200", + description = "List of dashboard datamodel versions", + content = @Content(mediaType = "application/json", schema = @Schema(implementation = EntityHistory.class))) + }) + public EntityHistory listVersions( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the dashboard datamodel", schema = @Schema(type = "UUID")) @PathParam("id") + UUID id) + throws IOException { + return super.listVersionsInternal(securityContext, id); + } + + @GET + @Path("/{id}") + @Operation( + operationId = "getDataModelByID", + summary = "Get a dashboard datamodel by Id", + tags = "dashboardDataModel", + description = "Get a dashboard datamodel by `id`.", + responses = { + @ApiResponse( + responseCode = "200", + description = "The dashboard datamodel", + content = + @Content(mediaType = "application/json", schema = @Schema(implementation = DashboardDataModel.class))), + @ApiResponse(responseCode = "404", description = "DataModel for instance {id} is not found") + }) + public DashboardDataModel get( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the dashboard datamodel", schema = @Schema(type = "UUID")) @PathParam("id") + UUID id, + @Parameter( + description = "Fields requested in the returned resource", + schema = @Schema(type = "string", example = FIELDS)) + @QueryParam("fields") + String fieldsParam, + @Parameter( + description = "Include all, deleted, or non-deleted entities.", + schema = @Schema(implementation = Include.class)) + @QueryParam("include") + @DefaultValue("non-deleted") + Include include) + throws IOException { + return getInternal(uriInfo, securityContext, id, fieldsParam, include); + } + + @GET + @Path("/name/{fqn}") + @Operation( + operationId = "getDataModelByFQN", + summary = "Get a dashboard datamodel by fully qualified name", + tags = "dashboardDataModel", + description = "Get a dashboard datamodel by `fullyQualifiedName`.", + responses = { + @ApiResponse( + responseCode = "200", + description = "The dashboard datamodel", + content = + @Content(mediaType = "application/json", schema = @Schema(implementation = DashboardDataModel.class))), + @ApiResponse(responseCode = "404", description = "DataModel for instance {fqn} is not found") + }) + public DashboardDataModel getByName( + @Context UriInfo uriInfo, + @Parameter(description = "Fully qualified name of the dashboard datamodel", schema = @Schema(type = "string")) + @PathParam("fqn") + String fqn, + @Context SecurityContext securityContext, + @Parameter( + description = "Fields requested in the returned resource", + schema = @Schema(type = "string", example = FIELDS)) + @QueryParam("fields") + String fieldsParam, + @Parameter( + description = "Include all, deleted, or non-deleted entities.", + schema = @Schema(implementation = Include.class)) + @QueryParam("include") + @DefaultValue("non-deleted") + Include include) + throws IOException { + return getByNameInternal(uriInfo, securityContext, fqn, fieldsParam, include); + } + + @GET + @Path("/{id}/versions/{version}") + @Operation( + operationId = "getSpecificDataModelVersion", + summary = "Get a version of the dashboard datamodel", + tags = "dashboardDataModel", + description = "Get a version of the dashboard datamodel by given `id`", + responses = { + @ApiResponse( + responseCode = "200", + description = "dashboard datamodel", + content = + @Content(mediaType = "application/json", schema = @Schema(implementation = DashboardDataModel.class))), + @ApiResponse( + responseCode = "404", + description = "DataModel for instance {id} and version {version} is " + "not found") + }) + public DashboardDataModel getVersion( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the dashboard datamodel", schema = @Schema(type = "UUID")) @PathParam("id") + UUID id, + @Parameter( + description = "DataModel version number in the form `major`.`minor`", + schema = @Schema(type = "string", example = "0.1 or 1.1")) + @PathParam("version") + String version) + throws IOException { + return super.getVersionInternal(securityContext, id, version); + } + + @POST + @Operation( + operationId = "createDataModel", + summary = "Create a dashboard datamodel", + tags = "dashboardDataModel", + description = "Create a dashboard datamodel under an existing `service`.", + responses = { + @ApiResponse( + responseCode = "200", + description = "The dashboard datamodel", + content = + @Content(mediaType = "application/json", schema = @Schema(implementation = DashboardDataModel.class))), + @ApiResponse(responseCode = "400", description = "Bad request") + }) + public Response create( + @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateDashboardDataModel create) + throws IOException { + DashboardDataModel dashboardDataModel = getDataModel(create, securityContext.getUserPrincipal().getName()); + return create(uriInfo, securityContext, dashboardDataModel); + } + + @PATCH + @Path("/{id}") + @Operation( + operationId = "patchDataModel", + summary = "Update a dashboard datamodel", + tags = "dashboardDataModel", + description = "Update an existing dashboard datamodel using JsonPatch.", + externalDocs = @ExternalDocumentation(description = "JsonPatch RFC", url = "https://tools.ietf.org/html/rfc6902")) + @Consumes(MediaType.APPLICATION_JSON_PATCH_JSON) + public Response patch( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the dashboard datamodel", schema = @Schema(type = "UUID")) @PathParam("id") + UUID id, + @RequestBody( + description = "JsonPatch with array of operations", + content = + @Content( + mediaType = MediaType.APPLICATION_JSON_PATCH_JSON, + examples = { + @ExampleObject("[" + "{op:remove, path:/a}," + "{op:add, path: /b, value: val}" + "]") + })) + JsonPatch patch) + throws IOException { + return patchInternal(uriInfo, securityContext, id, patch); + } + + @PUT + @Operation( + operationId = "createOrUpdateDataModel", + summary = "Create or update dashboard datamodel", + tags = "dashboardDataModel", + description = "Create a dashboard datamodel, it it does not exist or update an existing dashboard datamodel.", + responses = { + @ApiResponse( + responseCode = "200", + description = "The updated dashboard datamodel", + content = + @Content(mediaType = "application/json", schema = @Schema(implementation = DashboardDataModel.class))) + }) + public Response createOrUpdate( + @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateDashboardDataModel create) + throws IOException { + DashboardDataModel dashboardDataModel = getDataModel(create, securityContext.getUserPrincipal().getName()); + return createOrUpdate(uriInfo, securityContext, dashboardDataModel); + } + + @PUT + @Path("/{id}/followers") + @Operation( + operationId = "addFollowerToDataModel", + summary = "Add a follower", + tags = "dashboardDataModel", + description = "Add a user identified by `userId` as followed of this data model", + responses = { + @ApiResponse(responseCode = "200", description = "OK"), + @ApiResponse(responseCode = "404", description = "DataModel for instance {id} is not found") + }) + public Response addFollower( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the data model", schema = @Schema(type = "UUID")) @PathParam("id") UUID id, + @Parameter(description = "Id of the user to be added as follower", schema = @Schema(type = "UUID")) UUID userId) + throws IOException { + return dao.addFollower(securityContext.getUserPrincipal().getName(), id, userId).toResponse(); + } + + @DELETE + @Path("/{id}/followers/{userId}") + @Operation( + operationId = "deleteFollowerFromDataModel", + summary = "Remove a follower", + tags = "dashboardDataModel", + description = "Remove the user identified `userId` as a follower of the data model.") + public Response deleteFollower( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the data model", schema = @Schema(type = "UUID")) @PathParam("id") UUID id, + @Parameter(description = "Id of the user being removed as follower", schema = @Schema(type = "UUID")) + @PathParam("userId") + UUID userId) + throws IOException { + return dao.deleteFollower(securityContext.getUserPrincipal().getName(), id, userId).toResponse(); + } + + @DELETE + @Path("/{id}") + @Operation( + operationId = "deleteDataModel", + summary = "Delete a data model by `id`.", + tags = "dashboardDataModel", + description = "Delete a dashboard datamodel by `id`.", + responses = { + @ApiResponse(responseCode = "200", description = "OK"), + @ApiResponse(responseCode = "404", description = "DataModel for instance {id} is not found") + }) + public Response delete( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Hard delete the entity. (Default = `false`)") + @QueryParam("hardDelete") + @DefaultValue("false") + boolean hardDelete, + @Parameter(description = "Id of the data model", schema = @Schema(type = "UUID")) @PathParam("id") UUID id) + throws IOException { + return delete(uriInfo, securityContext, id, false, hardDelete); + } + + @DELETE + @Path("/name/{fqn}") + @Operation( + operationId = "deleteDataModelByFQN", + summary = "Delete a data model by fully qualified name.", + tags = "dashboardDataModel", + description = "Delete a data model by `fullyQualifiedName`.", + responses = { + @ApiResponse(responseCode = "200", description = "OK"), + @ApiResponse(responseCode = "404", description = "DataModel for instance {fqn} is not found") + }) + public Response delete( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Hard delete the entity. (Default = `false`)") + @QueryParam("hardDelete") + @DefaultValue("false") + boolean hardDelete, + @Parameter(description = "Fully qualified name of the data model", schema = @Schema(type = "string")) + @PathParam("fqn") + String fqn) + throws IOException { + return deleteByName(uriInfo, securityContext, fqn, false, hardDelete); + } + + @PUT + @Path("/restore") + @Operation( + operationId = "restore", + summary = "Restore a soft deleted data model.", + tags = "dashboardDataModel", + description = "Restore a soft deleted data model.", + responses = { + @ApiResponse( + responseCode = "200", + description = "Successfully restored the data model", + content = + @Content(mediaType = "application/json", schema = @Schema(implementation = DashboardDataModel.class))) + }) + public Response restoreDataModel( + @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid RestoreEntity restore) + throws IOException { + return restoreEntity(uriInfo, securityContext, restore.getId()); + } + + private DashboardDataModel getDataModel(CreateDashboardDataModel create, String user) throws IOException { + DatabaseUtil.validateColumns(create.getColumns()); + return copy(new DashboardDataModel(), create, user) + .withService(EntityUtil.getEntityReference(Entity.DASHBOARD_SERVICE, create.getService())) + .withDataModelType(create.getDataModelType()) + .withSql(create.getSql()) + .withDataModelType(create.getDataModelType()) + .withServiceType(create.getServiceType()) + .withColumns(create.getColumns()) + .withTags(create.getTags()); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/EntityUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/EntityUtil.java index ee1e4b46f14..c0821563a16 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/util/EntityUtil.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/EntityUtil.java @@ -46,6 +46,7 @@ import org.openmetadata.schema.type.Column; import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.type.Field; import org.openmetadata.schema.type.FieldChange; +import org.openmetadata.schema.type.Include; import org.openmetadata.schema.type.MetadataOperation; import org.openmetadata.schema.type.MlFeature; import org.openmetadata.schema.type.MlHyperParameter; @@ -486,4 +487,22 @@ public final class EntityUtil { // Sort tags by tag hierarchy. Tags with parents null come first, followed by tags with tags.sort(Comparator.comparing(Tag::getFullyQualifiedName)); } + + /** + * This method is used to populate the entity with all details of EntityReference Users/Tools can send minimum details + * required to set relationship as id, type are the only required fields in entity reference, whereas we need to send + * fully populated object such that ElasticSearch index has all the details. + */ + public static List getEntityReferences(List entities, Include include) + throws IOException { + if (nullOrEmpty(entities)) { + return Collections.emptyList(); + } + List refs = new ArrayList<>(); + for (EntityReference entityReference : entities) { + EntityReference entityRef = Entity.getEntityReference(entityReference, include); + refs.add(entityRef); + } + return refs; + } } diff --git a/openmetadata-service/src/main/resources/json/data/EventSubResourceDescriptor.json b/openmetadata-service/src/main/resources/json/data/EventSubResourceDescriptor.json index c3812dc8a67..a2a12208a4a 100644 --- a/openmetadata-service/src/main/resources/json/data/EventSubResourceDescriptor.json +++ b/openmetadata-service/src/main/resources/json/data/EventSubResourceDescriptor.json @@ -383,5 +383,15 @@ "matchUpdatedBy", "matchAnyFieldChange" ] + }, + { + "name" : "dashboardDataModel", + "supportedFilters" : [ + "matchAnyOwnerName", + "matchAnyEntityFqn", + "matchAnyEventType", + "matchUpdatedBy", + "matchAnyFieldChange" + ] } ] \ No newline at end of file diff --git a/openmetadata-service/src/main/resources/json/data/ResourceDescriptors.json b/openmetadata-service/src/main/resources/json/data/ResourceDescriptors.json index d0cae55fe98..ebd92bf71be 100644 --- a/openmetadata-service/src/main/resources/json/data/ResourceDescriptors.json +++ b/openmetadata-service/src/main/resources/json/data/ResourceDescriptors.json @@ -548,5 +548,34 @@ "ViewAll", "EditDescription" ] + }, + { + "name": "dataModel", + "operations": [ + "Create", + "Delete", + "ViewAll", + "EditAll", + "EditDescription", + "EditDisplayName", + "EditTags", + "EditOwner", + "EditLineage" + ] + }, + { + "name": "dashboardDataModel", + "operations": [ + "Create", + "Delete", + "ViewAll", + "ViewUsage", + "EditAll", + "EditDescription", + "EditDisplayName", + "EditTags", + "EditOwner", + "EditLineage" + ] } ] \ No newline at end of file diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/datamodels/DashboardDataModelResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/datamodels/DashboardDataModelResourceTest.java new file mode 100644 index 00000000000..80adbc00b39 --- /dev/null +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/datamodels/DashboardDataModelResourceTest.java @@ -0,0 +1,147 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.service.resources.datamodels; + +import static javax.ws.rs.core.Response.Status.BAD_REQUEST; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.openmetadata.service.util.TestUtils.ADMIN_AUTH_HEADERS; +import static org.openmetadata.service.util.TestUtils.assertListNotEmpty; +import static org.openmetadata.service.util.TestUtils.assertListNotNull; +import static org.openmetadata.service.util.TestUtils.assertListNull; +import static org.openmetadata.service.util.TestUtils.assertResponse; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import lombok.extern.slf4j.Slf4j; +import org.apache.http.client.HttpResponseException; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; +import org.openmetadata.schema.api.data.CreateDashboardDataModel; +import org.openmetadata.schema.entity.data.DashboardDataModel; +import org.openmetadata.schema.type.DataModelType; +import org.openmetadata.schema.type.EntityReference; +import org.openmetadata.service.Entity; +import org.openmetadata.service.resources.EntityResourceTest; +import org.openmetadata.service.util.ResultList; + +@Slf4j +public class DashboardDataModelResourceTest extends EntityResourceTest { + + public DashboardDataModelResourceTest() { + super( + Entity.DASHBOARD_DATA_MODEL, + DashboardDataModel.class, + DashboardDataModelResource.DashboardDataModelList.class, + "dashboard/datamodels", + DashboardDataModelResource.FIELDS); + } + + @Test + @Execution(ExecutionMode.CONCURRENT) + void post_dataModelWithoutRequiredFields_4xx(TestInfo test) { + // Service is required field + assertResponse( + () -> createEntity(createRequest(test).withService(null), ADMIN_AUTH_HEADERS), + BAD_REQUEST, + "[service must not be null]"); + } + + @Test + @Execution(ExecutionMode.CONCURRENT) + void post_dataModelWithDifferentService_200_ok(TestInfo test) throws IOException { + String[] differentServices = {METABASE_REFERENCE.getName(), LOOKER_REFERENCE.getName()}; + + // Create dataModel for each service and test APIs + for (String service : differentServices) { + createAndCheckEntity(createRequest(test).withService(service), ADMIN_AUTH_HEADERS); + + // List dataModels by filtering on service name and ensure right dataModels in the response + Map queryParams = new HashMap<>(); + queryParams.put("service", service); + ResultList list = listEntities(queryParams, ADMIN_AUTH_HEADERS); + for (DashboardDataModel dashboardDataModel : list.getData()) { + assertEquals(service, dashboardDataModel.getService().getName()); + } + } + } + + @Override + @Execution(ExecutionMode.CONCURRENT) + public DashboardDataModel validateGetWithDifferentFields(DashboardDataModel dashboardDataModel, boolean byName) + throws HttpResponseException { + String fields = ""; + dashboardDataModel = + byName + ? getEntityByName(dashboardDataModel.getFullyQualifiedName(), fields, ADMIN_AUTH_HEADERS) + : getEntity(dashboardDataModel.getId(), fields, ADMIN_AUTH_HEADERS); + assertListNotNull(dashboardDataModel.getService(), dashboardDataModel.getServiceType()); + assertListNull(dashboardDataModel.getOwner(), dashboardDataModel.getFollowers(), dashboardDataModel.getTags()); + + // .../datamodels?fields=owner + fields = "owner,followers,tags"; + dashboardDataModel = + byName + ? getEntityByName(dashboardDataModel.getFullyQualifiedName(), fields, ADMIN_AUTH_HEADERS) + : getEntity(dashboardDataModel.getId(), fields, ADMIN_AUTH_HEADERS); + assertListNotNull(dashboardDataModel.getService(), dashboardDataModel.getServiceType()); + // Checks for other owner, tags, and followers is done in the base class + return dashboardDataModel; + } + + @Override + public CreateDashboardDataModel createRequest(String name) { + return new CreateDashboardDataModel() + .withName(name) + .withService(getContainer().getName()) + .withServiceType(CreateDashboardDataModel.DashboardServiceType.Metabase) + .withSql("SELECT * FROM tab1;") + .withDataModelType(DataModelType.MetabaseDataModel) + .withColumns(COLUMNS); + } + + @Override + public EntityReference getContainer() { + return METABASE_REFERENCE; + } + + @Override + public EntityReference getContainer(DashboardDataModel entity) { + return entity.getService(); + } + + @Override + public void validateCreatedEntity( + DashboardDataModel dashboardDataModel, CreateDashboardDataModel createRequest, Map authHeaders) { + assertNotNull(dashboardDataModel.getServiceType()); + assertReference(createRequest.getService(), dashboardDataModel.getService()); + assertEquals(createRequest.getSql(), dashboardDataModel.getSql()); + assertEquals(createRequest.getDataModelType(), dashboardDataModel.getDataModelType()); + assertListNotEmpty(dashboardDataModel.getColumns()); + } + + @Override + public void compareEntities( + DashboardDataModel expected, DashboardDataModel patched, Map authHeaders) { + assertReference(expected.getService(), patched.getService()); + } + + @Override + public void assertFieldChange(String fieldName, Object expected, Object actual) throws IOException { + assertCommonFieldChange(fieldName, expected, actual); + } +} diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/services/DashboardServiceResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/services/DashboardServiceResourceTest.java index a740dddc71f..1b1bf820b63 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/services/DashboardServiceResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/services/DashboardServiceResourceTest.java @@ -39,8 +39,8 @@ import org.apache.http.client.HttpResponseException; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInfo; import org.openmetadata.schema.api.data.CreateChart; +import org.openmetadata.schema.api.data.CreateDashboardDataModel.DashboardServiceType; import org.openmetadata.schema.api.services.CreateDashboardService; -import org.openmetadata.schema.api.services.CreateDashboardService.DashboardServiceType; import org.openmetadata.schema.entity.data.Chart; import org.openmetadata.schema.entity.services.DashboardService; import org.openmetadata.schema.entity.services.connections.TestConnectionResult; @@ -187,7 +187,7 @@ public class DashboardServiceResourceTest extends EntityResourceTest authHeaders) { if (expectedDashboardConnection != null && actualDashboardConnection != null) { - if (dashboardServiceType == CreateDashboardService.DashboardServiceType.Metabase) { + if (dashboardServiceType == DashboardServiceType.Metabase) { MetabaseConnection expectedmetabaseConnection = (MetabaseConnection) expectedDashboardConnection.getConfig(); MetabaseConnection actualMetabaseConnection; if (actualDashboardConnection.getConfig() instanceof MetabaseConnection) { diff --git a/openmetadata-spec/src/main/resources/json/schema/api/data/createDashboard.json b/openmetadata-spec/src/main/resources/json/schema/api/data/createDashboard.json index 069883a90bb..f9d077bb3b6 100644 --- a/openmetadata-spec/src/main/resources/json/schema/api/data/createDashboard.json +++ b/openmetadata-spec/src/main/resources/json/schema/api/data/createDashboard.json @@ -31,6 +31,14 @@ }, "default": null }, + "dataModels": { + "description": "List of fully qualified name of data models included in this Dashboard.", + "type": "array", + "items": { + "$ref": "../../type/basic.json#/definitions/fullyQualifiedEntityName" + }, + "default": null + }, "tags": { "description": "Tags for this dashboard", "type": "array", diff --git a/openmetadata-spec/src/main/resources/json/schema/api/data/createDashboardDataModel.json b/openmetadata-spec/src/main/resources/json/schema/api/data/createDashboardDataModel.json new file mode 100644 index 00000000000..e48dc2eee07 --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/api/data/createDashboardDataModel.json @@ -0,0 +1,61 @@ +{ + "$id": "https://open-metadata.org/schema/api/data/createDashboardDataModel.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "CreateDashboardDataModelRequest", + "description": "Create Dashboard Data Model entity request.", + "type": "object", + "javaType": "org.openmetadata.schema.api.data.CreateDashboardDataModel", + "javaInterfaces": ["org.openmetadata.schema.CreateEntity"], + "properties": { + "name": { + "description": "Name that identifies this data model.", + "$ref": "../../type/basic.json#/definitions/entityName" + }, + "displayName": { + "description": "Display Name that identifies this data model. It could be title or label from the source services.", + "type": "string" + }, + "description": { + "description": "Description of the data model instance. What it has and how to use it.", + "$ref": "../../type/basic.json#/definitions/markdown" + }, + "tags": { + "description": "Tags for this data model.", + "type": "array", + "items": { + "$ref": "../../type/tagLabel.json" + }, + "default": null + }, + "owner": { + "description": "Owner of this data model.", + "$ref": "../../type/entityReference.json" + }, + "service": { + "description": "Link to the data model service where this data model is hosted in.", + "$ref": "../../type/basic.json#/definitions/fullyQualifiedEntityName" + }, + "serviceType": { + "description": "Service type where this data model is hosted in.", + "$ref": "../../entity/services/dashboardService.json#/definitions/dashboardServiceType" + }, + "dataModelType": { + "$ref": "../../entity/data/dashboardDataModel.json#/definitions/dataModelType" + }, + "sql": { + "description": "In case the Data Model is based on a SQL query.", + "$ref": "../../type/basic.json#/definitions/sqlQuery", + "default": null + }, + "columns": { + "description": "Columns from the data model.", + "type": "array", + "items": { + "$ref": "../../entity/data/table.json#/definitions/column" + }, + "default": null + } + }, + "required": ["name", "service", "modelType"], + "additionalProperties": false +} diff --git a/openmetadata-spec/src/main/resources/json/schema/api/services/createDashboardService.json b/openmetadata-spec/src/main/resources/json/schema/api/services/createDashboardService.json index 1449bba375e..287167e760e 100644 --- a/openmetadata-spec/src/main/resources/json/schema/api/services/createDashboardService.json +++ b/openmetadata-spec/src/main/resources/json/schema/api/services/createDashboardService.json @@ -6,7 +6,6 @@ "type": "object", "javaType": "org.openmetadata.schema.api.services.CreateDashboardService", "javaInterfaces": ["org.openmetadata.schema.CreateEntity"], - "properties": { "name": { "description": "Name that identifies the this entity instance uniquely", diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/data/dashboard.json b/openmetadata-spec/src/main/resources/json/schema/entity/data/dashboard.json index 38d26b44c5e..72ecfa43129 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/data/dashboard.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/data/dashboard.json @@ -46,10 +46,12 @@ }, "charts": { "description": "All the charts included in this Dashboard.", - "type": "array", - "items": { - "$ref": "../../type/entityReference.json" - }, + "$ref": "../../type/entityReference.json#/definitions/entityReferenceList", + "default": null + }, + "dataModels": { + "description": "List of data models used by this dashboard or the charts contained on it.", + "$ref": "../../type/entityReference.json#/definitions/entityReferenceList", "default": null }, "href": { diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/data/dashboardDataModel.json b/openmetadata-spec/src/main/resources/json/schema/entity/data/dashboardDataModel.json new file mode 100644 index 00000000000..f1aee023e33 --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/entity/data/dashboardDataModel.json @@ -0,0 +1,131 @@ +{ + "$id": "https://open-metadata.org/schema/entity/data/dashboardDataModel.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "DashboardDataModel", + "$comment": "@om-entity-type", + "description": "Dashboard Data Model entity definition. Data models are the schemas used to build dashboards, charts, or other data models.", + "type": "object", + "javaType": "org.openmetadata.schema.entity.data.DashboardDataModel", + "javaInterfaces": ["org.openmetadata.schema.EntityInterface"], + "definitions": { + "dataModelType": { + "javaType": "org.openmetadata.schema.type.DataModelType", + "description": "This schema defines the type used for describing different types of data models.", + "type": "string", + "$comment": "Data Model types supported.", + "enum": [ + "TableauSheet", + "SupersetDataModel", + "MetabaseDataModel" + ], + "javaEnums": [ + { + "name": "TableauSheet" + }, + { + "name": "SupersetDataModel" + }, + { + "name": "MetabaseDataModel" + } + ] + } + }, + "properties": { + "id": { + "description": "Unique identifier of this data model instance.", + "$ref": "../../type/basic.json#/definitions/uuid" + }, + "name": { + "description": "Name of a data model. Expected to be unique within a Dashboard.", + "$ref": "../../type/basic.json#/definitions/entityName" + }, + "displayName": { + "description": "Display Name that identifies this data model. It could be title or label from the source.", + "type": "string" + }, + "fullyQualifiedName": { + "description": "Fully qualified name of a data model in the form `serviceName.dashboardName.datamodel.datamodelName`.", + "$ref": "../../type/basic.json#/definitions/fullyQualifiedEntityName" + }, + "description": { + "description": "Description of a data model.", + "$ref": "../../type/basic.json#/definitions/markdown" + }, + "version": { + "description": "Metadata version of the entity.", + "$ref": "../../type/entityHistory.json#/definitions/entityVersion" + }, + "updatedAt": { + "description": "Last update time corresponding to the new version of the entity in Unix epoch time milliseconds.", + "$ref": "../../type/basic.json#/definitions/timestamp" + }, + "updatedBy": { + "description": "User who made the update.", + "type": "string" + }, + "href": { + "description": "Link to this data model entity.", + "$ref": "../../type/basic.json#/definitions/href" + }, + "owner": { + "description": "Owner of this data model.", + "$ref": "../../type/entityReference.json", + "default": null + }, + "tags": { + "description": "Tags for this data model.", + "type": "array", + "items": { + "$ref": "../../type/tagLabel.json" + }, + "default": null + }, + "changeDescription": { + "description": "Change that lead to this version of the entity.", + "$ref": "../../type/entityHistory.json#/definitions/changeDescription" + }, + "deleted": { + "description": "When `true` indicates the entity has been soft deleted.", + "type": "boolean", + "default": false + }, + "followers": { + "description": "Followers of this dashboard.", + "$ref": "../../type/entityReference.json#/definitions/entityReferenceList" + }, + "service": { + "description": "Link to service where this data model is hosted in.", + "$ref": "../../type/entityReference.json" + }, + "serviceType": { + "description": "Service type where this data model is hosted in.", + "$ref": "../services/dashboardService.json#/definitions/dashboardServiceType" + }, + "dataModelType": { + "$ref": "#/definitions/dataModelType" + }, + "sql": { + "description": "In case the Data Model is based on a SQL query.", + "$ref": "../../type/basic.json#/definitions/sqlQuery", + "default": null + }, + "columns": { + "description": "Columns from the data model.", + "type": "array", + "items": { + "$ref": "table.json#/definitions/column" + }, + "default": null + }, + "dataModels": { + "description": "List of data models used by this data model.", + "$ref": "../../type/entityReference.json#/definitions/entityReferenceList" + } + }, + "required": [ + "name", + "modelType" + ], + "additionalProperties": false +}