yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-10-03 12:53:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix#10584: Add Data Model as an entity (#10636)

Browse Source 
* Add Data Model as entity

* Add sample_data + update dashboard resource and repository with data models

* Fix Java style

* Addess PR comments

* Update bootstrap/sql/com.mysql.cj.jdbc.Driver/v009__create_db_connection_info.sql

* Pylint error

* Address PR comments

* Address PR comments

* Address PR comments

* Minor change

* Fix error in sample_data

* Fix failing test

* Add missing resource and event sub descriptors
This commit is contained in:
Nahuel 2023-03-28 17:07:38 +02:00 committed by GitHub
parent 86febae17c
commit bea38d7200
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
30 changed files with 1381 additions and 137 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
bootstrap/sql/com.mysql.cj.jdbc.Driver/v009__create_db_connection_info.sql
View File

@ -122,4 +122,15 @@ ALTER TABLE user_tokens MODIFY COLUMN expiryDate BIGINT UNSIGNED GENERATED ALWAY
DELETE FROM alert_entity;
drop table alert_action_def;
ALTER TABLE alert_entity RENAME TO event_subscription_entity;
ALTER TABLE alert_entity RENAME TO event_subscription_entity;
-- create data model table
CREATE TABLE IF NOT EXISTS dashboard_data_model_entity (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') NOT NULL,
json JSON NOT NULL,
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
PRIMARY KEY (id),
UNIQUE (fullyQualifiedName)
);

12
bootstrap/sql/org.postgresql.Driver/v009__create_db_connection_info.sql
View File

@ -126,3 +126,15 @@ DELETE FROM alert_entity;
drop table alert_action_def;
ALTER TABLE alert_entity RENAME TO event_subscription_entity;
-- create data model table
CREATE TABLE IF NOT EXISTS dashboard_data_model_entity (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
json JSONB NOT NULL,
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL,
PRIMARY KEY (id),
UNIQUE (fullyQualifiedName)
);

24
ingestion/examples/sample_data/dashboards/charts.json
View File

@ -4,7 +4,7 @@
"id": "2841fdb1-e378-4a2c-94f8-27c9f5d6ef8e",
"name": "114",
"displayName": "# of Games That Hit 100k in Sales By Release Year",
"fullyQualifiedName": "local_superset.101",
"fullyQualifiedName": "sample_superset.101",
"description": "",
"chartId": "114",
"chartType": "Area",
@ -15,7 +15,7 @@
"id": "3bcba490-9e5c-4946-a0e3-41e8ff8f4aa4",
"name":"166",
"displayName": "% Rural",
"fullyQualifiedName": "local_superset.110",
"fullyQualifiedName": "sample_superset.110",
"description": "",
"chartId": "166",
"chartType": "Other",
@ -26,7 +26,7 @@
"id": "22b95748-4a7b-48ad-859e-cf7c66a7f343",
"name": "92",
"displayName": "✈️ Relocation ability",
"fullyQualifiedName": "local_superset.92",
"fullyQualifiedName": "sample_superset.92",
"description": "",
"chartId": "92",
"chartType": "Other",
@ -37,7 +37,7 @@
"id": "62b31dcc-4619-46a0-99b1-0fa7cd6f93da",
"name": "117",
"displayName": "Age distribution of respondents",
"fullyQualifiedName": "local_superset.11",
"fullyQualifiedName": "sample_superset.11",
"description": "",
"chartId": "117",
"chartType": "Histogram",
@ -47,7 +47,7 @@
{
"id": "57944482-e187-439a-aaae-0e8aabd2f455",
"displayName": "Arcs",
"fullyQualifiedName": "local_superset.197",
"fullyQualifiedName": "sample_superset.197",
"description": "",
"name": "197",
"chartType": "Other",
@ -57,7 +57,7 @@
{
"id": "d88e2056-c74a-410d-829e-eb31b040c132",
"displayName": "Are you an ethnic minority in your city?",
"fullyQualifiedName": "local_superset.127",
"fullyQualifiedName": "sample_superset.127",
"description": "",
"name": "127",
"chartType": "Other",
@ -67,7 +67,7 @@
{
"id": "c1d3e156-4628-414e-8d6e-a6bdd486128f",
"displayName": "Average and Sum Trends",
"fullyQualifiedName": "local_superset.183",
"fullyQualifiedName": "sample_superset.183",
"description": "",
"name": "183",
"chartType": "Line",
@ -77,7 +77,7 @@
{
"id": "bfc57519-8cef-47e6-a423-375d5b89a6a4",
"displayName": "Birth in France by department in 2016",
"fullyQualifiedName": "local_superset.Birth in France by department in 2016",
"fullyQualifiedName": "sample_superset.Birth in France by department in 2016",
"description": "",
"name": "161",
"chartType": "Other",
@ -87,7 +87,7 @@
{
"id": "bf2eeac4-7226-46c6-bbef-918569c137a0",
"displayName": "Box plot",
"fullyQualifiedName": "local_superset.170",
"fullyQualifiedName": "sample_superset.170",
"description": "",
"name": "170",
"chartType": "Bar",
@ -97,7 +97,7 @@
{
"id": "167fd63b-42f1-4d7e-a37d-893fd8173b44",
"displayName": "Boy Name Cloud",
"fullyQualifiedName": "local_superset.180",
"fullyQualifiedName": "sample_superset.180",
"description": "",
"name": "180",
"chartType": "Other",
@ -107,7 +107,7 @@
{
"id": "8474e579-4eff-492b-8685-70ec9aa99f5f",
"displayName": "ETA Predictions Accuracy",
"fullyQualifiedName": "local_superset.210",
"fullyQualifiedName": "sample_superset.210",
"description": "",
"name": "210",
"chartType": "Line",
@ -117,7 +117,7 @@
{
"id": "12345e567-4eff-492b-8685-69ec9bb88g4g",
"displayName": "Sales Predictions Accuracy",
"fullyQualifiedName": "local_superset.211",
"fullyQualifiedName": "sample_superset.211",
"description": "",
"name": "211",
"chartType": "Line",

101
ingestion/examples/sample_data/dashboards/dashboardDataModels.json Normal file
View File

@ -0,0 +1,101 @@
{
"datamodels": [
{
"id": "e093dd27-390e-4360-8efd-e4d63ec167a9",
"name": "103",
"displayName": "Vaccine Candidates per Phase",
"fullyQualifiedName": "sample_superset.model.103",
"description": "Data of Vaccine Candidates per Phase",
"version": 0.1,
"updatedAt": 1638354087591,
"dataModelType": "SupersetDataModel",
"serviceType": "Superset",
"sql": "SELECT CASE\n WHEN stage_of_development = 'Pre-clinical' THEN '0. Pre-clinical'\n WHEN stage_of_development = 'Phase I' THEN '1. Phase I'\n WHEN stage_of_development = 'Phase I/II'\n or stage_of_development = 'Phase II' THEN '2. Phase II or Combined I/II'\n WHEN stage_of_development = 'Phase III' THEN '3. Phase III'\n WHEN stage_of_development = 'Authorized' THEN '4. Authorized'\n END AS clinical_stage,\n COUNT(*) AS count\nFROM covid_vaccines\nGROUP BY CASE\n WHEN stage_of_development = 'Pre-clinical' THEN '0. Pre-clinical'\n WHEN stage_of_development = 'Phase I' THEN '1. Phase I'\n WHEN stage_of_development = 'Phase I/II'\n or stage_of_development = 'Phase II' THEN '2. Phase II or Combined I/II'\n WHEN stage_of_development = 'Phase III' THEN '3. Phase III'\n WHEN stage_of_development = 'Authorized' THEN '4. Authorized'\n END\nORDER BY count DESC\nLIMIT 10000\nOFFSET 0;\n",
"columns": [
{
"name": "0. Pre-clinical",
"dataType": "NUMERIC",
"dataTypeDisplay": "numeric",
"description": "Vaccine Candidates in phase: 'Pre-clinical'",
"tags": [],
"ordinalPosition": 1
},
{
"name": "2. Phase II or Combined I/II",
"dataType": "NUMERIC",
"dataTypeDisplay": "numeric",
"description": "Vaccine Candidates in phase: 'Phase II or Combined I/II'",
"tags": [],
"ordinalPosition": 2
},
{
"name": "1. Phase I",
"dataType": "NUMERIC",
"dataTypeDisplay": "numeric",
"description": "Vaccine Candidates in phase: 'Phase I'",
"tags": [],
"ordinalPosition": 3
},
{
"name": "3. Phase III",
"dataType": "NUMERIC",
"dataTypeDisplay": "numeric",
"description": "Vaccine Candidates in phase: 'Phase III'",
"tags": [],
"ordinalPosition": 4
},
{
"name": "4. Authorized",
"dataType": "NUMERIC",
"dataTypeDisplay": "numeric",
"description": "Vaccine Candidates in phase: 'Authorize'",
"tags": [],
"ordinalPosition": 5
}
],
"tags": [],
"followers": []
},
{
"id": "5bab1ca3-7a22-4f21-9b34-e2c44dee1af6",
"name": "73",
"displayName": "Vaccine Candidates per Country",
"fullyQualifiedName": "sample_superset.model.73",
"description": "Data of Vaccine Candidates per Country",
"version": 0.1,
"updatedAt": 1638354087591,
"dataModelType": "SupersetDataModel",
"serviceType": "Superset",
"sql": "SELECT country_name AS country_name,\n COUNT(*) AS count,\n count(country_name) AS \"COUNT(Country_Name)\"\nFROM covid_vaccines\nGROUP BY country_name\nLIMIT 10000\nOFFSET 0;",
"columns": [
{
"name": "country_name",
"dataType": "VARCHAR",
"dataTypeDisplay": "varchar",
"dataLength": 256,
"description": "Name of the country.",
"tags": [],
"ordinalPosition": 1
},
{
"name": "count",
"dataType": "NUMERIC",
"dataTypeDisplay": "numeric",
"description": "Total number of vaccine candidates per country.",
"tags": [],
"ordinalPosition": 2
},
{
"name": "COUNT(Country_Name)",
"dataType": "NUMERIC",
"dataTypeDisplay": "numeric",
"description": "Total number of vaccine candidates.",
"tags": [],
"ordinalPosition": 3
}
],
"tags": [],
"followers": []
}
]
}

25
ingestion/examples/sample_data/dashboards/dashboards.json
View File

@ -4,7 +4,7 @@
"id": "d4dc7baf-1b17-45f8-acd5-a15b78cc7c5f",
"name": "8",
"displayName": "Orders dashboard",
"fullyQualifiedName": "local_superset.8",
"fullyQualifiedName": "sample_superset.8",
"description": "",
"dashboardUrl": "http://localhost:808/superset/dashboard/1/",
"charts": ["sample_superset.183", "sample_superset.170", "sample_superset.197"],
@ -14,17 +14,18 @@
"id": "063cd787-8630-4809-9702-34d3992c7248",
"name": "9",
"displayName": "COVID Vaccine Dashboard",
"fullyQualifiedName": "local_superset.9",
"fullyQualifiedName": "sample_superset.9",
"description": "",
"dashboardUrl": "http://localhost:808/superset/dashboard/8/",
"charts": ["sample_superset.117", "sample_superset.197"],
"dataModels": ["sample_superset.model.103", "sample_superset.model.73"],
"href": "http://localhost:8585/api/v1/dashboards/063cd787-8630-4809-9702-34d3992c7248"
},
{
"id": "df6c698e-066a-4440-be0a-121025573b73",
"name": "10",
"displayName": "deck.gl Demo",
"fullyQualifiedName": "local_superset.10",
"fullyQualifiedName": "sample_superset.10",
"description": "",
"dashboardUrl": "http://localhost:808/superset/dashboard/deck/",
"charts": ["sample_superset.127", "sample_superset.166", "sample_superset.114"],
@ -34,7 +35,7 @@
"id": "98b38a49-b5c6-431b-b61f-690e39f8ead2",
"name": "11",
"displayName": "FCC New Coder Survey 2018",
"fullyQualifiedName": "local_superset.11",
"fullyQualifiedName": "sample_superset.11",
"description": "",
"dashboardUrl": "http://localhost:808/superset/dashboard/7/",
"charts": ["sample_superset.183", "sample_superset.197", "sample_superset.170", "sample_superset.180"],
@ -44,7 +45,7 @@
"id": "dffcf9b2-4f43-4881-a5f5-10109655bf50",
"name": "12",
"displayName": "Misc Charts",
"fullyQualifiedName": "local_superset.12",
"fullyQualifiedName": "sample_superset.12",
"description": "",
"dashboardUrl": "http://localhost:808/superset/dashboard/misc_charts/",
"charts": ["sample_superset.127", "sample_superset.197"],
@ -54,7 +55,7 @@
"id": "2583737d-6236-421e-ba0f-cd0b79adb216",
"name": "31",
"displayName": "Sales Dashboard",
"fullyQualifiedName": "local_superset.31",
"fullyQualifiedName": "sample_superset.31",
"description": "",
"dashboardUrl": "http://localhost:808/superset/dashboard/6/",
"charts": ["sample_superset.92", "sample_superset.117", "sample_superset.166"],
@ -64,7 +65,7 @@
"id": "6bf9bfcb-4e80-4af0-9f0c-13e47bbc27a2",
"name": "33",
"displayName": "Slack Dashboard",
"fullyQualifiedName": "local_superset.33",
"fullyQualifiedName": "sample_superset.33",
"description": "",
"dashboardUrl": "http://localhost:808/superset/dashboard/10/",
"charts": ["sample_superset.114", "sample_superset.92", "sample_superset.127"],
@ -74,7 +75,7 @@
"id": "1f02caf2-c5e5-442d-bda3-b8ce3e757b45",
"name": "34",
"displayName": "Unicode Test",
"fullyQualifiedName": "local_superset.34",
"fullyQualifiedName": "sample_superset.34",
"description": "",
"dashboardUrl": "http://localhost:808/superset/dashboard/unicode-test/",
"charts": ["sample_superset.161", "sample_superset.170", "sample_superset.180"],
@ -84,7 +85,7 @@
"id": "a3ace318-ee37-4da1-974a-62eddbd77d20",
"name": "45",
"displayName": "USA Births Names",
"fullyQualifiedName": "local_superset.45",
"fullyQualifiedName": "sample_superset.45",
"description": "",
"dashboardUrl": "http://localhost:808/superset/dashboard/births/",
"charts": ["sample_superset.180"],
@ -94,7 +95,7 @@
"id": "e6e21717-1164-403f-8807-d12be277aec6",
"name": "51",
"displayName": "Video Game Sales",
"fullyQualifiedName": "local_superset.51",
"fullyQualifiedName": "sample_superset.51",
"description": "",
"dashboardUrl": "http://localhost:808/superset/dashboard/11/",
"charts": ["sample_superset.127", "sample_superset.183"],
@ -104,7 +105,7 @@
"id": "d2b0af00-f419-4905-bb43-036697ce53a5",
"name": "eta_predictions_performance",
"displayName": "ETA Predictions Performance",
"fullyQualifiedName": "local_superset.eta_predictions_performance",
"fullyQualifiedName": "sample_superset.eta_predictions_performance",
"description": "",
"dashboardUrl": "http://localhost:808/superset/dashboard/eta_predictions_performance/",
"charts": ["sample_superset.210"],
@ -114,7 +115,7 @@
"id": "f5a1af99-f123-7845-cc12-0312347ce53a",
"name": "forecast_sales_performance",
"displayName": "ETA Predictions Performance",
"fullyQualifiedName": "local_superset.forecast_sales_performance",
"fullyQualifiedName": "sample_superset.forecast_sales_performance",
"description": "",
"dashboardUrl": "http://localhost:808/superset/dashboard/forecast_sales_performance/",
"charts": ["sample_superset.211"],

23
ingestion/examples/sample_data/lineage/lineage.json
View File

@ -58,6 +58,27 @@
},
"edge_meta": { "fqn": "sample_airflow.dim_product_etl", "type": "pipeline" },
"sql_query": "create ecommerce_db.shopify.\"dim.product.variant\" as select * from sample_data.ecommerce_db.shopify.raw_customer"
},
{
"from": {
"fqn": "sample_superset.model.103",
"type": "dashboardDataModel"
},
"to": {
"fqn": "sample_superset.9",
"type": "dashboard"
},
"edge_meta": { "fqn": "", "type": "" }
},
{
"from": {
"fqn": "sample_superset.model.73",
"type": "dashboardDataModel"
},
"to": {
"fqn": "sample_superset.9",
"type": "dashboard"
},
"edge_meta": { "fqn": "", "type": "" }
}
]

12
ingestion/src/metadata/ingestion/ometa/ometa_api.py
View File

@ -39,6 +39,7 @@ from metadata.generated.schema.entity.classification.tag import Tag
from metadata.generated.schema.entity.data.chart import Chart
from metadata.generated.schema.entity.data.container import Container
from metadata.generated.schema.entity.data.dashboard import Dashboard
from metadata.generated.schema.entity.data.dashboardDataModel import DashboardDataModel
from metadata.generated.schema.entity.data.database import Database
from metadata.generated.schema.entity.data.databaseSchema import DatabaseSchema
from metadata.generated.schema.entity.data.glossary import Glossary
@ -249,6 +250,16 @@ class OpenMetadata(
):
return "/charts"
if issubclass(
entity,
get_args(
Union[
DashboardDataModel, self.get_create_entity_type(DashboardDataModel)
]
),
):
return "/dashboard/datamodels"
if issubclass(
entity, get_args(Union[Dashboard, self.get_create_entity_type(Dashboard)])
):
@ -529,6 +540,7 @@ class OpenMetadata(
file_name = (
class_name.lower()
.replace("glossaryterm", "glossaryTerm")
.replace("dashboarddatamodel", "dashboardDataModel")
.replace("testsuite", "testSuite")
.replace("testdefinition", "testDefinition")
.replace("testcase", "testCase")

48
ingestion/src/metadata/ingestion/source/database/sample_data.py
View File

@ -23,6 +23,9 @@ from pydantic import ValidationError
from metadata.generated.schema.api.data.createChart import CreateChartRequest
from metadata.generated.schema.api.data.createContainer import CreateContainerRequest
from metadata.generated.schema.api.data.createDashboard import CreateDashboardRequest
from metadata.generated.schema.api.data.createDashboardDataModel import (
CreateDashboardDataModelRequest,
)
from metadata.generated.schema.api.data.createDatabase import CreateDatabaseRequest
from metadata.generated.schema.api.data.createDatabaseSchema import (
CreateDatabaseSchemaRequest,
@ -46,6 +49,7 @@ from metadata.generated.schema.api.tests.createTestCase import CreateTestCaseReq
from metadata.generated.schema.api.tests.createTestSuite import CreateTestSuiteRequest
from metadata.generated.schema.entity.data.container import Container
from metadata.generated.schema.entity.data.dashboard import Dashboard
from metadata.generated.schema.entity.data.dashboardDataModel import DashboardDataModel
from metadata.generated.schema.entity.data.database import Database
from metadata.generated.schema.entity.data.databaseSchema import DatabaseSchema
from metadata.generated.schema.entity.data.location import Location
@ -158,6 +162,10 @@ def get_lineage_entity_ref(edge, metadata_config) -> EntityReference:
dashboard = metadata.get_by_name(entity=Dashboard, fqn=edge_fqn)
if dashboard:
return EntityReference(id=dashboard.id, type="dashboard")
if edge["type"] == "dashboardDataModel":
data_model = metadata.get_by_name(entity=DashboardDataModel, fqn=edge_fqn)
if data_model:
return EntityReference(id=data_model.id, type="dashboardDataModel")
return None
@ -328,6 +336,13 @@ class SampleDataSource(
encoding="utf-8",
)
)
self.data_models = json.load(
open( # pylint: disable=consider-using-with
sample_data_folder + "/dashboards/dashboardDataModels.json",
"r",
encoding="utf-8",
)
)
self.dashboards = json.load(
open( # pylint: disable=consider-using-with
sample_data_folder + "/dashboards/dashboards.json",
@ -478,6 +493,7 @@ class SampleDataSource(
yield from self.ingest_tables()
yield from self.ingest_topics()
yield from self.ingest_charts()
yield from self.ingest_data_models()
yield from self.ingest_dashboards()
yield from self.ingest_pipelines()
yield from self.ingest_lineage()
@ -716,7 +732,7 @@ class SampleDataSource(
name=chart["name"],
displayName=chart["displayName"],
description=chart["description"],
chartType=get_standard_chart_type(chart["chartType"]).value,
chartType=get_standard_chart_type(chart["chartType"]),
chartUrl=chart["chartUrl"],
service=self.dashboard_service.fullyQualifiedName,
)
@ -726,6 +742,29 @@ class SampleDataSource(
logger.debug(traceback.format_exc())
logger.warning(f"Unexpected exception ingesting chart [{chart}]: {err}")
def ingest_data_models(self) -> Iterable[CreateDashboardDataModelRequest]:
for data_model in self.data_models["datamodels"]:
try:
data_model_ev = CreateDashboardDataModelRequest(
name=data_model["name"],
displayName=data_model["displayName"],
description=data_model["description"],
columns=data_model["columns"],
dataModelType=data_model["dataModelType"],
sql=data_model["sql"],
serviceType=data_model["serviceType"],
service=self.dashboard_service.fullyQualifiedName,
)
self.status.scanned(
f"Data Model Scanned: {data_model_ev.name.__root__}"
)
yield data_model_ev
except ValidationError as err:
logger.debug(traceback.format_exc())
logger.warning(
f"Unexpected exception ingesting chart [{data_model}]: {err}"
)
def ingest_dashboards(self) -> Iterable[CreateDashboardRequest]:
for dashboard in self.dashboards["dashboards"]:
dashboard_ev = CreateDashboardRequest(
@ -734,6 +773,7 @@ class SampleDataSource(
description=dashboard["description"],
dashboardUrl=dashboard["dashboardUrl"],
charts=dashboard["charts"],
dataModels=dashboard.get("dataModels", None),
service=self.dashboard_service.fullyQualifiedName,
)
self.status.scanned(f"Dashboard Scanned: {dashboard_ev.name.__root__}")
@ -758,8 +798,10 @@ class SampleDataSource(
edge_entity_ref = get_lineage_entity_ref(
edge["edge_meta"], self.metadata_config
)
lineage_details = LineageDetails(
pipeline=edge_entity_ref, sqlQuery=edge.get("sql_query")
lineage_details = (
LineageDetails(pipeline=edge_entity_ref, sqlQuery=edge.get("sql_query"))
if edge_entity_ref
else None
)
lineage = AddLineageRequest(
edge=EntitiesEdge(

2
openmetadata-service/src/main/java/org/openmetadata/service/Entity.java
View File

@ -115,6 +115,8 @@ public final class Entity {
public static final String WEB_ANALYTIC_EVENT = "webAnalyticEvent";
public static final String DATA_INSIGHT_CHART = "dataInsightChart";
public static final String DASHBOARD_DATA_MODEL = "dashboardDataModel";
//
// Policy entity
//

21
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java
View File

@ -65,6 +65,7 @@ import org.openmetadata.schema.entity.classification.Tag;
import org.openmetadata.schema.entity.data.Chart;
import org.openmetadata.schema.entity.data.Container;
import org.openmetadata.schema.entity.data.Dashboard;
import org.openmetadata.schema.entity.data.DashboardDataModel;
import org.openmetadata.schema.entity.data.Database;
import org.openmetadata.schema.entity.data.DatabaseSchema;
import org.openmetadata.schema.entity.data.Glossary;
@ -270,6 +271,9 @@ public interface CollectionDAO {
@CreateSqlObject
WorkflowDAO workflowDAO();
@CreateSqlObject
DataModelDAO dataModelDAO();
interface DashboardDAO extends EntityDAO<Dashboard> {
@Override
default String getTableName() {
@ -3666,4 +3670,21 @@ public interface CollectionDAO {
@Define("nameColumn") String nameColumn,
@Define("sqlCondition") String sqlCondition);
}
interface DataModelDAO extends EntityDAO<DashboardDataModel> {
@Override
default String getTableName() {
return "dashboard_data_model_entity";
}
@Override
default Class<DashboardDataModel> getEntityClass() {
return DashboardDataModel.class;
}
@Override
default String getNameColumn() {
return "fullyQualifiedName";
}
}
}

27
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ColumnUtil.java
View File

@ -5,6 +5,8 @@ import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty;
import java.util.ArrayList;
import java.util.List;
import org.openmetadata.schema.type.Column;
import org.openmetadata.service.exception.CatalogExceptionMessage;
import org.openmetadata.service.util.FullyQualifiedName;
public final class ColumnUtil {
private ColumnUtil() {}
@ -35,4 +37,29 @@ public final class ColumnUtil {
.withOrdinalPosition(column.getOrdinalPosition())
.withChildren(children);
}
public static void setColumnFQN(String parentFQN, List<Column> columns) {
columns.forEach(
c -> {
String columnFqn = FullyQualifiedName.add(parentFQN, c.getName());
c.setFullyQualifiedName(columnFqn);
if (c.getChildren() != null) {
setColumnFQN(columnFqn, c.getChildren());
}
});
}
// Validate if a given column exists in the table
public static void validateColumnFQN(List<Column> columns, String columnFQN) {
boolean validColumn = false;
for (Column column : columns) {
if (column.getFullyQualifiedName().equals(columnFQN)) {
validColumn = true;
break;
}
}
if (!validColumn) {
throw new IllegalArgumentException(CatalogExceptionMessage.invalidColumnFQN(columnFQN));
}
}
}

174
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DashboardDataModelRepository.java Normal file
View File

@ -0,0 +1,174 @@
/*
* Copyright 2021 Collate
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.openmetadata.service.jdbi3;
import static org.openmetadata.common.utils.CommonUtil.listOrEmpty;
import static org.openmetadata.service.Entity.FIELD_FOLLOWERS;
import static org.openmetadata.service.Entity.FIELD_TAGS;
import com.fasterxml.jackson.core.JsonProcessingException;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.openmetadata.schema.entity.data.DashboardDataModel;
import org.openmetadata.schema.entity.services.DashboardService;
import org.openmetadata.schema.type.Column;
import org.openmetadata.schema.type.EntityReference;
import org.openmetadata.schema.type.Include;
import org.openmetadata.schema.type.Relationship;
import org.openmetadata.schema.type.TagLabel;
import org.openmetadata.service.Entity;
import org.openmetadata.service.resources.databases.DatabaseUtil;
import org.openmetadata.service.resources.datamodels.DashboardDataModelResource;
import org.openmetadata.service.util.EntityUtil;
import org.openmetadata.service.util.EntityUtil.Fields;
import org.openmetadata.service.util.FullyQualifiedName;
@Slf4j
public class DashboardDataModelRepository extends EntityRepository<DashboardDataModel> {
private static final String DATA_MODELS_FIELD = "dataModels";
private static final String DATA_MODEL_UPDATE_FIELDS = "owner,tags,followers";
private static final String DATA_MODEL_PATCH_FIELDS = "owner,tags,followers";
public DashboardDataModelRepository(CollectionDAO dao) {
super(
DashboardDataModelResource.COLLECTION_PATH,
Entity.DASHBOARD_DATA_MODEL,
DashboardDataModel.class,
dao.dataModelDAO(),
dao,
DATA_MODEL_PATCH_FIELDS,
DATA_MODEL_UPDATE_FIELDS);
}
@Override
public void setFullyQualifiedName(DashboardDataModel dashboardDataModel) {
dashboardDataModel.setFullyQualifiedName(
FullyQualifiedName.add(dashboardDataModel.getService().getName() + ".model", dashboardDataModel.getName()));
ColumnUtil.setColumnFQN(dashboardDataModel.getFullyQualifiedName(), dashboardDataModel.getColumns());
}
@Override
public void prepare(DashboardDataModel dashboardDataModel) throws IOException {
DashboardService dashboardService = Entity.getEntity(dashboardDataModel.getService(), "", Include.ALL);
dashboardDataModel.setService(dashboardService.getEntityReference());
dashboardDataModel.setServiceType(dashboardService.getServiceType());
}
@Override
public void storeEntity(DashboardDataModel dashboardDataModel, boolean update) throws JsonProcessingException {
// Relationships and fields such as href are derived and not stored as part of json
EntityReference owner = dashboardDataModel.getOwner();
List<TagLabel> tags = dashboardDataModel.getTags();
List<EntityReference> dataModels = dashboardDataModel.getDataModels();
EntityReference service = dashboardDataModel.getService();
// Don't store owner, database, href and tags as JSON. Build it on the fly based on relationships
dashboardDataModel.withOwner(null).withService(null).withHref(null).withTags(null).withDataModels(null);
store(dashboardDataModel, update);
// Restore the relationships
dashboardDataModel.withOwner(owner).withService(service).withTags(tags).withDataModels(dataModels);
}
@Override
@SneakyThrows
public void storeRelationships(DashboardDataModel dashboardDataModel) {
EntityReference service = dashboardDataModel.getService();
addRelationship(
service.getId(),
dashboardDataModel.getId(),
service.getType(),
Entity.DASHBOARD_DATA_MODEL,
Relationship.CONTAINS);
storeOwner(dashboardDataModel, dashboardDataModel.getOwner());
applyTags(dashboardDataModel);
}
@Override
public DashboardDataModel setFields(DashboardDataModel dashboardDataModel, Fields fields) throws IOException {
getColumnTags(fields.contains(FIELD_TAGS), dashboardDataModel.getColumns());
return dashboardDataModel
.withService(getContainer(dashboardDataModel.getId()))
.withFollowers(fields.contains(FIELD_FOLLOWERS) ? getFollowers(dashboardDataModel) : null)
.withTags(fields.contains(FIELD_TAGS) ? getTags(dashboardDataModel.getFullyQualifiedName()) : null)
.withDataModels(fields.contains(DATA_MODELS_FIELD) ? getDataModels(dashboardDataModel) : null);
}
@Override
public void restorePatchAttributes(DashboardDataModel original, DashboardDataModel updated) {
// Patch can't make changes to following fields. Ignore the changes
updated
.withFullyQualifiedName(original.getFullyQualifiedName())
.withName(original.getName())
.withService(original.getService())
.withId(original.getId());
}
protected List<EntityReference> getDataModels(DashboardDataModel dashboardDataModel) throws IOException {
if (dashboardDataModel == null) {
return Collections.emptyList();
}
List<CollectionDAO.EntityRelationshipRecord> tableIds =
findTo(dashboardDataModel.getId(), entityType, Relationship.USES, Entity.DASHBOARD_DATA_MODEL);
return EntityUtil.populateEntityReferences(tableIds, Entity.TABLE);
}
private void getColumnTags(boolean setTags, List<Column> columns) {
for (Column c : listOrEmpty(columns)) {
c.setTags(setTags ? getTags(c.getFullyQualifiedName()) : null);
getColumnTags(setTags, c.getChildren());
}
}
private void applyTags(List<Column> columns) {
// Add column level tags by adding tag to column relationship
for (Column column : columns) {
applyTags(column.getTags(), column.getFullyQualifiedName());
if (column.getChildren() != null) {
applyTags(column.getChildren());
}
}
}
@Override
public void applyTags(DashboardDataModel dashboardDataModel) {
// Add table level tags by adding tag to table relationship
super.applyTags(dashboardDataModel);
applyTags(dashboardDataModel.getColumns());
}
@Override
public EntityUpdater getUpdater(DashboardDataModel original, DashboardDataModel updated, Operation operation) {
return new DataModelUpdater(original, updated, operation);
}
public class DataModelUpdater extends ColumnEntityUpdater {
public DataModelUpdater(DashboardDataModel original, DashboardDataModel updated, Operation operation) {
super(original, updated, operation);
}
@Override
public void entitySpecificUpdate() throws IOException {
DatabaseUtil.validateColumns(original.getColumns());
updateColumns("columns", original.getColumns(), updated.getColumns(), EntityUtil.columnMatch);
}
}
}

77
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DashboardRepository.java
View File

@ -14,7 +14,6 @@
package org.openmetadata.service.jdbi3;
import static org.openmetadata.common.utils.CommonUtil.listOrEmpty;
import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty;
import static org.openmetadata.service.Entity.FIELD_FOLLOWERS;
import com.fasterxml.jackson.core.JsonProcessingException;
@ -35,8 +34,8 @@ import org.openmetadata.service.util.EntityUtil.Fields;
import org.openmetadata.service.util.FullyQualifiedName;
public class DashboardRepository extends EntityRepository<Dashboard> {
private static final String DASHBOARD_UPDATE_FIELDS = "owner,tags,charts,extension,followers";
private static final String DASHBOARD_PATCH_FIELDS = "owner,tags,charts,extension,followers";
private static final String DASHBOARD_UPDATE_FIELDS = "owner,tags,charts,extension,followers,dataModels";
private static final String DASHBOARD_PATCH_FIELDS = "owner,tags,charts,extension,followers,dataModels";
public DashboardRepository(CollectionDAO dao) {
super(
@ -58,7 +57,9 @@ public class DashboardRepository extends EntityRepository<Dashboard> {
public Dashboard setFields(Dashboard dashboard, Fields fields) throws IOException {
dashboard.setService(getContainer(dashboard.getId()));
dashboard.setFollowers(fields.contains(FIELD_FOLLOWERS) ? getFollowers(dashboard) : null);
dashboard.setCharts(fields.contains("charts") ? getCharts(dashboard) : null);
dashboard.setCharts(fields.contains("charts") ? getRelatedEntities(dashboard, Entity.CHART) : null);
dashboard.setDataModels(
fields.contains("dataModels") ? getRelatedEntities(dashboard, Entity.DASHBOARD_DATA_MODEL) : null);
return dashboard.withUsageSummary(
fields.contains("usageSummary")
? EntityUtil.getLatestUsage(daoCollection.usageDAO(), dashboard.getId())
@ -92,16 +93,20 @@ public class DashboardRepository extends EntityRepository<Dashboard> {
@Override
public void prepare(Dashboard dashboard) throws IOException {
populateService(dashboard);
dashboard.setCharts(getCharts(dashboard.getCharts()));
dashboard.setCharts(EntityUtil.getEntityReferences(dashboard.getCharts(), Include.NON_DELETED));
dashboard.setDataModels(EntityUtil.getEntityReferences(dashboard.getDataModels(), Include.NON_DELETED));
}
@Override
public void storeEntity(Dashboard dashboard, boolean update) throws JsonProcessingException {
// Relationships and fields such as service are not stored as part of json
EntityReference service = dashboard.getService();
dashboard.withService(null);
List<EntityReference> charts = dashboard.getCharts();
List<EntityReference> dataModels = dashboard.getDataModels();
dashboard.withService(null).withCharts(null).withDataModels(null);
store(dashboard, update);
dashboard.withService(service);
dashboard.withService(service).withCharts(charts).withDataModels(dataModels);
}
@Override
@ -114,6 +119,15 @@ public class DashboardRepository extends EntityRepository<Dashboard> {
addRelationship(dashboard.getId(), chart.getId(), Entity.DASHBOARD, Entity.CHART, Relationship.HAS);
}
}
// Add relationship from dashboard to data models
if (dashboard.getDataModels() != null) {
for (EntityReference dataModel : dashboard.getDataModels()) {
addRelationship(
dashboard.getId(), dataModel.getId(), Entity.DASHBOARD, Entity.DASHBOARD_DATA_MODEL, Relationship.HAS);
}
}
// Add owner relationship
storeOwner(dashboard, dashboard.getOwner());
@ -126,30 +140,12 @@ public class DashboardRepository extends EntityRepository<Dashboard> {
return new DashboardUpdater(original, updated, operation);
}
private List<EntityReference> getCharts(Dashboard dashboard) throws IOException {
private List<EntityReference> getRelatedEntities(Dashboard dashboard, String entityType) throws IOException {
if (dashboard == null) {
return Collections.emptyList();
}
List<EntityRelationshipRecord> chartIds =
findTo(dashboard.getId(), Entity.DASHBOARD, Relationship.HAS, Entity.CHART);
return EntityUtil.populateEntityReferences(chartIds, Entity.CHART);
}
/**
* This method is used to populate the dashboard entity with all details of Chart EntityReference Users/Tools can send
* minimum details required to set relationship as id, type are the only required fields in entity reference, whereas
* we need to send fully populated object such that ElasticSearch index has all the details.
*/
private List<EntityReference> getCharts(List<EntityReference> charts) throws IOException {
if (nullOrEmpty(charts)) {
return Collections.emptyList();
}
List<EntityReference> chartRefs = new ArrayList<>();
for (EntityReference chart : charts) {
EntityReference chartRef = Entity.getEntityReference(chart, Include.NON_DELETED);
chartRefs.add(chartRef);
}
return chartRefs.isEmpty() ? null : chartRefs;
List<EntityRelationshipRecord> ids = findTo(dashboard.getId(), Entity.DASHBOARD, Relationship.HAS, entityType);
return EntityUtil.populateEntityReferences(ids, entityType);
}
/** Handles entity updated from PUT and POST operation. */
@ -160,23 +156,28 @@ public class DashboardRepository extends EntityRepository<Dashboard> {
@Override
public void entitySpecificUpdate() throws IOException {
updateCharts();
update(Entity.CHART, "charts", listOrEmpty(updated.getCharts()), listOrEmpty(original.getCharts()));
update(
Entity.DASHBOARD_DATA_MODEL,
"dataModels",
listOrEmpty(updated.getDataModels()),
listOrEmpty(original.getDataModels()));
}
private void updateCharts() throws JsonProcessingException {
// Remove all charts associated with this dashboard
deleteFrom(updated.getId(), Entity.DASHBOARD, Relationship.HAS, Entity.CHART);
private void update(
String entityType, String field, List<EntityReference> updEntities, List<EntityReference> oriEntities)
throws JsonProcessingException {
// Remove all entity type associated with this dashboard
deleteFrom(updated.getId(), Entity.DASHBOARD, Relationship.HAS, entityType);
// Add relationship from dashboard to chart
List<EntityReference> updatedCharts = listOrEmpty(updated.getCharts());
List<EntityReference> origCharts = listOrEmpty(original.getCharts());
for (EntityReference chart : updatedCharts) {
addRelationship(updated.getId(), chart.getId(), Entity.DASHBOARD, Entity.CHART, Relationship.HAS);
// Add relationship from dashboard to entity type
for (EntityReference entity : updEntities) {
addRelationship(updated.getId(), entity.getId(), Entity.DASHBOARD, entityType, Relationship.HAS);
}
List<EntityReference> added = new ArrayList<>();
List<EntityReference> deleted = new ArrayList<>();
recordListChange("charts", origCharts, updatedCharts, added, deleted, EntityUtil.entityReferenceMatch);
recordListChange(field, oriEntities, updEntities, added, deleted, EntityUtil.entityReferenceMatch);
}
}
}

6
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/LineageRepository.java
View File

@ -100,13 +100,13 @@ public class LineageRepository {
for (String fromColumn : columnLineage.getFromColumns()) {
// From column belongs to the fromNode
if (fromColumn.startsWith(fromTable.getFullyQualifiedName())) {
TableRepository.validateColumnFQN(fromTable, fromColumn);
ColumnUtil.validateColumnFQN(fromTable.getColumns(), fromColumn);
} else {
Table otherTable = dao.tableDAO().findEntityByName(FullyQualifiedName.getTableFQN(fromColumn));
TableRepository.validateColumnFQN(otherTable, fromColumn);
ColumnUtil.validateColumnFQN(otherTable.getColumns(), fromColumn);
}
}
TableRepository.validateColumnFQN(toTable, columnLineage.getToColumn());
ColumnUtil.validateColumnFQN(toTable.getColumns(), columnLineage.getToColumn());
}
}
return JsonUtils.pojoToJson(details);

1
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/QueryRepository.java
View File

@ -76,7 +76,6 @@ public class QueryRepository extends EntityRepository<Query> {
entity.setChecksum(checkSum);
entity.setName(checkSum);
}
entity.setUsers(EntityUtil.populateEntityReferences(entity.getUsers()));
}

53
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java
View File

@ -68,7 +68,6 @@ import org.openmetadata.schema.type.TableProfile;
import org.openmetadata.schema.type.TableProfilerConfig;
import org.openmetadata.schema.type.TagLabel;
import org.openmetadata.service.Entity;
import org.openmetadata.service.exception.CatalogExceptionMessage;
import org.openmetadata.service.exception.EntityNotFoundException;
import org.openmetadata.service.resources.databases.DatabaseUtil;
import org.openmetadata.service.resources.databases.TableResource;
@ -146,7 +145,7 @@ public class TableRepository extends EntityRepository<Table> {
public void setFullyQualifiedName(Table table) {
table.setFullyQualifiedName(
FullyQualifiedName.add(table.getDatabaseSchema().getFullyQualifiedName(), table.getName()));
setColumnFQN(table.getFullyQualifiedName(), table.getColumns());
ColumnUtil.setColumnFQN(table.getFullyQualifiedName(), table.getColumns());
}
@Transaction
@ -577,17 +576,6 @@ public class TableRepository extends EntityRepository<Table> {
deleteFrom(tableId, TABLE, Relationship.HAS, LOCATION);
}
private void setColumnFQN(String parentFQN, List<Column> columns) {
columns.forEach(
c -> {
String columnFqn = FullyQualifiedName.add(parentFQN, c.getName());
c.setFullyQualifiedName(columnFqn);
if (c.getChildren() != null) {
setColumnFQN(columnFqn, c.getChildren());
}
});
}
private void addDerivedColumnTags(List<Column> columns) {
if (nullOrEmpty(columns)) {
return;
@ -697,19 +685,6 @@ public class TableRepository extends EntityRepository<Table> {
}
}
private void getColumnProfile(boolean setProfile, List<Column> columns) throws IOException {
if (setProfile) {
for (Column c : listOrEmpty(columns)) {
c.setProfile(
JsonUtils.readValue(
daoCollection
.entityExtensionTimeSeriesDao()
.getLatestExtension(c.getFullyQualifiedName(), TABLE_COLUMN_PROFILE_EXTENSION),
ColumnProfile.class));
}
}
}
private void validateTableFQN(String fqn) {
try {
dao.existsByName(fqn);
@ -726,20 +701,6 @@ public class TableRepository extends EntityRepository<Table> {
}
}
// Validate if a given column exists in the table
public static void validateColumnFQN(Table table, String columnFQN) {
boolean validColumn = false;
for (Column column : table.getColumns()) {
if (column.getFullyQualifiedName().equals(columnFQN)) {
validColumn = true;
break;
}
}
if (!validColumn) {
throw new IllegalArgumentException(CatalogExceptionMessage.invalidColumnFQN(columnFQN));
}
}
private void validateColumnFQNs(List<JoinedWith> joinedWithList) {
for (JoinedWith joinedWith : joinedWithList) {
// Validate table
@ -747,7 +708,7 @@ public class TableRepository extends EntityRepository<Table> {
Table joinedWithTable = dao.findEntityByName(tableFQN);
// Validate column
validateColumnFQN(joinedWithTable, joinedWith.getFullyQualifiedName());
ColumnUtil.validateColumnFQN(joinedWithTable.getColumns(), joinedWith.getFullyQualifiedName());
}
}
@ -918,14 +879,6 @@ public class TableRepository extends EntityRepository<Table> {
return dc -> CommonUtil.dateInRange(RestUtil.DATE_FORMAT, dc.getDate(), 0, 30);
}
private TableProfile getTableProfile(Table table) throws IOException {
return JsonUtils.readValue(
daoCollection
.entityExtensionTimeSeriesDao()
.getLatestExtension(table.getFullyQualifiedName(), TABLE_PROFILE_EXTENSION),
TableProfile.class);
}
private List<CustomMetric> getCustomMetrics(Table table, String columnName) throws IOException {
String extension = TABLE_COLUMN_EXTENSION + columnName + CUSTOM_METRICS_EXTENSION;
return JsonUtils.readObjects(
@ -950,7 +903,7 @@ public class TableRepository extends EntityRepository<Table> {
public void entitySpecificUpdate() throws IOException {
Table origTable = original;
Table updatedTable = updated;
DatabaseUtil.validateColumns(updatedTable);
DatabaseUtil.validateColumns(updatedTable.getColumns());
recordChange("tableType", origTable.getTableType(), updatedTable.getTableType());
updateConstraints(origTable, updatedTable);
updateColumns("columns", origTable.getColumns(), updated.getColumns(), EntityUtil.columnMatch);

6
openmetadata-service/src/main/java/org/openmetadata/service/resources/dashboards/DashboardResource.java
View File

@ -66,12 +66,15 @@ import org.openmetadata.service.util.ResultList;
public class DashboardResource extends EntityResource<Dashboard, DashboardRepository> {
public static final String COLLECTION_PATH = "v1/dashboards/";
protected static final String FIELDS = "owner,charts,followers,tags,usageSummary,extension,dataModels";
@Override
public Dashboard addHref(UriInfo uriInfo, Dashboard dashboard) {
Entity.withHref(uriInfo, dashboard.getOwner());
Entity.withHref(uriInfo, dashboard.getService());
Entity.withHref(uriInfo, dashboard.getCharts());
Entity.withHref(uriInfo, dashboard.getFollowers());
Entity.withHref(uriInfo, dashboard.getDataModels());
return dashboard;
}
@ -86,8 +89,6 @@ public class DashboardResource extends EntityResource<Dashboard, DashboardReposi
}
}
static final String FIELDS = "owner,charts,followers,tags,usageSummary,extension";
@GET
@Valid
@Operation(
@ -433,6 +434,7 @@ public class DashboardResource extends EntityResource<Dashboard, DashboardReposi
return copy(new Dashboard(), create, user)
.withService(getEntityReference(Entity.DASHBOARD_SERVICE, create.getService()))
.withCharts(getEntityReferences(Entity.CHART, create.getCharts()))
.withDataModels(getEntityReferences(Entity.DASHBOARD_DATA_MODEL, create.getDataModels()))
.withDashboardUrl(create.getDashboardUrl())
.withTags(create.getTags());
}

7
openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/DatabaseUtil.java
View File

@ -18,7 +18,6 @@ import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import org.openmetadata.schema.entity.data.Table;
import org.openmetadata.schema.type.Column;
import org.openmetadata.schema.type.ColumnConstraint;
import org.openmetadata.schema.type.ColumnDataType;
@ -88,9 +87,9 @@ public final class DatabaseUtil {
}
}
public static void validateColumns(Table table) {
validateColumnNames(table.getColumns());
for (Column c : table.getColumns()) {
public static void validateColumns(List<Column> columns) {
validateColumnNames(columns);
for (Column c : columns) {
validateColumnDataTypeDisplay(c);
validateColumnDataLength(c);
validateArrayColumn(c);

2
openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/TableResource.java
View File

@ -980,7 +980,7 @@ public class TableResource extends EntityResource<Table, TableRepository> {
DatabaseUtil.validateConstraints(table.getColumns(), table.getTableConstraints());
DatabaseUtil.validateTablePartition(table.getColumns(), table.getTablePartition());
DatabaseUtil.validateViewDefinition(table.getTableType(), table.getViewDefinition());
DatabaseUtil.validateColumns(table);
DatabaseUtil.validateColumns(table.getColumns());
return table;
}

460
openmetadata-service/src/main/java/org/openmetadata/service/resources/datamodels/DashboardDataModelResource.java Normal file
View File

@ -0,0 +1,460 @@
/*
* Copyright 2021 Collate
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.openmetadata.service.resources.datamodels;
import io.swagger.annotations.Api;
import io.swagger.v3.oas.annotations.ExternalDocumentation;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.media.Content;
import io.swagger.v3.oas.annotations.media.ExampleObject;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.parameters.RequestBody;
import io.swagger.v3.oas.annotations.responses.ApiResponse;
import java.io.IOException;
import java.util.UUID;
import javax.json.JsonPatch;
import javax.validation.Valid;
import javax.validation.constraints.Max;
import javax.validation.constraints.Min;
import javax.ws.rs.Consumes;
import javax.ws.rs.DELETE;
import javax.ws.rs.DefaultValue;
import javax.ws.rs.GET;
import javax.ws.rs.PATCH;
import javax.ws.rs.POST;
import javax.ws.rs.PUT;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.SecurityContext;
import javax.ws.rs.core.UriInfo;
import org.openmetadata.schema.api.data.CreateDashboardDataModel;
import org.openmetadata.schema.api.data.RestoreEntity;
import org.openmetadata.schema.entity.data.DashboardDataModel;
import org.openmetadata.schema.type.EntityHistory;
import org.openmetadata.schema.type.Include;
import org.openmetadata.service.Entity;
import org.openmetadata.service.jdbi3.CollectionDAO;
import org.openmetadata.service.jdbi3.DashboardDataModelRepository;
import org.openmetadata.service.jdbi3.ListFilter;
import org.openmetadata.service.resources.Collection;
import org.openmetadata.service.resources.EntityResource;
import org.openmetadata.service.resources.databases.DatabaseUtil;
import org.openmetadata.service.security.Authorizer;
import org.openmetadata.service.util.EntityUtil;
import org.openmetadata.service.util.RestUtil;
import org.openmetadata.service.util.ResultList;
@Path("/v1/dashboard/datamodels")
@Api(value = "Data Model data asset collection", tags = "Data Model data asset collection")
@Produces(MediaType.APPLICATION_JSON)
@Consumes(MediaType.APPLICATION_JSON)
@Collection(name = "datamodels")
public class DashboardDataModelResource extends EntityResource<DashboardDataModel, DashboardDataModelRepository> {
public static final String COLLECTION_PATH = "/v1/dashboard/datamodels";
protected static final String FIELDS = "owner,tags,followers";
@Override
public DashboardDataModel addHref(UriInfo uriInfo, DashboardDataModel dashboardDataModel) {
dashboardDataModel.setHref(RestUtil.getHref(uriInfo, COLLECTION_PATH, dashboardDataModel.getId()));
Entity.withHref(uriInfo, dashboardDataModel.getOwner());
Entity.withHref(uriInfo, dashboardDataModel.getService());
Entity.withHref(uriInfo, dashboardDataModel.getFollowers());
return dashboardDataModel;
}
public DashboardDataModelResource(CollectionDAO dao, Authorizer authorizer) {
super(DashboardDataModel.class, new DashboardDataModelRepository(dao), authorizer);
}
public static class DashboardDataModelList extends ResultList<DashboardDataModel> {
@SuppressWarnings("unused")
DashboardDataModelList() {
// Empty constructor needed for deserialization
}
}
@GET
@Operation(
operationId = "listDashboardDataModels",
summary = "List Dashboard Data Models",
tags = "dashboardDataModel",
description =
"Get a list of dashboard datamodels, optionally filtered by `service` it belongs to. Use `fields` "
+ "parameter to get only necessary fields. Use cursor-based pagination to limit the number "
+ "entries in the list using `limit` and `before` or `after` query params.",
responses = {
@ApiResponse(
responseCode = "200",
description = "List of dashboard datamodels",
content =
@Content(
mediaType = "application/json",
schema = @Schema(implementation = DashboardDataModelList.class)))
})
public ResultList<DashboardDataModel> list(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(
description = "Fields requested in the returned resource",
schema = @Schema(type = "string", example = FIELDS))
@QueryParam("fields")
String fieldsParam,
@Parameter(
description = "Filter dashboardDataModel by service name",
schema = @Schema(type = "string", example = "superset"))
@QueryParam("service")
String serviceParam,
@Parameter(description = "Limit the number dashboardDataModel returned. (1 to 1000000, default = 10)")
@DefaultValue("10")
@QueryParam("limit")
@Min(0)
@Max(1000000)
int limitParam,
@Parameter(
description = "Returns list of dashboardDataModel before this cursor",
schema = @Schema(type = "string"))
@QueryParam("before")
String before,
@Parameter(
description = "Returns list of dashboardDataModel after this cursor",
schema = @Schema(type = "string"))
@QueryParam("after")
String after,
@Parameter(
description = "Include all, deleted, or non-deleted entities.",
schema = @Schema(implementation = Include.class))
@QueryParam("include")
@DefaultValue("non-deleted")
Include include)
throws IOException {
ListFilter filter = new ListFilter(include).addQueryParam("service", serviceParam);
return super.listInternal(uriInfo, securityContext, fieldsParam, filter, limitParam, before, after);
}
@GET
@Path("/{id}/versions")
@Operation(
operationId = "listAllDataModelVersions",
summary = "List dashboard datamodel versions",
tags = "dashboardDataModel",
description = "Get a list of all the versions of a dashboard datamodel identified by `id`",
responses = {
@ApiResponse(
responseCode = "200",
description = "List of dashboard datamodel versions",
content = @Content(mediaType = "application/json", schema = @Schema(implementation = EntityHistory.class)))
})
public EntityHistory listVersions(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(description = "Id of the dashboard datamodel", schema = @Schema(type = "UUID")) @PathParam("id")
UUID id)
throws IOException {
return super.listVersionsInternal(securityContext, id);
}
@GET
@Path("/{id}")
@Operation(
operationId = "getDataModelByID",
summary = "Get a dashboard datamodel by Id",
tags = "dashboardDataModel",
description = "Get a dashboard datamodel by `id`.",
responses = {
@ApiResponse(
responseCode = "200",
description = "The dashboard datamodel",
content =
@Content(mediaType = "application/json", schema = @Schema(implementation = DashboardDataModel.class))),
@ApiResponse(responseCode = "404", description = "DataModel for instance {id} is not found")
})
public DashboardDataModel get(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(description = "Id of the dashboard datamodel", schema = @Schema(type = "UUID")) @PathParam("id")
UUID id,
@Parameter(
description = "Fields requested in the returned resource",
schema = @Schema(type = "string", example = FIELDS))
@QueryParam("fields")
String fieldsParam,
@Parameter(
description = "Include all, deleted, or non-deleted entities.",
schema = @Schema(implementation = Include.class))
@QueryParam("include")
@DefaultValue("non-deleted")
Include include)
throws IOException {
return getInternal(uriInfo, securityContext, id, fieldsParam, include);
}
@GET
@Path("/name/{fqn}")
@Operation(
operationId = "getDataModelByFQN",
summary = "Get a dashboard datamodel by fully qualified name",
tags = "dashboardDataModel",
description = "Get a dashboard datamodel by `fullyQualifiedName`.",
responses = {
@ApiResponse(
responseCode = "200",
description = "The dashboard datamodel",
content =
@Content(mediaType = "application/json", schema = @Schema(implementation = DashboardDataModel.class))),
@ApiResponse(responseCode = "404", description = "DataModel for instance {fqn} is not found")
})
public DashboardDataModel getByName(
@Context UriInfo uriInfo,
@Parameter(description = "Fully qualified name of the dashboard datamodel", schema = @Schema(type = "string"))
@PathParam("fqn")
String fqn,
@Context SecurityContext securityContext,
@Parameter(
description = "Fields requested in the returned resource",
schema = @Schema(type = "string", example = FIELDS))
@QueryParam("fields")
String fieldsParam,
@Parameter(
description = "Include all, deleted, or non-deleted entities.",
schema = @Schema(implementation = Include.class))
@QueryParam("include")
@DefaultValue("non-deleted")
Include include)
throws IOException {
return getByNameInternal(uriInfo, securityContext, fqn, fieldsParam, include);
}
@GET
@Path("/{id}/versions/{version}")
@Operation(
operationId = "getSpecificDataModelVersion",
summary = "Get a version of the dashboard datamodel",
tags = "dashboardDataModel",
description = "Get a version of the dashboard datamodel by given `id`",
responses = {
@ApiResponse(
responseCode = "200",
description = "dashboard datamodel",
content =
@Content(mediaType = "application/json", schema = @Schema(implementation = DashboardDataModel.class))),
@ApiResponse(
responseCode = "404",
description = "DataModel for instance {id} and version {version} is " + "not found")
})
public DashboardDataModel getVersion(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(description = "Id of the dashboard datamodel", schema = @Schema(type = "UUID")) @PathParam("id")
UUID id,
@Parameter(
description = "DataModel version number in the form `major`.`minor`",
schema = @Schema(type = "string", example = "0.1 or 1.1"))
@PathParam("version")
String version)
throws IOException {
return super.getVersionInternal(securityContext, id, version);
}
@POST
@Operation(
operationId = "createDataModel",
summary = "Create a dashboard datamodel",
tags = "dashboardDataModel",
description = "Create a dashboard datamodel under an existing `service`.",
responses = {
@ApiResponse(
responseCode = "200",
description = "The dashboard datamodel",
content =
@Content(mediaType = "application/json", schema = @Schema(implementation = DashboardDataModel.class))),
@ApiResponse(responseCode = "400", description = "Bad request")
})
public Response create(
@Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateDashboardDataModel create)
throws IOException {
DashboardDataModel dashboardDataModel = getDataModel(create, securityContext.getUserPrincipal().getName());
return create(uriInfo, securityContext, dashboardDataModel);
}
@PATCH
@Path("/{id}")
@Operation(
operationId = "patchDataModel",
summary = "Update a dashboard datamodel",
tags = "dashboardDataModel",
description = "Update an existing dashboard datamodel using JsonPatch.",
externalDocs = @ExternalDocumentation(description = "JsonPatch RFC", url = "https://tools.ietf.org/html/rfc6902"))
@Consumes(MediaType.APPLICATION_JSON_PATCH_JSON)
public Response patch(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(description = "Id of the dashboard datamodel", schema = @Schema(type = "UUID")) @PathParam("id")
UUID id,
@RequestBody(
description = "JsonPatch with array of operations",
content =
@Content(
mediaType = MediaType.APPLICATION_JSON_PATCH_JSON,
examples = {
@ExampleObject("[" + "{op:remove, path:/a}," + "{op:add, path: /b, value: val}" + "]")
}))
JsonPatch patch)
throws IOException {
return patchInternal(uriInfo, securityContext, id, patch);
}
@PUT
@Operation(
operationId = "createOrUpdateDataModel",
summary = "Create or update dashboard datamodel",
tags = "dashboardDataModel",
description = "Create a dashboard datamodel, it it does not exist or update an existing dashboard datamodel.",
responses = {
@ApiResponse(
responseCode = "200",
description = "The updated dashboard datamodel",
content =
@Content(mediaType = "application/json", schema = @Schema(implementation = DashboardDataModel.class)))
})
public Response createOrUpdate(
@Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateDashboardDataModel create)
throws IOException {
DashboardDataModel dashboardDataModel = getDataModel(create, securityContext.getUserPrincipal().getName());
return createOrUpdate(uriInfo, securityContext, dashboardDataModel);
}
@PUT
@Path("/{id}/followers")
@Operation(
operationId = "addFollowerToDataModel",
summary = "Add a follower",
tags = "dashboardDataModel",
description = "Add a user identified by `userId` as followed of this data model",
responses = {
@ApiResponse(responseCode = "200", description = "OK"),
@ApiResponse(responseCode = "404", description = "DataModel for instance {id} is not found")
})
public Response addFollower(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(description = "Id of the data model", schema = @Schema(type = "UUID")) @PathParam("id") UUID id,
@Parameter(description = "Id of the user to be added as follower", schema = @Schema(type = "UUID")) UUID userId)
throws IOException {
return dao.addFollower(securityContext.getUserPrincipal().getName(), id, userId).toResponse();
}
@DELETE
@Path("/{id}/followers/{userId}")
@Operation(
operationId = "deleteFollowerFromDataModel",
summary = "Remove a follower",
tags = "dashboardDataModel",
description = "Remove the user identified `userId` as a follower of the data model.")
public Response deleteFollower(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(description = "Id of the data model", schema = @Schema(type = "UUID")) @PathParam("id") UUID id,
@Parameter(description = "Id of the user being removed as follower", schema = @Schema(type = "UUID"))
@PathParam("userId")
UUID userId)
throws IOException {
return dao.deleteFollower(securityContext.getUserPrincipal().getName(), id, userId).toResponse();
}
@DELETE
@Path("/{id}")
@Operation(
operationId = "deleteDataModel",
summary = "Delete a data model by `id`.",
tags = "dashboardDataModel",
description = "Delete a dashboard datamodel by `id`.",
responses = {
@ApiResponse(responseCode = "200", description = "OK"),
@ApiResponse(responseCode = "404", description = "DataModel for instance {id} is not found")
})
public Response delete(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(description = "Hard delete the entity. (Default = `false`)")
@QueryParam("hardDelete")
@DefaultValue("false")
boolean hardDelete,
@Parameter(description = "Id of the data model", schema = @Schema(type = "UUID")) @PathParam("id") UUID id)
throws IOException {
return delete(uriInfo, securityContext, id, false, hardDelete);
}
@DELETE
@Path("/name/{fqn}")
@Operation(
operationId = "deleteDataModelByFQN",
summary = "Delete a data model by fully qualified name.",
tags = "dashboardDataModel",
description = "Delete a data model by `fullyQualifiedName`.",
responses = {
@ApiResponse(responseCode = "200", description = "OK"),
@ApiResponse(responseCode = "404", description = "DataModel for instance {fqn} is not found")
})
public Response delete(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(description = "Hard delete the entity. (Default = `false`)")
@QueryParam("hardDelete")
@DefaultValue("false")
boolean hardDelete,
@Parameter(description = "Fully qualified name of the data model", schema = @Schema(type = "string"))
@PathParam("fqn")
String fqn)
throws IOException {
return deleteByName(uriInfo, securityContext, fqn, false, hardDelete);
}
@PUT
@Path("/restore")
@Operation(
operationId = "restore",
summary = "Restore a soft deleted data model.",
tags = "dashboardDataModel",
description = "Restore a soft deleted data model.",
responses = {
@ApiResponse(
responseCode = "200",
description = "Successfully restored the data model",
content =
@Content(mediaType = "application/json", schema = @Schema(implementation = DashboardDataModel.class)))
})
public Response restoreDataModel(
@Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid RestoreEntity restore)
throws IOException {
return restoreEntity(uriInfo, securityContext, restore.getId());
}
private DashboardDataModel getDataModel(CreateDashboardDataModel create, String user) throws IOException {
DatabaseUtil.validateColumns(create.getColumns());
return copy(new DashboardDataModel(), create, user)
.withService(EntityUtil.getEntityReference(Entity.DASHBOARD_SERVICE, create.getService()))
.withDataModelType(create.getDataModelType())
.withSql(create.getSql())
.withDataModelType(create.getDataModelType())
.withServiceType(create.getServiceType())
.withColumns(create.getColumns())
.withTags(create.getTags());
}
}

19
openmetadata-service/src/main/java/org/openmetadata/service/util/EntityUtil.java
View File

@ -46,6 +46,7 @@ import org.openmetadata.schema.type.Column;
import org.openmetadata.schema.type.EntityReference;
import org.openmetadata.schema.type.Field;
import org.openmetadata.schema.type.FieldChange;
import org.openmetadata.schema.type.Include;
import org.openmetadata.schema.type.MetadataOperation;
import org.openmetadata.schema.type.MlFeature;
import org.openmetadata.schema.type.MlHyperParameter;
@ -486,4 +487,22 @@ public final class EntityUtil {
// Sort tags by tag hierarchy. Tags with parents null come first, followed by tags with
tags.sort(Comparator.comparing(Tag::getFullyQualifiedName));
}
/**
* This method is used to populate the entity with all details of EntityReference Users/Tools can send minimum details
* required to set relationship as id, type are the only required fields in entity reference, whereas we need to send
* fully populated object such that ElasticSearch index has all the details.
*/
public static List<EntityReference> getEntityReferences(List<EntityReference> entities, Include include)
throws IOException {
if (nullOrEmpty(entities)) {
return Collections.emptyList();
}
List<EntityReference> refs = new ArrayList<>();
for (EntityReference entityReference : entities) {
EntityReference entityRef = Entity.getEntityReference(entityReference, include);
refs.add(entityRef);
}
return refs;
}
}

10
openmetadata-service/src/main/resources/json/data/EventSubResourceDescriptor.json
View File

@ -383,5 +383,15 @@
"matchUpdatedBy",
"matchAnyFieldChange"
]
},
{
"name" : "dashboardDataModel",
"supportedFilters" : [
"matchAnyOwnerName",
"matchAnyEntityFqn",
"matchAnyEventType",
"matchUpdatedBy",
"matchAnyFieldChange"
]
}
]

29
openmetadata-service/src/main/resources/json/data/ResourceDescriptors.json
View File

@ -548,5 +548,34 @@
"ViewAll",
"EditDescription"
]
},
{
"name": "dataModel",
"operations": [
"Create",
"Delete",
"ViewAll",
"EditAll",
"EditDescription",
"EditDisplayName",
"EditTags",
"EditOwner",
"EditLineage"
]
},
{
"name": "dashboardDataModel",
"operations": [
"Create",
"Delete",
"ViewAll",
"ViewUsage",
"EditAll",
"EditDescription",
"EditDisplayName",
"EditTags",
"EditOwner",
"EditLineage"
]
}
]

147
openmetadata-service/src/test/java/org/openmetadata/service/resources/datamodels/DashboardDataModelResourceTest.java Normal file
View File

@ -0,0 +1,147 @@
/*
* Copyright 2021 Collate
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.openmetadata.service.resources.datamodels;
import static javax.ws.rs.core.Response.Status.BAD_REQUEST;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.openmetadata.service.util.TestUtils.ADMIN_AUTH_HEADERS;
import static org.openmetadata.service.util.TestUtils.assertListNotEmpty;
import static org.openmetadata.service.util.TestUtils.assertListNotNull;
import static org.openmetadata.service.util.TestUtils.assertListNull;
import static org.openmetadata.service.util.TestUtils.assertResponse;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.HttpResponseException;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInfo;
import org.junit.jupiter.api.parallel.Execution;
import org.junit.jupiter.api.parallel.ExecutionMode;
import org.openmetadata.schema.api.data.CreateDashboardDataModel;
import org.openmetadata.schema.entity.data.DashboardDataModel;
import org.openmetadata.schema.type.DataModelType;
import org.openmetadata.schema.type.EntityReference;
import org.openmetadata.service.Entity;
import org.openmetadata.service.resources.EntityResourceTest;
import org.openmetadata.service.util.ResultList;
@Slf4j
public class DashboardDataModelResourceTest extends EntityResourceTest<DashboardDataModel, CreateDashboardDataModel> {
public DashboardDataModelResourceTest() {
super(
Entity.DASHBOARD_DATA_MODEL,
DashboardDataModel.class,
DashboardDataModelResource.DashboardDataModelList.class,
"dashboard/datamodels",
DashboardDataModelResource.FIELDS);
}
@Test
@Execution(ExecutionMode.CONCURRENT)
void post_dataModelWithoutRequiredFields_4xx(TestInfo test) {
// Service is required field
assertResponse(
() -> createEntity(createRequest(test).withService(null), ADMIN_AUTH_HEADERS),
BAD_REQUEST,
"[service must not be null]");
}
@Test
@Execution(ExecutionMode.CONCURRENT)
void post_dataModelWithDifferentService_200_ok(TestInfo test) throws IOException {
String[] differentServices = {METABASE_REFERENCE.getName(), LOOKER_REFERENCE.getName()};
// Create dataModel for each service and test APIs
for (String service : differentServices) {
createAndCheckEntity(createRequest(test).withService(service), ADMIN_AUTH_HEADERS);
// List dataModels by filtering on service name and ensure right dataModels in the response
Map<String, String> queryParams = new HashMap<>();
queryParams.put("service", service);
ResultList<DashboardDataModel> list = listEntities(queryParams, ADMIN_AUTH_HEADERS);
for (DashboardDataModel dashboardDataModel : list.getData()) {
assertEquals(service, dashboardDataModel.getService().getName());
}
}
}
@Override
@Execution(ExecutionMode.CONCURRENT)
public DashboardDataModel validateGetWithDifferentFields(DashboardDataModel dashboardDataModel, boolean byName)
throws HttpResponseException {
String fields = "";
dashboardDataModel =
byName
? getEntityByName(dashboardDataModel.getFullyQualifiedName(), fields, ADMIN_AUTH_HEADERS)
: getEntity(dashboardDataModel.getId(), fields, ADMIN_AUTH_HEADERS);
assertListNotNull(dashboardDataModel.getService(), dashboardDataModel.getServiceType());
assertListNull(dashboardDataModel.getOwner(), dashboardDataModel.getFollowers(), dashboardDataModel.getTags());
// .../datamodels?fields=owner
fields = "owner,followers,tags";
dashboardDataModel =
byName
? getEntityByName(dashboardDataModel.getFullyQualifiedName(), fields, ADMIN_AUTH_HEADERS)
: getEntity(dashboardDataModel.getId(), fields, ADMIN_AUTH_HEADERS);
assertListNotNull(dashboardDataModel.getService(), dashboardDataModel.getServiceType());
// Checks for other owner, tags, and followers is done in the base class
return dashboardDataModel;
}
@Override
public CreateDashboardDataModel createRequest(String name) {
return new CreateDashboardDataModel()
.withName(name)
.withService(getContainer().getName())
.withServiceType(CreateDashboardDataModel.DashboardServiceType.Metabase)
.withSql("SELECT * FROM tab1;")
.withDataModelType(DataModelType.MetabaseDataModel)
.withColumns(COLUMNS);
}
@Override
public EntityReference getContainer() {
return METABASE_REFERENCE;
}
@Override
public EntityReference getContainer(DashboardDataModel entity) {
return entity.getService();
}
@Override
public void validateCreatedEntity(
DashboardDataModel dashboardDataModel, CreateDashboardDataModel createRequest, Map<String, String> authHeaders) {
assertNotNull(dashboardDataModel.getServiceType());
assertReference(createRequest.getService(), dashboardDataModel.getService());
assertEquals(createRequest.getSql(), dashboardDataModel.getSql());
assertEquals(createRequest.getDataModelType(), dashboardDataModel.getDataModelType());
assertListNotEmpty(dashboardDataModel.getColumns());
}
@Override
public void compareEntities(
DashboardDataModel expected, DashboardDataModel patched, Map<String, String> authHeaders) {
assertReference(expected.getService(), patched.getService());
}
@Override
public void assertFieldChange(String fieldName, Object expected, Object actual) throws IOException {
assertCommonFieldChange(fieldName, expected, actual);
}
}

8
openmetadata-service/src/test/java/org/openmetadata/service/resources/services/DashboardServiceResourceTest.java
View File

@ -39,8 +39,8 @@ import org.apache.http.client.HttpResponseException;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInfo;
import org.openmetadata.schema.api.data.CreateChart;
import org.openmetadata.schema.api.data.CreateDashboardDataModel.DashboardServiceType;
import org.openmetadata.schema.api.services.CreateDashboardService;
import org.openmetadata.schema.api.services.CreateDashboardService.DashboardServiceType;
import org.openmetadata.schema.entity.data.Chart;
import org.openmetadata.schema.entity.services.DashboardService;
import org.openmetadata.schema.entity.services.connections.TestConnectionResult;
@ -187,7 +187,7 @@ public class DashboardServiceResourceTest extends EntityResourceTest<DashboardSe
try {
return new CreateDashboardService()
.withName(name)
.withServiceType(CreateDashboardService.DashboardServiceType.Metabase)
.withServiceType(DashboardServiceType.Metabase)
.withConnection(
new DashboardConnection()
.withConfig(
@ -207,7 +207,7 @@ public class DashboardServiceResourceTest extends EntityResourceTest<DashboardSe
try {
return new CreateDashboardService()
.withName(name)
.withServiceType(CreateDashboardService.DashboardServiceType.Metabase)
.withServiceType(DashboardServiceType.Metabase)
.withConnection(
new DashboardConnection()
.withConfig(
@ -269,7 +269,7 @@ public class DashboardServiceResourceTest extends EntityResourceTest<DashboardSe
DashboardServiceType dashboardServiceType,
Map<String, String> authHeaders) {
if (expectedDashboardConnection != null && actualDashboardConnection != null) {
if (dashboardServiceType == CreateDashboardService.DashboardServiceType.Metabase) {
if (dashboardServiceType == DashboardServiceType.Metabase) {
MetabaseConnection expectedmetabaseConnection = (MetabaseConnection) expectedDashboardConnection.getConfig();
MetabaseConnection actualMetabaseConnection;
if (actualDashboardConnection.getConfig() instanceof MetabaseConnection) {

8
openmetadata-spec/src/main/resources/json/schema/api/data/createDashboard.json
View File

@ -31,6 +31,14 @@
},
"default": null
},
"dataModels": {
"description": "List of fully qualified name of data models included in this Dashboard.",
"type": "array",
"items": {
"$ref": "../../type/basic.json#/definitions/fullyQualifiedEntityName"
},
"default": null
},
"tags": {
"description": "Tags for this dashboard",
"type": "array",

61
openmetadata-spec/src/main/resources/json/schema/api/data/createDashboardDataModel.json Normal file
View File

@ -0,0 +1,61 @@
{
"$id": "https://open-metadata.org/schema/api/data/createDashboardDataModel.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "CreateDashboardDataModelRequest",
"description": "Create Dashboard Data Model entity request.",
"type": "object",
"javaType": "org.openmetadata.schema.api.data.CreateDashboardDataModel",
"javaInterfaces": ["org.openmetadata.schema.CreateEntity"],
"properties": {
"name": {
"description": "Name that identifies this data model.",
"$ref": "../../type/basic.json#/definitions/entityName"
},
"displayName": {
"description": "Display Name that identifies this data model. It could be title or label from the source services.",
"type": "string"
},
"description": {
"description": "Description of the data model instance. What it has and how to use it.",
"$ref": "../../type/basic.json#/definitions/markdown"
},
"tags": {
"description": "Tags for this data model.",
"type": "array",
"items": {
"$ref": "../../type/tagLabel.json"
},
"default": null
},
"owner": {
"description": "Owner of this data model.",
"$ref": "../../type/entityReference.json"
},
"service": {
"description": "Link to the data model service where this data model is hosted in.",
"$ref": "../../type/basic.json#/definitions/fullyQualifiedEntityName"
},
"serviceType": {
"description": "Service type where this data model is hosted in.",
"$ref": "../../entity/services/dashboardService.json#/definitions/dashboardServiceType"
},
"dataModelType": {
"$ref": "../../entity/data/dashboardDataModel.json#/definitions/dataModelType"
},
"sql": {
"description": "In case the Data Model is based on a SQL query.",
"$ref": "../../type/basic.json#/definitions/sqlQuery",
"default": null
},
"columns": {
"description": "Columns from the data model.",
"type": "array",
"items": {
"$ref": "../../entity/data/table.json#/definitions/column"
},
"default": null
}
},
"required": ["name", "service", "modelType"],
"additionalProperties": false
}

1
openmetadata-spec/src/main/resources/json/schema/api/services/createDashboardService.json
View File

@ -6,7 +6,6 @@
"type": "object",
"javaType": "org.openmetadata.schema.api.services.CreateDashboardService",
"javaInterfaces": ["org.openmetadata.schema.CreateEntity"],
"properties": {
"name": {
"description": "Name that identifies the this entity instance uniquely",

10
openmetadata-spec/src/main/resources/json/schema/entity/data/dashboard.json
View File

@ -46,10 +46,12 @@
},
"charts": {
"description": "All the charts included in this Dashboard.",
"type": "array",
"items": {
"$ref": "../../type/entityReference.json"
},
"$ref": "../../type/entityReference.json#/definitions/entityReferenceList",
"default": null
},
"dataModels": {
"description": "List of data models used by this dashboard or the charts contained on it.",
"$ref": "../../type/entityReference.json#/definitions/entityReferenceList",
"default": null
},
"href": {

131
openmetadata-spec/src/main/resources/json/schema/entity/data/dashboardDataModel.json Normal file
View File

@ -0,0 +1,131 @@
{
"$id": "https://open-metadata.org/schema/entity/data/dashboardDataModel.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "DashboardDataModel",
"$comment": "@om-entity-type",
"description": "Dashboard Data Model entity definition. Data models are the schemas used to build dashboards, charts, or other data models.",
"type": "object",
"javaType": "org.openmetadata.schema.entity.data.DashboardDataModel",
"javaInterfaces": ["org.openmetadata.schema.EntityInterface"],
"definitions": {
"dataModelType": {
"javaType": "org.openmetadata.schema.type.DataModelType",
"description": "This schema defines the type used for describing different types of data models.",
"type": "string",
"$comment": "Data Model types supported.",
"enum": [
"TableauSheet",
"SupersetDataModel",
"MetabaseDataModel"
],
"javaEnums": [
{
"name": "TableauSheet"
},
{
"name": "SupersetDataModel"
},
{
"name": "MetabaseDataModel"
}
]
}
},
"properties": {
"id": {
"description": "Unique identifier of this data model instance.",
"$ref": "../../type/basic.json#/definitions/uuid"
},
"name": {
"description": "Name of a data model. Expected to be unique within a Dashboard.",
"$ref": "../../type/basic.json#/definitions/entityName"
},
"displayName": {
"description": "Display Name that identifies this data model. It could be title or label from the source.",
"type": "string"
},
"fullyQualifiedName": {
"description": "Fully qualified name of a data model in the form `serviceName.dashboardName.datamodel.datamodelName`.",
"$ref": "../../type/basic.json#/definitions/fullyQualifiedEntityName"
},
"description": {
"description": "Description of a data model.",
"$ref": "../../type/basic.json#/definitions/markdown"
},
"version": {
"description": "Metadata version of the entity.",
"$ref": "../../type/entityHistory.json#/definitions/entityVersion"
},
"updatedAt": {
"description": "Last update time corresponding to the new version of the entity in Unix epoch time milliseconds.",
"$ref": "../../type/basic.json#/definitions/timestamp"
},
"updatedBy": {
"description": "User who made the update.",
"type": "string"
},
"href": {
"description": "Link to this data model entity.",
"$ref": "../../type/basic.json#/definitions/href"
},
"owner": {
"description": "Owner of this data model.",
"$ref": "../../type/entityReference.json",
"default": null
},
"tags": {
"description": "Tags for this data model.",
"type": "array",
"items": {
"$ref": "../../type/tagLabel.json"
},
"default": null
},
"changeDescription": {
"description": "Change that lead to this version of the entity.",
"$ref": "../../type/entityHistory.json#/definitions/changeDescription"
},
"deleted": {
"description": "When `true` indicates the entity has been soft deleted.",
"type": "boolean",
"default": false
},
"followers": {
"description": "Followers of this dashboard.",
"$ref": "../../type/entityReference.json#/definitions/entityReferenceList"
},
"service": {
"description": "Link to service where this data model is hosted in.",
"$ref": "../../type/entityReference.json"
},
"serviceType": {
"description": "Service type where this data model is hosted in.",
"$ref": "../services/dashboardService.json#/definitions/dashboardServiceType"
},
"dataModelType": {
"$ref": "#/definitions/dataModelType"
},
"sql": {
"description": "In case the Data Model is based on a SQL query.",
"$ref": "../../type/basic.json#/definitions/sqlQuery",
"default": null
},
"columns": {
"description": "Columns from the data model.",
"type": "array",
"items": {
"$ref": "table.json#/definitions/column"
},
"default": null
},
"dataModels": {
"description": "List of data models used by this data model.",
"$ref": "../../type/entityReference.json#/definitions/entityReferenceList"
}
},
"required": [
"name",
"modelType"
],
"additionalProperties": false
}