yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-08-15 20:46:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

Removed datamodels references from dashboard and datamodels (#12371)

Browse Source
This commit is contained in:
Onkar Ravgan 2023-07-13 14:26:28 +05:30 committed by GitHub
parent 90160ac761
commit e200a42926
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 11 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
bootstrap/sql/com.mysql.cj.jdbc.Driver/v014__create_db_connection_info.sql
View File

@ -25,3 +25,7 @@ CREATE TABLE IF NOT EXISTS data_product_entity (
UPDATE dbservice_entity
SET json = JSON_REPLACE(json, '$.connection.config.scheme', 'hive')
WHERE JSON_EXTRACT(json, '$.connection.config.scheme') IN ('impala', 'impala4');
-- remove the dataModel references from Data Models
UPDATE dashboard_data_model_entity
SET json = JSON_REMOVE(json, '$.dataModels');

3
bootstrap/sql/org.postgresql.Driver/v014__create_db_connection_info.sql
View File

@ -25,3 +25,6 @@ CREATE TABLE IF NOT EXISTS data_product_entity (
update dbservice_entity
set json = jsonb_set(json::jsonb, '{connection,config,scheme}', '"hive"')
where json#>>'{connection,config,scheme}' in ('impala', 'impala4');
-- remove the dataModel references from Data Models
UPDATE dashboard_data_model_entity SET json = json #- '{dataModels}';

2
ingestion/src/metadata/ingestion/source/dashboard/tableau/models.py
View File

@ -103,7 +103,7 @@ class DatasourceField(BaseModel):
class Workbook(BaseModel):
id: str
luid: str
name: str
name: Optional[str]
class UpstreamTableColumn(BaseModel):

20
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DashboardDataModelRepository.java
View File

@ -19,7 +19,6 @@ import static org.openmetadata.service.Entity.FIELD_TAGS;
import com.fasterxml.jackson.core.JsonProcessingException;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
import javax.json.JsonPatch;
import lombok.SneakyThrows;
@ -45,8 +44,6 @@ import org.openmetadata.service.util.JsonUtils;
@Slf4j
public class DashboardDataModelRepository extends EntityRepository<DashboardDataModel> {
private static final String DATA_MODELS_FIELD = "dataModels";
private static final String DATA_MODEL_UPDATE_FIELDS = "owner,tags,followers";
private static final String DATA_MODEL_PATCH_FIELDS = "owner,tags,followers";
@ -116,16 +113,15 @@ public class DashboardDataModelRepository extends EntityRepository<DashboardData
// Relationships and fields such as href are derived and not stored as part of json
EntityReference owner = dashboardDataModel.getOwner();
List<TagLabel> tags = dashboardDataModel.getTags();
List<EntityReference> dataModels = dashboardDataModel.getDataModels();
EntityReference service = dashboardDataModel.getService();
// Don't store owner, database, href and tags as JSON. Build it on the fly based on relationships
dashboardDataModel.withOwner(null).withService(null).withHref(null).withTags(null).withDataModels(null);
dashboardDataModel.withOwner(null).withService(null).withHref(null).withTags(null);
store(dashboardDataModel, update);
// Restore the relationships
dashboardDataModel.withOwner(owner).withService(service).withTags(tags).withDataModels(dataModels);
dashboardDataModel.withOwner(owner).withService(service).withTags(tags);
}
@Override
@ -146,8 +142,7 @@ public class DashboardDataModelRepository extends EntityRepository<DashboardData
return dashboardDataModel
.withService(getContainer(dashboardDataModel.getId()))
.withFollowers(fields.contains(FIELD_FOLLOWERS) ? getFollowers(dashboardDataModel) : null)
.withTags(fields.contains(FIELD_TAGS) ? getTags(dashboardDataModel.getFullyQualifiedName()) : null)
.withDataModels(fields.contains(DATA_MODELS_FIELD) ? getDataModels(dashboardDataModel) : null);
.withTags(fields.contains(FIELD_TAGS) ? getTags(dashboardDataModel.getFullyQualifiedName()) : null);
}
@Override
@ -160,15 +155,6 @@ public class DashboardDataModelRepository extends EntityRepository<DashboardData
.withId(original.getId());
}
protected List<EntityReference> getDataModels(DashboardDataModel dashboardDataModel) throws IOException {
if (dashboardDataModel == null) {
return Collections.emptyList();
}
List<CollectionDAO.EntityRelationshipRecord> tableIds =
findTo(dashboardDataModel.getId(), entityType, Relationship.USES, Entity.DASHBOARD_DATA_MODEL);
return EntityUtil.populateEntityReferences(tableIds, Entity.TABLE);
}
private void getColumnTags(boolean setTags, List<Column> columns) {
for (Column c : listOrEmpty(columns)) {
c.setTags(setTags ? getTags(c.getFullyQualifiedName()) : null);

4
openmetadata-spec/src/main/resources/json/schema/entity/data/dashboardDataModel.json
View File

@ -130,10 +130,6 @@
},
"default": null
},
"dataModels": {
"description": "List of data models used by this data model.",
"$ref": "../../type/entityReferenceList.json#/definitions/entityReferenceList"
},
"project": {
"description": "Name of the project / workspace / collection in which the dataModel is contained",
"type": "string"