yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-11-28 00:26:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #838 from open-metadata/issue837

Browse Source
This commit is contained in:
Suresh Srinivas 2021-10-17 22:22:12 -07:00 committed by GitHub
commit 4c8df43b4e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 159 additions and 225 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/ChartRepository.java
View File

@ -60,8 +60,8 @@ public abstract class ChartRepository {
private static final Fields CHART_UPDATE_FIELDS = new Fields(ChartResource.FIELD_LIST, "owner");
private static final Fields CHART_PATCH_FIELDS = new Fields(ChartResource.FIELD_LIST, "owner,service,tags");
public static String getFQN(EntityReference service, Chart chart) {
return (service.getName() + "." + chart.getName());
public static String getFQN(Chart chart) {
return (chart.getService().getName() + "." + chart.getName());
}
@CreateSqlObject
@ -137,8 +137,9 @@ public abstract class ChartRepository {
}
@Transaction
public Chart create(Chart chart, EntityReference service, EntityReference owner) throws IOException {
return createInternal(chart, service, owner);
public Chart create(Chart chart) throws IOException {
validateRelationships(chart);
return createInternal(chart);
}
@Transaction
@ -153,18 +154,14 @@ public abstract class ChartRepository {
}
@Transaction
public PutResponse<Chart> createOrUpdate(Chart updated, EntityReference service, EntityReference newOwner)
throws IOException {
getService(service); // Validate service
String fqn = getFQN(service, updated);
Chart stored = JsonUtils.readValue(chartDAO().findByFQN(fqn), Chart.class);
public PutResponse<Chart> createOrUpdate(Chart updated) throws IOException {
validateRelationships(updated);
Chart stored = JsonUtils.readValue(chartDAO().findByFQN(updated.getFullyQualifiedName()), Chart.class);
if (stored == null) { // Chart does not exist. Create a new one
return new PutResponse<>(Status.CREATED, createInternal(updated, service, newOwner));
return new PutResponse<>(Status.CREATED, createInternal(updated));
}
setFields(stored, CHART_UPDATE_FIELDS);
updated.setId(stored.getId());
validateRelationships(updated, service, newOwner);
ChartUpdater chartUpdater = new ChartUpdater(stored, updated, false);
chartUpdater.updateAll();
chartUpdater.store();
@ -180,17 +177,16 @@ public abstract class ChartRepository {
return updated;
}
public Chart createInternal(Chart chart, EntityReference service, EntityReference owner) throws IOException {
validateRelationships(chart, service, owner);
public Chart createInternal(Chart chart) throws IOException {
storeChart(chart, false);
addRelationships(chart);
return chart;
}
private void validateRelationships(Chart chart, EntityReference service, EntityReference owner) throws IOException {
chart.setFullyQualifiedName(getFQN(service, chart));
EntityUtil.populateOwner(userDAO(), teamDAO(), owner); // Validate owner
getService(service);
private void validateRelationships(Chart chart) throws IOException {
chart.setFullyQualifiedName(getFQN(chart));
EntityUtil.populateOwner(userDAO(), teamDAO(), chart.getOwner()); // Validate owner
getService(chart.getService());
chart.setTags(EntityUtil.addDerivedTags(tagDAO(), chart.getTags()));
}
@ -230,7 +226,7 @@ public abstract class ChartRepository {
// Patch can't make changes to following fields. Ignore the changes
updated.withFullyQualifiedName(original.getFullyQualifiedName()).withName(original.getName())
.withService(original.getService()).withId(original.getId());
validateRelationships(updated, updated.getService(), updated.getOwner());
validateRelationships(updated);
ChartUpdater chartUpdater = new ChartUpdater(original, updated, true);
chartUpdater.updateAll();
chartUpdater.store();

74
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DashboardRepository.java
View File

@ -17,17 +17,17 @@
package org.openmetadata.catalog.jdbi3;
import com.fasterxml.jackson.core.JsonProcessingException;
import org.openmetadata.catalog.Entity;
import org.openmetadata.catalog.entity.data.Chart;
import org.openmetadata.catalog.entity.data.Dashboard;
import org.openmetadata.catalog.entity.services.DashboardService;
import org.openmetadata.catalog.exception.EntityNotFoundException;
import org.openmetadata.catalog.jdbi3.TeamRepository.TeamDAO;
import org.openmetadata.catalog.jdbi3.UserRepository.UserDAO;
import org.openmetadata.catalog.resources.dashboards.DashboardResource;
import org.openmetadata.catalog.Entity;
import org.openmetadata.catalog.entity.data.Dashboard;
import org.openmetadata.catalog.jdbi3.UsageRepository.UsageDAO;
import org.openmetadata.catalog.jdbi3.ChartRepository.ChartDAO;
import org.openmetadata.catalog.jdbi3.DashboardServiceRepository.DashboardServiceDAO;
import org.openmetadata.catalog.jdbi3.TeamRepository.TeamDAO;
import org.openmetadata.catalog.jdbi3.UsageRepository.UsageDAO;
import org.openmetadata.catalog.jdbi3.UserRepository.UserDAO;
import org.openmetadata.catalog.resources.dashboards.DashboardResource;
import org.openmetadata.catalog.resources.dashboards.DashboardResource.DashboardList;
import org.openmetadata.catalog.type.EntityReference;
import org.openmetadata.catalog.type.TagLabel;
@ -52,11 +52,9 @@ import java.io.UnsupportedEncodingException;
import java.security.GeneralSecurityException;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.Date;
import java.util.List;
import java.util.UUID;
import java.util.stream.Collectors;
import static org.openmetadata.catalog.exception.CatalogExceptionMessage.entityNotFound;
@ -66,8 +64,8 @@ public abstract class DashboardRepository {
private static final Fields DASHBOARD_PATCH_FIELDS = new Fields(DashboardResource.FIELD_LIST,
"owner,service,tags,charts");
public static String getFQN(EntityReference service, Dashboard dashboard) {
return (service.getName() + "." + dashboard.getName());
public static String getFQN(Dashboard dashboard) {
return (dashboard.getService().getName() + "." + dashboard.getName());
}
@CreateSqlObject
@ -94,7 +92,7 @@ public abstract class DashboardRepository {
@CreateSqlObject
abstract TagRepository.TagDAO tagDAO();
EntityRepository<Dashboard> entityRepository = new EntityRepository<Dashboard>() {
EntityRepository<Dashboard> entityRepository = new EntityRepository<>() {
@Override
public List<String> listAfter(String fqnPrefix, int limitParam, String after) {
return dashboardDAO().listAfter(fqnPrefix, limitParam, after);
@ -116,7 +114,7 @@ public abstract class DashboardRepository {
}
@Override
public Dashboard setFields(Dashboard entity, Fields fields) throws IOException, ParseException {
public Dashboard setFields(Dashboard entity, Fields fields) throws IOException {
return DashboardRepository.this.setFields(entity, fields);
}
@ -150,23 +148,20 @@ public abstract class DashboardRepository {
}
@Transaction
public Dashboard create(Dashboard dashboard, EntityReference service, EntityReference owner) throws IOException {
getService(service); // Validate service
return createInternal(dashboard, service, owner);
public Dashboard create(Dashboard dashboard) throws IOException {
validateRelationships(dashboard);
return createInternal(dashboard);
}
@Transaction
public PutResponse<Dashboard> createOrUpdate(Dashboard updated, EntityReference service,
EntityReference newOwner) throws IOException {
getService(service); // Validate service
String fqn = getFQN(service, updated);
Dashboard stored = JsonUtils.readValue(dashboardDAO().findByFQN(fqn), Dashboard.class);
public PutResponse<Dashboard> createOrUpdate(Dashboard updated) throws IOException {
validateRelationships(updated);
Dashboard stored = JsonUtils.readValue(dashboardDAO().findByFQN(updated.getFullyQualifiedName()), Dashboard.class);
if (stored == null) {
return new PutResponse<>(Status.CREATED, createInternal(updated, service, newOwner));
return new PutResponse<>(Status.CREATED, createInternal(updated));
}
setFields(stored, DASHBOARD_UPDATE_FIELDS);
updated.setId(stored.getId());
validateRelationships(updated, service, newOwner);
DashboardUpdater dashboardUpdater = new DashboardUpdater(stored, updated, false);
dashboardUpdater.updateAll();
@ -229,9 +224,8 @@ public abstract class DashboardRepository {
}
private Dashboard createInternal(Dashboard dashboard, EntityReference service, EntityReference owner)
private Dashboard createInternal(Dashboard dashboard)
throws IOException {
validateRelationships(dashboard, service, owner);
storeDashboard(dashboard, false);
addRelationships(dashboard);
return dashboard;
@ -268,17 +262,17 @@ public abstract class DashboardRepository {
// Patch can't make changes to following fields. Ignore the changes
updated.withId(original.getId()).withFullyQualifiedName(original.getFullyQualifiedName())
.withName(original.getName()).withService(original.getService()).withId(original.getId());
validateRelationships(updated, updated.getService(), updated.getOwner());
validateRelationships(updated);
DashboardUpdater DashboardUpdater = new DashboardUpdater(original, updated, true);
DashboardUpdater.updateAll();
DashboardUpdater.store();
}
private void validateRelationships(Dashboard dashboard, EntityReference service, EntityReference owner)
private void validateRelationships(Dashboard dashboard)
throws IOException {
dashboard.setFullyQualifiedName(getFQN(service, dashboard));
EntityUtil.populateOwner(userDAO(), teamDAO(), owner); // Validate owner
getService(service);
dashboard.setFullyQualifiedName(getFQN(dashboard));
EntityUtil.populateOwner(userDAO(), teamDAO(), dashboard.getOwner()); // Validate owner
getService(dashboard.getService());
dashboard.setTags(EntityUtil.addDerivedTags(tagDAO(), dashboard.getTags()));
}
@ -335,26 +329,6 @@ public abstract class DashboardRepository {
applyTags(dashboard);
}
private void updateChartRelationships(Dashboard dashboard) throws IOException {
String dashboardId = dashboard.getId().toString();
// Add relationship from dashboard to chart
if (dashboard.getCharts() != null) {
// Remove any existing charts associated with this dashboard
List<Chart> existingCharts = getCharts(dashboard);
if (existingCharts != null) {
for (Chart chart: existingCharts) {
relationshipDAO().delete(dashboardId, chart.getId().toString(), Relationship.CONTAINS.ordinal());
}
}
for (EntityReference chart : dashboard.getCharts()) {
relationshipDAO().insert(dashboardId, chart.getId().toString(), Entity.DASHBOARD, Entity.CHART,
Relationship.CONTAINS.ordinal());
}
}
}
private Dashboard validateDashboard(String id) throws IOException {
return EntityUtil.validate(id, dashboardDAO().findById(id), Dashboard.class);
}
@ -491,7 +465,7 @@ public abstract class DashboardRepository {
updateCharts();
}
private void updateCharts() throws IOException {
private void updateCharts() {
String dashboardId = updated.getId().toString();
// Remove all charts associated with this dashboard

34
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DatabaseRepository.java
View File

@ -64,8 +64,8 @@ public abstract class DatabaseRepository {
private static final Fields DATABASE_PATCH_FIELDS = new Fields(DatabaseResource.FIELD_LIST,
"owner,service, usageSummary");
public static String getFQN(EntityReference service, Database database) {
return (service.getName() + "." + database.getName());
public static String getFQN(Database database) {
return (database.getService().getName() + "." + database.getName());
}
public static List<EntityReference> toEntityReference(List<Table> tables) {
@ -151,8 +151,9 @@ public abstract class DatabaseRepository {
}
@Transaction
public Database create(Database database, EntityReference service, EntityReference owner) throws IOException {
return createInternal(database, service, owner);
public Database create(Database database) throws IOException {
validateRelationships(database);
return createInternal(database);
}
@Transaction
@ -167,18 +168,14 @@ public abstract class DatabaseRepository {
}
@Transaction
public PutResponse<Database> createOrUpdate(Database updated, EntityReference service, EntityReference newOwner)
throws IOException {
getService(service); // Validate service
String fqn = getFQN(service, updated);
Database stored = JsonUtils.readValue(databaseDAO().findByFQN(fqn), Database.class);
public PutResponse<Database> createOrUpdate(Database updated) throws IOException {
validateRelationships(updated);
Database stored = JsonUtils.readValue(databaseDAO().findByFQN(updated.getFullyQualifiedName()), Database.class);
if (stored == null) { // Database does not exist. Create a new one
return new PutResponse<>(Status.CREATED, createInternal(updated, service, newOwner));
return new PutResponse<>(Status.CREATED, createInternal(updated));
}
setFields(stored, DATABASE_UPDATE_FIELDS);
updated.setId(stored.getId());
validateRelationships(updated, service, newOwner);
DatabaseUpdater databaseUpdater = new DatabaseUpdater(stored, updated, false);
databaseUpdater.updateAll();
@ -198,17 +195,16 @@ public abstract class DatabaseRepository {
return updated;
}
public Database createInternal(Database database, EntityReference service, EntityReference owner) throws IOException {
validateRelationships(database, service, owner);
public Database createInternal(Database database) throws IOException {
storeDatabase(database, false);
addRelationships(database);
return database;
}
private void validateRelationships(Database database, EntityReference service, EntityReference owner) throws IOException {
database.setFullyQualifiedName(getFQN(service, database));
database.setOwner(EntityUtil.populateOwner(userDAO(), teamDAO(), owner)); // Validate owner
database.setService(getService(service));
private void validateRelationships(Database database) throws IOException {
database.setFullyQualifiedName(getFQN(database));
database.setOwner(EntityUtil.populateOwner(userDAO(), teamDAO(), database.getOwner())); // Validate owner
getService(database.getService());
}
private void addRelationships(Database database) throws IOException {
@ -238,7 +234,7 @@ public abstract class DatabaseRepository {
// Patch can't make changes to following fields. Ignore the changes
updated.withFullyQualifiedName(original.getFullyQualifiedName()).withName(original.getName())
.withService(original.getService()).withId(original.getId());
validateRelationships(updated, updated.getService(), updated.getOwner());
validateRelationships(updated);
DatabaseUpdater databaseUpdater = new DatabaseUpdater(original, updated, true);
databaseUpdater.updateAll();
databaseUpdater.store();

1
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/MessagingServiceRepository.java
View File

@ -71,6 +71,7 @@ public abstract class MessagingServiceRepository {
return messagingService;
}
@Transaction
public MessagingService update(String id, String description, List<String> brokers, URI schemaRegistry,
Schedule ingestionSchedule)
throws IOException {

24
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/ModelRepository.java
View File

@ -137,20 +137,20 @@ public abstract class ModelRepository {
}
@Transaction
public Model create(Model model, EntityReference owner) throws IOException {
return createInternal(model, owner);
public Model create(Model model) throws IOException {
validateRelationships(model);
return createInternal(model);
}
@Transaction
public PutResponse<Model> createOrUpdate(Model updated, EntityReference newOwner) throws IOException {
String fqn = getFQN(updated);
Model stored = JsonUtils.readValue(modelDAO().findByFQN(fqn), Model.class);
public PutResponse<Model> createOrUpdate(Model updated) throws IOException {
validateRelationships(updated);
Model stored = JsonUtils.readValue(modelDAO().findByFQN(updated.getFullyQualifiedName()), Model.class);
if (stored == null) {
return new PutResponse<>(Status.CREATED, createInternal(updated, newOwner));
return new PutResponse<>(Status.CREATED, createInternal(updated));
}
setFields(stored, MODEL_UPDATE_FIELDS);
updated.setId(stored.getId());
validateRelationships(updated, newOwner);
ModelUpdater modelUpdater = new ModelUpdater(stored, updated, false);
modelUpdater.updateAll();
@ -216,17 +216,15 @@ public abstract class ModelRepository {
}
private Model createInternal(Model model, EntityReference owner)
throws IOException {
validateRelationships(model, owner);
private Model createInternal(Model model) throws IOException {
storeModel(model, false);
addRelationships(model);
return model;
}
private void validateRelationships(Model model, EntityReference owner) throws IOException {
private void validateRelationships(Model model) throws IOException {
model.setFullyQualifiedName(getFQN(model));
EntityUtil.populateOwner(userDAO(), teamDAO(), owner); // Validate owner
EntityUtil.populateOwner(userDAO(), teamDAO(), model.getOwner()); // Validate owner
if (model.getDashboard() != null) {
String dashboardId = model.getDashboard().getId().toString();
model.setDashboard(EntityUtil.getEntityReference(
@ -264,7 +262,7 @@ public abstract class ModelRepository {
// Patch can't make changes to following fields. Ignore the changes
updated.withFullyQualifiedName(original.getFullyQualifiedName()).withName(original.getName())
.withId(original.getId());
validateRelationships(updated, updated.getOwner());
validateRelationships(updated);
ModelRepository.ModelUpdater modelUpdater = new ModelRepository.ModelUpdater(original, updated, true);
modelUpdater.updateAll();
modelUpdater.store();

39
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PipelineRepository.java
View File

@ -59,8 +59,8 @@ public abstract class PipelineRepository {
private static final Fields PIPELINE_PATCH_FIELDS = new Fields(PipelineResource.FIELD_LIST,
"owner,service,tags,tasks");
public static String getFQN(EntityReference service, Pipeline pipeline) {
return (service.getName() + "." + pipeline.getName());
public static String getFQN(Pipeline pipeline) {
return (pipeline.getService().getName() + "." + pipeline.getName());
}
@CreateSqlObject
@ -134,22 +134,20 @@ public abstract class PipelineRepository {
}
@Transaction
public Pipeline create(Pipeline pipeline, EntityReference service, EntityReference owner) throws IOException {
return createInternal(pipeline, service, owner);
public Pipeline create(Pipeline pipeline) throws IOException {
validateRelationships(pipeline);
return createInternal(pipeline);
}
@Transaction
public PutResponse<Pipeline> createOrUpdate(Pipeline updated, EntityReference service,
EntityReference newOwner) throws IOException {
getService(service); // Validate service
String fqn = getFQN(service, updated);
Pipeline stored = JsonUtils.readValue(pipelineDAO().findByFQN(fqn), Pipeline.class);
public PutResponse<Pipeline> createOrUpdate(Pipeline updated) throws IOException {
validateRelationships(updated);
Pipeline stored = JsonUtils.readValue(pipelineDAO().findByFQN(updated.getFullyQualifiedName()), Pipeline.class);
if (stored == null) {
return new PutResponse<>(Status.CREATED, createInternal(updated, service, newOwner));
return new PutResponse<>(Status.CREATED, createInternal(updated));
}
setFields(stored, PIPELINE_UPDATE_FIELDS);
updated.setId(stored.getId());
validateRelationships(updated, service, newOwner);
PipelineUpdater pipelineUpdater = new PipelineUpdater(stored, updated, false);
pipelineUpdater.updateAll();
@ -222,18 +220,16 @@ public abstract class PipelineRepository {
}
private Pipeline createInternal(Pipeline pipeline, EntityReference service, EntityReference owner)
throws IOException {
validateRelationships(pipeline, service, owner);
private Pipeline createInternal(Pipeline pipeline) throws IOException {
storePipeline(pipeline, false);
addRelationships(pipeline);
return pipeline;
}
private void validateRelationships(Pipeline pipeline, EntityReference service, EntityReference owner) throws IOException {
pipeline.setFullyQualifiedName(getFQN(service, pipeline));
EntityUtil.populateOwner(userDAO(), teamDAO(), owner); // Validate owner
getService(service);
private void validateRelationships(Pipeline pipeline) throws IOException {
pipeline.setFullyQualifiedName(getFQN(pipeline));
EntityUtil.populateOwner(userDAO(), teamDAO(), pipeline.getOwner()); // Validate owner
getService(pipeline.getService());
pipeline.setTags(EntityUtil.addDerivedTags(tagDAO(), pipeline.getTags()));
}
@ -287,7 +283,7 @@ public abstract class PipelineRepository {
// Patch can't make changes to following fields. Ignore the changes
updated.withFullyQualifiedName(original.getFullyQualifiedName()).withName(original.getName())
.withService(original.getService()).withId(original.getId());
validateRelationships(updated, updated.getService(), updated.getOwner());
validateRelationships(updated);
PipelineRepository.PipelineUpdater pipelineUpdater = new PipelineRepository.PipelineUpdater(original, updated, true);
pipelineUpdater.updateAll();
pipelineUpdater.store();
@ -303,11 +299,6 @@ public abstract class PipelineRepository {
pipeline.setOwner(owner);
}
private void updateOwner(Pipeline pipeline, EntityReference origOwner, EntityReference newOwner) {
EntityUtil.updateOwner(relationshipDAO(), origOwner, newOwner, pipeline.getId(), Entity.PIPELINE);
pipeline.setOwner(newOwner);
}
private void applyTags(Pipeline pipeline) throws IOException {
// Add pipeline level tags by adding tag to pipeline relationship
EntityUtil.applyTags(tagDAO(), pipeline.getTags(), pipeline.getFullyQualifiedName());

45
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TableRepository.java
View File

@ -117,7 +117,7 @@ public abstract class TableRepository {
@CreateSqlObject
abstract TagDAO tagDAO();
EntityRepository<Table> entityRepository = new EntityRepository<Table>() {
EntityRepository<Table> entityRepository = new EntityRepository<>() {
@Override
public List<String> listAfter(String fqnPrefix, int limitParam, String after) {
return tableDAO().listAfter(fqnPrefix, limitParam, after);
@ -150,6 +150,10 @@ public abstract class TableRepository {
}
};
public static String getFQN(Table table) {
return (table.getDatabase().getName() + "." + table.getName());
}
@Transaction
public ResultList<Table> listAfter(Fields fields, String databaseFQN, int limitParam, String after)
throws IOException, ParseException, GeneralSecurityException {
@ -174,8 +178,9 @@ public abstract class TableRepository {
}
@Transaction
public Table create(Table table, EntityReference owner, UUID databaseId) throws IOException {
return createInternal(databaseId, table, owner);
public Table create(Table table, UUID databaseId) throws IOException {
validateRelationships(table, databaseId);
return createInternal(table);
}
@Transaction
@ -197,23 +202,19 @@ public abstract class TableRepository {
}
@Transaction
public PutResponse<Table> createOrUpdate(Table updated, EntityReference newOwner, UUID databaseId) throws
IOException, ParseException {
Database database = EntityUtil.validate(databaseId.toString(), databaseDAO().findById(databaseId.toString()),
Database.class);
String tableFQName = database.getFullyQualifiedName() + "." + updated.getName();
Table stored = JsonUtils.readValue(tableDAO().findByFQN(tableFQName), Table.class);
public PutResponse<Table> createOrUpdate(Table updated, UUID databaseId) throws IOException, ParseException {
validateRelationships(updated, databaseId);
Table stored = JsonUtils.readValue(tableDAO().findByFQN(updated.getFullyQualifiedName()), Table.class);
if (stored == null) {
return new PutResponse<>(Status.CREATED, createInternal(database.getId(), updated, newOwner));
return new PutResponse<>(Status.CREATED, createInternal(updated));
}
setFields(stored, TABLE_UPDATE_FIELDS);
updated.setId(stored.getId());
validateRelationships(updated, database, newOwner);
validateRelationships(updated);
TableUpdater tableUpdater = new TableUpdater(stored, updated, false);
tableUpdater.updateAll();
tableUpdater.store();
// setFields(updated, TABLE_UPDATE_FIELDS); // TODO remove this
// if (updated) {
// // TODO clean this up
@ -310,9 +311,7 @@ public abstract class TableRepository {
// No @Transaction variation method to avoid nested transaction
private Table createInternal(UUID databaseId, Table table, EntityReference owner) throws IOException {
LOG.info("Creating table {} {}", table.getFullyQualifiedName(), table.getId());
validateRelationships(table, databaseId, owner);
private Table createInternal(Table table) throws IOException {
storeTable(table, false);
addRelationships(table);
@ -321,12 +320,13 @@ public abstract class TableRepository {
return table;
}
private void validateRelationships(Table table, UUID databaseId, EntityReference owner) throws IOException {
private void validateRelationships(Table table, UUID databaseId) throws IOException {
// Validate database
Database db = EntityUtil.validate(databaseId.toString(), databaseDAO().findById(databaseId.toString()),
Database.class);
table.setDatabase(EntityUtil.getEntityReference(db));
// Validate and set other relationships
validateRelationships(table, db, owner);
validateRelationships(table);
}
private void setColumnFQN(String parentFQN, List<Column> columns) {
@ -352,14 +352,13 @@ public abstract class TableRepository {
}
}
private void validateRelationships(Table table, Database database, EntityReference owner) throws IOException {
private void validateRelationships(Table table) throws IOException {
// Set data in table entity based on database relationship
table.setDatabase(EntityUtil.getEntityReference(database));
table.setFullyQualifiedName(database.getFullyQualifiedName() + "." + table.getName());
table.setFullyQualifiedName(getFQN(table));
setColumnFQN(table.getFullyQualifiedName(), table.getColumns());
// Check if owner is valid and set the relationship
table.setOwner(EntityUtil.populateOwner(userDAO(), teamDAO(), owner));
table.setOwner(EntityUtil.populateOwner(userDAO(), teamDAO(), table.getOwner()));
// Validate table tags and add derived tags to the list
table.setTags(EntityUtil.addDerivedTags(tagDAO(), table.getTags()));
@ -456,7 +455,7 @@ public abstract class TableRepository {
.withDatabase(original.getDatabase()).withId(original.getId());
// TODO checking database is unnecessary as it has not changed
validateRelationships(updated, updated.getDatabase().getId(), updated.getOwner());
validateRelationships(updated, updated.getDatabase().getId());
TableUpdater tableUpdater = new TableUpdater(original, updated, true);
tableUpdater.updateAll();
@ -731,7 +730,7 @@ public abstract class TableRepository {
int delete(@Bind("id") String id);
}
class TableEntityInterface implements EntityInterface {
static class TableEntityInterface implements EntityInterface {
private final Table table;
TableEntityInterface(Table table) {

45
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TaskRepository.java
View File

@ -19,13 +19,8 @@ package org.openmetadata.catalog.jdbi3;
import com.fasterxml.jackson.core.JsonProcessingException;
import org.openmetadata.catalog.Entity;
import org.openmetadata.catalog.entity.data.Task;
import org.openmetadata.catalog.entity.data.Task;
import org.openmetadata.catalog.entity.data.Task;
import org.openmetadata.catalog.entity.services.PipelineService;
import org.openmetadata.catalog.exception.CatalogExceptionMessage;
import org.openmetadata.catalog.exception.EntityNotFoundException;
import org.openmetadata.catalog.jdbi3.ChartRepository.ChartUpdater;
import org.openmetadata.catalog.jdbi3.TaskRepository.TaskEntityInterface;
import org.openmetadata.catalog.jdbi3.PipelineServiceRepository.PipelineServiceDAO;
import org.openmetadata.catalog.jdbi3.TeamRepository.TeamDAO;
import org.openmetadata.catalog.jdbi3.UserRepository.UserDAO;
@ -66,8 +61,8 @@ public abstract class TaskRepository {
"taskConfig,tags");
private static final Fields TASK_PATCH_FIELDS = new Fields(TaskResource.FIELD_LIST, "owner,service,tags");
public static String getFQN(EntityReference service, Task task) {
return (service.getName() + "." + task.getName());
public static String getFQN(Task task) {
return (task.getService().getName() + "." + task.getName());
}
@CreateSqlObject
@ -143,9 +138,9 @@ public abstract class TaskRepository {
}
@Transaction
public Task create(Task task, EntityReference service, EntityReference owner) throws IOException {
getService(service); // Validate service
return createInternal(task, service, owner);
public Task create(Task task) throws IOException {
validateRelationships(task);
return createInternal(task);
}
@Transaction
@ -160,18 +155,14 @@ public abstract class TaskRepository {
}
@Transaction
public PutResponse<Task> createOrUpdate(Task updated, EntityReference service, EntityReference newOwner)
throws IOException {
getService(service); // Validate service
String fqn = getFQN(service, updated);
Task stored = JsonUtils.readValue(taskDAO().findByFQN(fqn), Task.class);
public PutResponse<Task> createOrUpdate(Task updated) throws IOException {
validateRelationships(updated);
Task stored = JsonUtils.readValue(taskDAO().findByFQN(updated.getFullyQualifiedName()), Task.class);
if (stored == null) { // Task does not exist. Create a new one
return new PutResponse<>(Status.CREATED, createInternal(updated, service, newOwner));
return new PutResponse<>(Status.CREATED, createInternal(updated));
}
setFields(stored, TASK_UPDATE_FIELDS);
updated.setId(stored.getId());
validateRelationships(updated, service, newOwner);
TaskUpdater taskUpdater = new TaskUpdater(stored, updated, false);
taskUpdater.updateAll();
@ -188,17 +179,16 @@ public abstract class TaskRepository {
return updated;
}
public Task createInternal(Task task, EntityReference service, EntityReference owner) throws IOException {
validateRelationships(task, service, owner);
public Task createInternal(Task task) throws IOException {
storeTask(task, false);
addRelationships(task);
return task;
}
private void validateRelationships(Task task, EntityReference service, EntityReference owner) throws IOException {
task.setFullyQualifiedName(getFQN(service, task));
EntityUtil.populateOwner(userDAO(), teamDAO(), owner); // Validate owner
getService(service);
private void validateRelationships(Task task) throws IOException {
task.setFullyQualifiedName(getFQN(task));
EntityUtil.populateOwner(userDAO(), teamDAO(), task.getOwner()); // Validate owner
getService(task.getService());
task.setTags(EntityUtil.addDerivedTags(tagDAO(), task.getTags()));
}
@ -238,7 +228,7 @@ public abstract class TaskRepository {
// Patch can't make changes to following fields. Ignore the changes
updated.withFullyQualifiedName(original.getFullyQualifiedName()).withName(original.getName())
.withService(original.getService()).withId(original.getId());
validateRelationships(updated, updated.getService(), updated.getOwner());
validateRelationships(updated);
TaskUpdater taskUpdater = new TaskUpdater(original, updated, true);
taskUpdater.updateAll();
taskUpdater.store();
@ -253,11 +243,6 @@ public abstract class TaskRepository {
task.setOwner(owner);
}
private void updateOwner(Task task, EntityReference origOwner, EntityReference newOwner) {
EntityUtil.updateOwner(relationshipDAO(), origOwner, newOwner, task.getId(), Entity.TASK);
task.setOwner(newOwner);
}
private Task validateTask(String id) throws IOException {
return EntityUtil.validate(id, taskDAO().findById(id), Task.class);
}

34
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TopicRepository.java
View File

@ -60,8 +60,8 @@ public abstract class TopicRepository {
private static final Fields TOPIC_UPDATE_FIELDS = new Fields(TopicResource.FIELD_LIST, "owner,tags");
private static final Fields TOPIC_PATCH_FIELDS = new Fields(TopicResource.FIELD_LIST, "owner,service,tags");
public static String getFQN(EntityReference service, Topic topic) {
return (service.getName() + "." + topic.getName());
public static String getFQN(Topic topic) {
return (topic.getService().getName() + "." + topic.getName());
}
@CreateSqlObject
@ -136,9 +136,9 @@ public abstract class TopicRepository {
}
@Transaction
public Topic create(Topic topic, EntityReference service, EntityReference owner) throws IOException {
getService(service); // Validate service
return createInternal(topic, service, owner);
public Topic create(Topic topic) throws IOException {
validateRelationships(topic);
return createInternal(topic);
}
@Transaction
@ -153,17 +153,14 @@ public abstract class TopicRepository {
}
@Transaction
public PutResponse<Topic> createOrUpdate(Topic updated, EntityReference service, EntityReference newOwner)
throws IOException {
getService(service); // Validate service
String fqn = getFQN(service, updated);
Topic stored = JsonUtils.readValue(topicDAO().findByFQN(fqn), Topic.class);
public PutResponse<Topic> createOrUpdate(Topic updated) throws IOException {
validateRelationships(updated);
Topic stored = JsonUtils.readValue(topicDAO().findByFQN(updated.getFullyQualifiedName()), Topic.class);
if (stored == null) { // Topic does not exist. Create a new one
return new PutResponse<>(Status.CREATED, createInternal(updated, service, newOwner));
return new PutResponse<>(Status.CREATED, createInternal(updated));
}
setFields(stored, TOPIC_UPDATE_FIELDS);
updated.setId(stored.getId());
validateRelationships(updated, service, newOwner);
TopicUpdater topicUpdater = new TopicUpdater(stored, updated, false);
topicUpdater.updateAll();
@ -185,17 +182,16 @@ public abstract class TopicRepository {
return EntityUtil.populateOwner(userDAO(), teamDAO(), topic.getOwner());
}
public Topic createInternal(Topic topic, EntityReference service, EntityReference owner) throws IOException {
validateRelationships(topic, service, owner);
public Topic createInternal(Topic topic) throws IOException {
storeTopic(topic, false);
addRelationships(topic);
return topic;
}
private void validateRelationships(Topic topic, EntityReference service, EntityReference owner) throws IOException {
topic.setFullyQualifiedName(getFQN(service, topic));
EntityUtil.populateOwner(userDAO(), teamDAO(), owner); // Validate owner
getService(service);
private void validateRelationships(Topic topic) throws IOException {
topic.setFullyQualifiedName(getFQN(topic));
EntityUtil.populateOwner(userDAO(), teamDAO(), topic.getOwner()); // Validate owner
getService(topic.getService());
topic.setTags(EntityUtil.addDerivedTags(tagDAO(), topic.getTags()));
}
@ -234,7 +230,7 @@ public abstract class TopicRepository {
// Patch can't make changes to following fields. Ignore the changes
updated.withFullyQualifiedName(original.getFullyQualifiedName()).withName(original.getName())
.withService(original.getService()).withId(original.getId());
validateRelationships(updated, updated.getService(), updated.getOwner());
validateRelationships(updated);
TopicUpdater topicUpdater = new TopicUpdater(original, updated, true);
topicUpdater.updateAll();
topicUpdater.store();

11
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/UserRepository.java
View File

@ -188,7 +188,8 @@ public abstract class UserRepository {
@Transaction
public User create(User user, List<UUID> teamIds) throws IOException {
return createInternal(user, teamIds);
validateRelationships(user, teamIds);
return createInternal(user);
}
@Transaction
@ -205,7 +206,6 @@ public abstract class UserRepository {
@Transaction
public RestUtil.PutResponse<User> createOrUpdate(User updated) throws IOException {
// TODO method for this
User stored = JsonUtils.readValue(userDAO().findByName(updated.getName()), User.class);
// TODO why are we doing this?
@ -213,12 +213,12 @@ public abstract class UserRepository {
if (updated.getTeams() != null) {
teamIds.addAll(EntityUtil.getIDList(updated.getTeams()));
}
validateRelationships(updated, teamIds);
if (stored == null) {
return new RestUtil.PutResponse<>(Response.Status.CREATED, createInternal(updated, teamIds));
return new RestUtil.PutResponse<>(Response.Status.CREATED, createInternal(updated));
}
setFields(stored, USER_UPDATE_FIELDS);
updated.setId(stored.getId());
validateRelationships(updated, teamIds);
UserUpdater userUpdater = new UserUpdater(stored, updated, false);
userUpdater.updateAll();
@ -289,8 +289,7 @@ public abstract class UserRepository {
return EntityUtil.validate(userId, userDAO().findById(userId), User.class);
}
private User createInternal(User user, List<UUID> teamIds) throws IOException {
validateRelationships(user, teamIds);
private User createInternal(User user) throws IOException {
storeUser(user, false);
addRelationships(user);
return user;

4
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/charts/ChartResource.java
View File

@ -229,7 +229,7 @@ public class ChartResource {
.withOwner(create.getOwner())
.withUpdatedBy(securityContext.getUserPrincipal().getName())
.withUpdatedAt(new Date());
chart = addHref(uriInfo, dao.create(chart, create.getService(), create.getOwner()));
chart = addHref(uriInfo, dao.create(chart));
return Response.created(chart.getHref()).entity(chart).build();
}
@ -279,7 +279,7 @@ public class ChartResource {
.withOwner(create.getOwner())
.withUpdatedBy(securityContext.getUserPrincipal().getName())
.withUpdatedAt(new Date());
PutResponse<Chart> response = dao.createOrUpdate(chart, create.getService(), create.getOwner());
PutResponse<Chart> response = dao.createOrUpdate(chart);
chart = addHref(uriInfo, response.getEntity());
return Response.status(response.getStatus()).entity(chart).build();
}

4
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/dashboards/DashboardResource.java
View File

@ -229,7 +229,7 @@ public class DashboardResource {
.withOwner(create.getOwner())
.withUpdatedBy(securityContext.getUserPrincipal().getName())
.withUpdatedAt(new Date());
dashboard = addHref(uriInfo, dao.create(dashboard, dashboard.getService(), dashboard.getOwner()));
dashboard = addHref(uriInfo, dao.create(dashboard));
return Response.created(dashboard.getHref()).entity(dashboard).build();
}
@ -278,7 +278,7 @@ public class DashboardResource {
.withUpdatedBy(securityContext.getUserPrincipal().getName())
.withUpdatedAt(new Date());
PutResponse<Dashboard> response = dao.createOrUpdate(dashboard, dashboard.getService(), dashboard.getOwner());
PutResponse<Dashboard> response = dao.createOrUpdate(dashboard);
dashboard = addHref(uriInfo, response.getEntity());
return Response.status(response.getStatus()).entity(dashboard).build();
}

9
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/databases/DatabaseResource.java
View File

@ -227,10 +227,11 @@ public class DatabaseResource {
@Valid CreateDatabase create) throws IOException {
SecurityUtil.checkAdminOrBotRole(authorizer, securityContext);
Database database = new Database().withId(UUID.randomUUID()).withName(create.getName())
.withDescription(create.getDescription())
.withDescription(create.getDescription()).withService(create.getService())
.withOwner(create.getOwner())
.withUpdatedBy(securityContext.getUserPrincipal().getName())
.withUpdatedAt(new Date());
database = addHref(uriInfo, dao.create(database, create.getService(), create.getOwner()));
database = addHref(uriInfo, dao.create(database));
return Response.created(database.getHref()).entity(database).build();
}
@ -271,9 +272,9 @@ public class DatabaseResource {
Database database = new Database().withId(UUID.randomUUID()).withName(create.getName())
.withDescription(create.getDescription()).withOwner(create.getOwner())
.withUpdatedBy(securityContext.getUserPrincipal().getName())
.withService(create.getService()).withUpdatedBy(securityContext.getUserPrincipal().getName())
.withUpdatedAt(new Date());
PutResponse<Database> response = dao.createOrUpdate(database, create.getService(), create.getOwner());
PutResponse<Database> response = dao.createOrUpdate(database);
Database db = addHref(uriInfo, response.getEntity());
return Response.status(response.getStatus()).entity(db).build();
}

6
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/databases/TableResource.java
View File

@ -227,8 +227,9 @@ public class TableResource {
.withTableConstraints(create.getTableConstraints()).withTableType(create.getTableType())
.withTags(create.getTags()).withViewDefinition(create.getViewDefinition())
.withUpdatedBy(securityContext.getUserPrincipal().getName())
.withOwner(create.getOwner())
.withUpdatedAt(new Date());
table = addHref(uriInfo, dao.create(validateNewTable(table), create.getOwner(), create.getDatabase()));
table = addHref(uriInfo, dao.create(validateNewTable(table), create.getDatabase()));
return Response.created(table.getHref()).entity(table).build();
}
@ -249,9 +250,10 @@ public class TableResource {
.withTableConstraints(create.getTableConstraints()).withTableType(create.getTableType())
.withTags(create.getTags()).withViewDefinition(create.getViewDefinition())
.withUpdatedBy(securityContext.getUserPrincipal().getName())
.withOwner(create.getOwner())
.withUpdatedAt(new Date());
SecurityUtil.checkAdminRoleOrPermissions(authorizer, securityContext, dao.getOwnerReference(table));
PutResponse<Table> response = dao.createOrUpdate(validateNewTable(table), create.getOwner(), create.getDatabase());
PutResponse<Table> response = dao.createOrUpdate(validateNewTable(table), create.getDatabase());
table = addHref(uriInfo, response.getEntity());
return Response.status(response.getStatus()).entity(table).build();
}

4
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/models/ModelResource.java
View File

@ -224,7 +224,7 @@ public class ModelResource {
.withOwner(create.getOwner())
.withUpdatedBy(securityContext.getUserPrincipal().getName())
.withUpdatedAt(new Date());
model = addHref(uriInfo, dao.create(model, model.getOwner()));
model = addHref(uriInfo, dao.create(model));
return Response.created(model.getHref()).entity(model).build();
}
@ -275,7 +275,7 @@ public class ModelResource {
.withUpdatedBy(securityContext.getUserPrincipal().getName())
.withUpdatedAt(new Date());
PutResponse<Model> response = dao.createOrUpdate(model, model.getOwner());
PutResponse<Model> response = dao.createOrUpdate(model);
model = addHref(uriInfo, response.getEntity());
return Response.status(response.getStatus()).entity(model).build();
}

4
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/pipelines/PipelineResource.java
View File

@ -229,7 +229,7 @@ public class PipelineResource {
.withUpdatedBy(securityContext.getUserPrincipal().getName())
.withUpdatedAt(new Date());
pipeline = addHref(uriInfo, dao.create(pipeline, pipeline.getService(), pipeline.getOwner()));
pipeline = addHref(uriInfo, dao.create(pipeline));
return Response.created(pipeline.getHref()).entity(pipeline).build();
}
@ -278,7 +278,7 @@ public class PipelineResource {
.withUpdatedBy(securityContext.getUserPrincipal().getName())
.withUpdatedAt(new Date());
PutResponse<Pipeline> response = dao.createOrUpdate(pipeline, pipeline.getService(), pipeline.getOwner());
PutResponse<Pipeline> response = dao.createOrUpdate(pipeline);
pipeline = addHref(uriInfo, response.getEntity());
return Response.status(response.getStatus()).entity(pipeline).build();
}

4
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/tasks/TaskResource.java
View File

@ -232,7 +232,7 @@ public class TaskResource {
.withOwner(create.getOwner())
.withUpdatedBy(securityContext.getUserPrincipal().getName())
.withUpdatedAt(new Date());
task = addHref(uriInfo, dao.create(task, create.getService(), create.getOwner()));
task = addHref(uriInfo, dao.create(task));
return Response.created(task.getHref()).entity(task).build();
}
@ -287,7 +287,7 @@ public class TaskResource {
.withOwner(create.getOwner())
.withUpdatedBy(securityContext.getUserPrincipal().getName())
.withUpdatedAt(new Date());
PutResponse<Task> response = dao.createOrUpdate(task, create.getService(), create.getOwner());
PutResponse<Task> response = dao.createOrUpdate(task);
task = addHref(uriInfo, response.getEntity());
return Response.status(response.getStatus()).entity(task).build();
}

4
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/topics/TopicResource.java
View File

@ -233,7 +233,7 @@ public class TopicResource {
.withUpdatedBy(securityContext.getUserPrincipal().getName())
.withUpdatedAt(new Date());
topic = addHref(uriInfo, dao.create(topic, create.getService(), create.getOwner()));
topic = addHref(uriInfo, dao.create(topic));
return Response.created(topic.getHref()).entity(topic).build();
}
@ -288,7 +288,7 @@ public class TopicResource {
.withUpdatedBy(securityContext.getUserPrincipal().getName())
.withUpdatedAt(new Date());
PutResponse<Topic> response = dao.createOrUpdate(topic, create.getService(), create.getOwner());
PutResponse<Topic> response = dao.createOrUpdate(topic);
topic = addHref(uriInfo, response.getEntity());
return Response.status(response.getStatus()).entity(topic).build();
}

4
catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/databases/TableResourceTest.java
View File

@ -531,10 +531,6 @@ public class TableResourceTest extends CatalogApplicationTest {
assertEquals(bankTagUsageCount + 1, getTagUsageCount(USER_BANK_ACCOUNT_TAG_LABEL.getTagFQN(), userAuthHeaders()));
}
private void validateTags(Column expected, Column actual) throws HttpResponseException {
TestUtils.validateTags(expected.getFullyQualifiedName(), expected.getTags(), actual.getTags());
}
@Test
public void put_tableJoins_200(TestInfo test) throws HttpResponseException, ParseException {
Table table1 = createAndCheckTable(create(test, 1), adminAuthHeaders());