diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java b/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java index 4c2eff4e0bb..dfee2e3a66e 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java @@ -222,10 +222,18 @@ public final class Entity { return entityRepository.getFields(fields); } + public static T getEntity(EntityReference ref, String fields, Include include) throws IOException { + return getEntity(ref.getType(), ref.getId(), fields, include); + } + public static T getEntity(EntityReference ref, EntityUtil.Fields fields, Include include) throws IOException { return getEntity(ref.getType(), ref.getId(), fields, include); } + public static T getEntity(String entityType, UUID id, String fields, Include include) throws IOException { + return getEntity(entityType, id, getFields(entityType, fields), include); + } + /** Retrieve the entity using id from given entity reference and fields */ public static T getEntity(String entityType, UUID id, EntityUtil.Fields fields, Include include) throws IOException { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DatabaseSchemaRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DatabaseSchemaRepository.java index ea08f90aa47..94c0ce69f06 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DatabaseSchemaRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DatabaseSchemaRepository.java @@ -114,9 +114,13 @@ public class DatabaseSchemaRepository extends EntityRepository { } private void populateDatabase(DatabaseSchema schema) throws IOException { - Database database = Entity.getEntity(schema.getDatabase(), Fields.EMPTY_FIELDS, ALL); - schema.setDatabase(database.getEntityReference()); - schema.setService(database.getService()); - schema.setServiceType(database.getServiceType()); + Database database = Entity.getEntity(schema.getDatabase(), "owner", ALL); + schema + .withDatabase(database.getEntityReference()) + .withService(database.getService()) + .withServiceType(database.getServiceType()); + + // Carry forward ownership from database, if necessary + schema.withOwner(schema.getOwner() == null ? database.getOwner() : schema.getOwner()); } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java index 704158c5872..6c3f7ff272e 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java @@ -477,11 +477,13 @@ public abstract class EntityRepository { return update(uriInfo, original, updated); } + @SuppressWarnings("unused") protected void postCreate(T entity) { // Override to perform any operation required after creation. // For example ingestion pipeline creates a pipeline in AirFlow. } + @SuppressWarnings("unused") protected void postUpdate(T entity) { // Override to perform any operation required after an entity update. // For example ingestion pipeline creates a pipeline in AirFlow. diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java index 39e3e730d4f..492789a0a89 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java @@ -105,8 +105,7 @@ public class GlossaryTermRepository extends EntityRepository { validateHierarchy(entity); // Validate glossary - Fields glossaryFields = Entity.getFields(GLOSSARY, "reviewers"); - Glossary glossary = Entity.getEntity(entity.getGlossary(), glossaryFields, Include.NON_DELETED); + Glossary glossary = Entity.getEntity(entity.getGlossary(), "reviewers", Include.NON_DELETED); entity.setGlossary(glossary.getEntityReference()); // If reviewers is not set in the glossary term, then carry it from the glossary diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java index 900e61f5555..50acf9bb72a 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java @@ -17,6 +17,7 @@ import static java.util.stream.Collectors.groupingBy; import static java.util.stream.Collectors.toUnmodifiableList; import static org.openmetadata.common.utils.CommonUtil.listOrEmpty; import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; +import static org.openmetadata.schema.type.Include.ALL; import static org.openmetadata.service.Entity.DATABASE_SCHEMA; import static org.openmetadata.service.Entity.FIELD_DESCRIPTION; import static org.openmetadata.service.Entity.FIELD_DISPLAY_NAME; @@ -66,7 +67,6 @@ import org.openmetadata.schema.type.ColumnProfilerConfig; import org.openmetadata.schema.type.DailyCount; import org.openmetadata.schema.type.DataModel; import org.openmetadata.schema.type.EntityReference; -import org.openmetadata.schema.type.Include; import org.openmetadata.schema.type.JoinedWith; import org.openmetadata.schema.type.Relationship; import org.openmetadata.schema.type.SQLQuery; @@ -139,7 +139,7 @@ public class TableRepository extends EntityRepository { private void setDefaultFields(Table table) throws IOException { EntityReference schemaRef = getContainer(table.getId()); - DatabaseSchema schema = Entity.getEntity(schemaRef, Fields.EMPTY_FIELDS, Include.ALL); + DatabaseSchema schema = Entity.getEntity(schemaRef, Fields.EMPTY_FIELDS, ALL); table.withDatabaseSchema(schemaRef).withDatabase(schema.getDatabase()).withService(schema.getService()); } @@ -537,11 +537,15 @@ public class TableRepository extends EntityRepository
{ @Override public void prepare(Table table) throws IOException { - DatabaseSchema schema = Entity.getEntity(table.getDatabaseSchema(), Fields.EMPTY_FIELDS, Include.ALL); - table.setDatabaseSchema(schema.getEntityReference()); - table.setDatabase(schema.getDatabase()); - table.setService(schema.getService()); - table.setServiceType(schema.getServiceType()); + DatabaseSchema schema = Entity.getEntity(table.getDatabaseSchema(), "owner", ALL); + table + .withDatabaseSchema(schema.getEntityReference()) + .withDatabase(schema.getDatabase()) + .withService(schema.getService()) + .withServiceType(schema.getServiceType()); + + // Carry forward ownership from database schema + table.setOwner(table.getOwner() == null ? schema.getOwner() : table.getOwner()); // Validate column tags addDerivedColumnTags(table.getColumns()); diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/TableResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/TableResourceTest.java index da0b3a81e73..38e92a80822 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/TableResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/TableResourceTest.java @@ -93,6 +93,7 @@ import org.openmetadata.schema.api.data.CreateTable; import org.openmetadata.schema.api.data.CreateTableProfile; import org.openmetadata.schema.api.tests.CreateCustomMetric; import org.openmetadata.schema.entity.data.Database; +import org.openmetadata.schema.entity.data.DatabaseSchema; import org.openmetadata.schema.entity.data.Location; import org.openmetadata.schema.entity.data.Table; import org.openmetadata.schema.entity.services.DatabaseService; @@ -1056,13 +1057,8 @@ public class TableResourceTest extends EntityResourceTest { void putProfileConfig(Table table, Map authHeaders) throws IOException { List columnProfilerConfigs = new ArrayList<>(); columnProfilerConfigs.add( - new ColumnProfilerConfig() - .withColumnName("c1") - .withMetrics(List.of("valuesCount", "valuePercentage", "validCount", "duplicateCount"))); - columnProfilerConfigs.add( - new ColumnProfilerConfig() - .withColumnName("\"c.3\"") - .withMetrics(List.of("duplicateCount", "nullCount", "missingCount"))); + getColumnProfilerConfig("c1", "valuesCount", "valuePercentage", "validCount", "duplicateCount")); + columnProfilerConfigs.add(getColumnProfilerConfig("\"c.3\"", "duplicateCount", "nullCount", "missingCount")); TableProfilerConfig tableProfilerConfig = new TableProfilerConfig() .withProfileQuery("SELECT * FROM dual") @@ -1075,9 +1071,7 @@ public class TableResourceTest extends EntityResourceTest { columnProfilerConfigs.remove(0); columnProfilerConfigs.add( - new ColumnProfilerConfig() - .withColumnName("c2") - .withMetrics(List.of("valuesCount", "valuePercentage", "validCount", "duplicateCount"))); + getColumnProfilerConfig("c2", "valuesCount", "valuePercentage", "validCount", "duplicateCount")); tableProfilerConfig = new TableProfilerConfig() .withProfileQuery("SELECT * FROM dual1") @@ -1120,17 +1114,9 @@ public class TableResourceTest extends EntityResourceTest { void putTableProfile(Table table, Table table1, Map authHeaders) throws IOException, ParseException { Long timestamp = TestUtils.dateToTimestamp("2021-09-09"); - ColumnProfile c1Profile = - new ColumnProfile().withName("c1").withMax(100.0).withMin(10.0).withUniqueCount(100.0).withTimestamp(timestamp); - ColumnProfile c2Profile = - new ColumnProfile().withName("c2").withMax(99.0).withMin(20.0).withUniqueCount(89.0).withTimestamp(timestamp); - ColumnProfile c3Profile = - new ColumnProfile() - .withName("\"c.3\"") - .withMax(75.0) - .withMin(25.0) - .withUniqueCount(77.0) - .withTimestamp(timestamp); + ColumnProfile c1Profile = getColumnProfile("c1", 100.0, 10.0, 100.0, timestamp); + ColumnProfile c2Profile = getColumnProfile("c2", 99.0, 20.0, 89.0, timestamp); + ColumnProfile c3Profile = getColumnProfile("\"c.3\"", 75.0, 25.0, 77.0, timestamp); // Add column profiles List columnProfiles = List.of(c1Profile, c2Profile, c3Profile); List columnProfileResults = new ArrayList<>(); @@ -1158,17 +1144,9 @@ public class TableResourceTest extends EntityResourceTest { // Add new date for TableProfile TableProfile newTableProfile = new TableProfile().withRowCount(7.0).withColumnCount(3.0).withTimestamp(timestamp); - c1Profile = - new ColumnProfile().withName("c1").withMax(100.0).withMin(10.0).withUniqueCount(100.0).withTimestamp(timestamp); - c2Profile = - new ColumnProfile().withName("c2").withMax(99.0).withMin(20.0).withUniqueCount(89.0).withTimestamp(timestamp); - c3Profile = - new ColumnProfile() - .withName("\"c.3\"") - .withMax(75.0) - .withMin(25.0) - .withUniqueCount(77.0) - .withTimestamp(timestamp); + c1Profile = getColumnProfile("c1", 100.0, 10.0, 100.0, timestamp); + c2Profile = getColumnProfile("c2", 99.0, 20.0, 89.0, timestamp); + c3Profile = getColumnProfile("\"c.3\"", 75.0, 25.0, 77.0, timestamp); columnProfiles = List.of(c1Profile, c2Profile, c3Profile); columnProfileResults.add(c1Profile); createTableProfile = new CreateTableProfile().withTableProfile(newTableProfile).withColumnProfile(columnProfiles); @@ -1216,22 +1194,9 @@ public class TableResourceTest extends EntityResourceTest { timestamp = TestUtils.dateToTimestamp(dateStr + i); tableProfile = new TableProfile().withRowCount(21.0).withColumnCount(3.0).withTimestamp(timestamp); createTableProfile.setTableProfile(tableProfile); - c1Profile = - new ColumnProfile() - .withName("c1") - .withMax(100.0) - .withMin(10.0) - .withUniqueCount(100.0) - .withTimestamp(timestamp); - c2Profile = - new ColumnProfile().withName("c2").withMax(99.0).withMin(20.0).withUniqueCount(89.0).withTimestamp(timestamp); - c3Profile = - new ColumnProfile() - .withName("\"c.3\"") - .withMax(75.0) - .withMin(25.0) - .withUniqueCount(77.0) - .withTimestamp(timestamp); + c1Profile = getColumnProfile("c1", 100.0, 10.0, 100.0, timestamp); + c2Profile = getColumnProfile("c2", 99.0, 20.0, 89.0, timestamp); + c3Profile = getColumnProfile("\"c.3\"", 75.0, 25.0, 77.0, timestamp); columnProfiles = List.of(c1Profile, c2Profile, c3Profile); columnProfileResults.add(c1Profile); createTableProfile = new CreateTableProfile().withTableProfile(tableProfile).withColumnProfile(columnProfiles); @@ -1256,29 +1221,14 @@ public class TableResourceTest extends EntityResourceTest { // Add profiles for table1 List table1ProfileList = new ArrayList<>(); - List> table1ColumnProfileList = new ArrayList<>(); dateStr = "2021-10-"; for (int i = 11; i <= 15; i++) { timestamp = TestUtils.dateToTimestamp(dateStr + i); tableProfile = new TableProfile().withRowCount(21.0).withColumnCount(3.0).withTimestamp(timestamp); - c1Profile = - new ColumnProfile() - .withName("c1") - .withMax(100.0) - .withMin(10.0) - .withUniqueCount(100.0) - .withTimestamp(timestamp); - c2Profile = - new ColumnProfile().withName("c2").withMax(99.0).withMin(20.0).withUniqueCount(89.0).withTimestamp(timestamp); - c3Profile = - new ColumnProfile() - .withName("\"c.3\"") - .withMax(75.0) - .withMin(25.0) - .withUniqueCount(77.0) - .withTimestamp(timestamp); + c1Profile = getColumnProfile("c1", 100.0, 10.0, 100.0, timestamp); + c2Profile = getColumnProfile("c2", 88.0, 20.0, 89.0, timestamp); + c3Profile = getColumnProfile("\"c.3\"", 75.0, 25.0, 77.0, timestamp); columnProfiles = List.of(c1Profile, c2Profile, c3Profile); - table1ColumnProfileList.add(columnProfiles); createTableProfile = new CreateTableProfile().withTableProfile(tableProfile).withColumnProfile(columnProfiles); putTableProfileData(table1.getId(), createTableProfile, authHeaders); table1ProfileList.add(tableProfile); @@ -1308,17 +1258,9 @@ public class TableResourceTest extends EntityResourceTest { void put_tableInvalidTableProfileData_4xx(TestInfo test) throws IOException, ParseException { Table table = createAndCheckEntity(createRequest(test), ADMIN_AUTH_HEADERS); Long timestamp = TestUtils.dateToTimestamp("2021-09-10"); - ColumnProfile c1Profile = - new ColumnProfile().withName("c1").withTimestamp(timestamp).withMax(100.0).withMin(10.0).withUniqueCount(100.0); - ColumnProfile c2Profile = - new ColumnProfile().withName("c2").withTimestamp(timestamp).withMax(99.0).withMin(20.0).withUniqueCount(89.0); - ColumnProfile c3Profile = - new ColumnProfile() - .withName("invalidColumn") - .withTimestamp(timestamp) - .withMax(75.0) - .withMin(25.0) - .withUniqueCount(77.0); + ColumnProfile c1Profile = getColumnProfile("c1", 100.0, 10.0, 100.0, timestamp); + ColumnProfile c2Profile = getColumnProfile("c2", 99.0, 20.0, 89.0, timestamp); + ColumnProfile c3Profile = getColumnProfile("invalidColumn", 75.0, 25.0, 77.0, timestamp); List columnProfiles = List.of(c1Profile, c2Profile, c3Profile); TableProfile tableProfile = new TableProfile().withRowCount(6.0).withColumnCount(3.0).withTimestamp(timestamp); CreateTableProfile createTableProfile = @@ -1517,10 +1459,10 @@ public class TableResourceTest extends EntityResourceTest { assertEquals(3, getTagCategoryUsageCount("User", ADMIN_AUTH_HEADERS)); // Total 1 glossary1 tags - 1 column - assertEquals(1, getGlossaryUsageCount("g1", ADMIN_AUTH_HEADERS)); + assertEquals(1, getGlossaryUsageCount("g1")); // Total 1 glossary2 tags - 1 table - assertEquals(1, getGlossaryUsageCount("g2", ADMIN_AUTH_HEADERS)); + assertEquals(1, getGlossaryUsageCount("g2")); // Total 3 USER_ADDRESS tags - 1 table tag and 2 column tags assertEquals(3, getTagUsageCount(USER_ADDRESS_TAG_LABEL.getTagFQN(), ADMIN_AUTH_HEADERS)); @@ -1758,7 +1700,7 @@ public class TableResourceTest extends EntityResourceTest { Location location = locationResourceTest.createEntity(create, ADMIN_AUTH_HEADERS); addAndCheckLocation(table, location.getId(), OK, TEST_AUTH_HEADERS); // Delete location and make sure it is deleted - deleteAndCheckLocation(table, TEST_AUTH_HEADERS); + deleteAndCheckLocation(table); } @Test @@ -1798,10 +1740,29 @@ public class TableResourceTest extends EntityResourceTest { CatalogExceptionMessage.mutuallyExclusiveLabels(TIER2_TAG_LABEL, TIER1_TAG_LABEL)); } - private void deleteAndCheckLocation(Table table, Map authHeaders) throws HttpResponseException { + @Test + void test_ownershipInheritance(TestInfo test) throws HttpResponseException { + // When a databaseSchema has no owner set, it inherits the ownership from database + // When a table has no owner set, it inherits the ownership from databaseSchema + DatabaseResourceTest dbTest = new DatabaseResourceTest(); + Database db = dbTest.createEntity(dbTest.createRequest(test).withOwner(USER1_REF), ADMIN_AUTH_HEADERS); + + DatabaseSchemaResourceTest schemaTest = new DatabaseSchemaResourceTest(); + CreateDatabaseSchema createSchema = + schemaTest.createRequest(test).withDatabase(db.getEntityReference()).withOwner(null); + DatabaseSchema schema = schemaTest.createEntity(createSchema, ADMIN_AUTH_HEADERS); + assertEquals(USER1_REF, schema.getOwner()); // Ensure databaseSchema owner is inherited from database + + Table table = + createEntity( + createRequest(test).withOwner(null).withDatabaseSchema(schema.getEntityReference()), ADMIN_AUTH_HEADERS); + assertEquals(USER1_REF, table.getOwner()); // Ensure table owner is inherited from databaseSchema + } + + private void deleteAndCheckLocation(Table table) throws HttpResponseException { WebTarget target = OpenMetadataApplicationTest.getResource(String.format("tables/%s/location", table.getId())); - TestUtils.delete(target, authHeaders); - checkLocationDeleted(table.getId(), authHeaders); + TestUtils.delete(target, TestUtils.TEST_AUTH_HEADERS); + checkLocationDeleted(table.getId(), TestUtils.TEST_AUTH_HEADERS); } public void checkLocationDeleted(UUID tableId, Map authHeaders) throws HttpResponseException { @@ -2044,8 +2005,10 @@ public class TableResourceTest extends EntityResourceTest { return TagResourceTest.getCategory(name, "usageCount", authHeaders).getUsageCount(); } - private static int getGlossaryUsageCount(String name, Map authHeaders) throws HttpResponseException { - return new GlossaryResourceTest().getEntityByName(name, null, "usageCount", authHeaders).getUsageCount(); + private static int getGlossaryUsageCount(String name) throws HttpResponseException { + return new GlossaryResourceTest() + .getEntityByName(name, null, "usageCount", TestUtils.ADMIN_AUTH_HEADERS) + .getUsageCount(); } private static int getGlossaryTermUsageCount(String name, Map authHeaders) @@ -2241,4 +2204,17 @@ public class TableResourceTest extends EntityResourceTest { assertCommonFieldChange(fieldName, expected, actual); } } + + public ColumnProfilerConfig getColumnProfilerConfig(String name, String... metrics) { + return new ColumnProfilerConfig().withColumnName(name).withMetrics(List.of(metrics)); + } + + public ColumnProfile getColumnProfile(String name, Object max, Object min, Double uniqueCount, Long timestamp) { + return new ColumnProfile() + .withName(name) + .withMax(max) + .withMin(min) + .withUniqueCount(uniqueCount) + .withTimestamp(timestamp); + } }