yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-11-08 15:04:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixes #8821 Inherit ownership from parent container when creating children (#8824)

Browse Source
This commit is contained in:
Suresh Srinivas 2022-11-16 13:59:45 -08:00 committed by GitHub
parent 27816adddd
commit 45d45aab70
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 91 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
openmetadata-service/src/main/java/org/openmetadata/service/Entity.java
View File

@ -222,10 +222,18 @@ public final class Entity {
return entityRepository.getFields(fields);
}
public static <T> T getEntity(EntityReference ref, String fields, Include include) throws IOException {
return getEntity(ref.getType(), ref.getId(), fields, include);
}
public static <T> T getEntity(EntityReference ref, EntityUtil.Fields fields, Include include) throws IOException {
return getEntity(ref.getType(), ref.getId(), fields, include);
}
public static <T> T getEntity(String entityType, UUID id, String fields, Include include) throws IOException {
return getEntity(entityType, id, getFields(entityType, fields), include);
}
/** Retrieve the entity using id from given entity reference and fields */
public static <T> T getEntity(String entityType, UUID id, EntityUtil.Fields fields, Include include)
throws IOException {

12
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DatabaseSchemaRepository.java
View File

@ -114,9 +114,13 @@ public class DatabaseSchemaRepository extends EntityRepository<DatabaseSchema> {
}
private void populateDatabase(DatabaseSchema schema) throws IOException {
Database database = Entity.getEntity(schema.getDatabase(), Fields.EMPTY_FIELDS, ALL);
schema.setDatabase(database.getEntityReference());
schema.setService(database.getService());
schema.setServiceType(database.getServiceType());
Database database = Entity.getEntity(schema.getDatabase(), "owner", ALL);
schema
.withDatabase(database.getEntityReference())
.withService(database.getService())
.withServiceType(database.getServiceType());
// Carry forward ownership from database, if necessary
schema.withOwner(schema.getOwner() == null ? database.getOwner() : schema.getOwner());
}
}

2
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java
View File

@ -477,11 +477,13 @@ public abstract class EntityRepository<T extends EntityInterface> {
return update(uriInfo, original, updated);
}
@SuppressWarnings("unused")
protected void postCreate(T entity) {
// Override to perform any operation required after creation.
// For example ingestion pipeline creates a pipeline in AirFlow.
}
@SuppressWarnings("unused")
protected void postUpdate(T entity) {
// Override to perform any operation required after an entity update.
// For example ingestion pipeline creates a pipeline in AirFlow.

3
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java
View File

@ -105,8 +105,7 @@ public class GlossaryTermRepository extends EntityRepository<GlossaryTerm> {
validateHierarchy(entity);
// Validate glossary
Fields glossaryFields = Entity.getFields(GLOSSARY, "reviewers");
Glossary glossary = Entity.getEntity(entity.getGlossary(), glossaryFields, Include.NON_DELETED);
Glossary glossary = Entity.getEntity(entity.getGlossary(), "reviewers", Include.NON_DELETED);
entity.setGlossary(glossary.getEntityReference());
// If reviewers is not set in the glossary term, then carry it from the glossary

18
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java
View File

@ -17,6 +17,7 @@ import static java.util.stream.Collectors.groupingBy;
import static java.util.stream.Collectors.toUnmodifiableList;
import static org.openmetadata.common.utils.CommonUtil.listOrEmpty;
import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty;
import static org.openmetadata.schema.type.Include.ALL;
import static org.openmetadata.service.Entity.DATABASE_SCHEMA;
import static org.openmetadata.service.Entity.FIELD_DESCRIPTION;
import static org.openmetadata.service.Entity.FIELD_DISPLAY_NAME;
@ -66,7 +67,6 @@ import org.openmetadata.schema.type.ColumnProfilerConfig;
import org.openmetadata.schema.type.DailyCount;
import org.openmetadata.schema.type.DataModel;
import org.openmetadata.schema.type.EntityReference;
import org.openmetadata.schema.type.Include;
import org.openmetadata.schema.type.JoinedWith;
import org.openmetadata.schema.type.Relationship;
import org.openmetadata.schema.type.SQLQuery;
@ -139,7 +139,7 @@ public class TableRepository extends EntityRepository<Table> {
private void setDefaultFields(Table table) throws IOException {
EntityReference schemaRef = getContainer(table.getId());
DatabaseSchema schema = Entity.getEntity(schemaRef, Fields.EMPTY_FIELDS, Include.ALL);
DatabaseSchema schema = Entity.getEntity(schemaRef, Fields.EMPTY_FIELDS, ALL);
table.withDatabaseSchema(schemaRef).withDatabase(schema.getDatabase()).withService(schema.getService());
}
@ -537,11 +537,15 @@ public class TableRepository extends EntityRepository<Table> {
@Override
public void prepare(Table table) throws IOException {
DatabaseSchema schema = Entity.getEntity(table.getDatabaseSchema(), Fields.EMPTY_FIELDS, Include.ALL);
table.setDatabaseSchema(schema.getEntityReference());
table.setDatabase(schema.getDatabase());
table.setService(schema.getService());
table.setServiceType(schema.getServiceType());
DatabaseSchema schema = Entity.getEntity(table.getDatabaseSchema(), "owner", ALL);
table
.withDatabaseSchema(schema.getEntityReference())
.withDatabase(schema.getDatabase())
.withService(schema.getService())
.withServiceType(schema.getServiceType());
// Carry forward ownership from database schema
table.setOwner(table.getOwner() == null ? schema.getOwner() : table.getOwner());
// Validate column tags
addDerivedColumnTags(table.getColumns());

146
openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/TableResourceTest.java
View File

@ -93,6 +93,7 @@ import org.openmetadata.schema.api.data.CreateTable;
import org.openmetadata.schema.api.data.CreateTableProfile;
import org.openmetadata.schema.api.tests.CreateCustomMetric;
import org.openmetadata.schema.entity.data.Database;
import org.openmetadata.schema.entity.data.DatabaseSchema;
import org.openmetadata.schema.entity.data.Location;
import org.openmetadata.schema.entity.data.Table;
import org.openmetadata.schema.entity.services.DatabaseService;
@ -1056,13 +1057,8 @@ public class TableResourceTest extends EntityResourceTest<Table, CreateTable> {
void putProfileConfig(Table table, Map<String, String> authHeaders) throws IOException {
List<ColumnProfilerConfig> columnProfilerConfigs = new ArrayList<>();
columnProfilerConfigs.add(
new ColumnProfilerConfig()
.withColumnName("c1")
.withMetrics(List.of("valuesCount", "valuePercentage", "validCount", "duplicateCount")));
columnProfilerConfigs.add(
new ColumnProfilerConfig()
.withColumnName("\"c.3\"")
.withMetrics(List.of("duplicateCount", "nullCount", "missingCount")));
getColumnProfilerConfig("c1", "valuesCount", "valuePercentage", "validCount", "duplicateCount"));
columnProfilerConfigs.add(getColumnProfilerConfig("\"c.3\"", "duplicateCount", "nullCount", "missingCount"));
TableProfilerConfig tableProfilerConfig =
new TableProfilerConfig()
.withProfileQuery("SELECT * FROM dual")
@ -1075,9 +1071,7 @@ public class TableResourceTest extends EntityResourceTest<Table, CreateTable> {
columnProfilerConfigs.remove(0);
columnProfilerConfigs.add(
new ColumnProfilerConfig()
.withColumnName("c2")
.withMetrics(List.of("valuesCount", "valuePercentage", "validCount", "duplicateCount")));
getColumnProfilerConfig("c2", "valuesCount", "valuePercentage", "validCount", "duplicateCount"));
tableProfilerConfig =
new TableProfilerConfig()
.withProfileQuery("SELECT * FROM dual1")
@ -1120,17 +1114,9 @@ public class TableResourceTest extends EntityResourceTest<Table, CreateTable> {
void putTableProfile(Table table, Table table1, Map<String, String> authHeaders) throws IOException, ParseException {
Long timestamp = TestUtils.dateToTimestamp("2021-09-09");
ColumnProfile c1Profile =
new ColumnProfile().withName("c1").withMax(100.0).withMin(10.0).withUniqueCount(100.0).withTimestamp(timestamp);
ColumnProfile c2Profile =
new ColumnProfile().withName("c2").withMax(99.0).withMin(20.0).withUniqueCount(89.0).withTimestamp(timestamp);
ColumnProfile c3Profile =
new ColumnProfile()
.withName("\"c.3\"")
.withMax(75.0)
.withMin(25.0)
.withUniqueCount(77.0)
.withTimestamp(timestamp);
ColumnProfile c1Profile = getColumnProfile("c1", 100.0, 10.0, 100.0, timestamp);
ColumnProfile c2Profile = getColumnProfile("c2", 99.0, 20.0, 89.0, timestamp);
ColumnProfile c3Profile = getColumnProfile("\"c.3\"", 75.0, 25.0, 77.0, timestamp);
// Add column profiles
List<ColumnProfile> columnProfiles = List.of(c1Profile, c2Profile, c3Profile);
List<ColumnProfile> columnProfileResults = new ArrayList<>();
@ -1158,17 +1144,9 @@ public class TableResourceTest extends EntityResourceTest<Table, CreateTable> {
// Add new date for TableProfile
TableProfile newTableProfile = new TableProfile().withRowCount(7.0).withColumnCount(3.0).withTimestamp(timestamp);
c1Profile =
new ColumnProfile().withName("c1").withMax(100.0).withMin(10.0).withUniqueCount(100.0).withTimestamp(timestamp);
c2Profile =
new ColumnProfile().withName("c2").withMax(99.0).withMin(20.0).withUniqueCount(89.0).withTimestamp(timestamp);
c3Profile =
new ColumnProfile()
.withName("\"c.3\"")
.withMax(75.0)
.withMin(25.0)
.withUniqueCount(77.0)
.withTimestamp(timestamp);
c1Profile = getColumnProfile("c1", 100.0, 10.0, 100.0, timestamp);
c2Profile = getColumnProfile("c2", 99.0, 20.0, 89.0, timestamp);
c3Profile = getColumnProfile("\"c.3\"", 75.0, 25.0, 77.0, timestamp);
columnProfiles = List.of(c1Profile, c2Profile, c3Profile);
columnProfileResults.add(c1Profile);
createTableProfile = new CreateTableProfile().withTableProfile(newTableProfile).withColumnProfile(columnProfiles);
@ -1216,22 +1194,9 @@ public class TableResourceTest extends EntityResourceTest<Table, CreateTable> {
timestamp = TestUtils.dateToTimestamp(dateStr + i);
tableProfile = new TableProfile().withRowCount(21.0).withColumnCount(3.0).withTimestamp(timestamp);
createTableProfile.setTableProfile(tableProfile);
c1Profile =
new ColumnProfile()
.withName("c1")
.withMax(100.0)
.withMin(10.0)
.withUniqueCount(100.0)
.withTimestamp(timestamp);
c2Profile =
new ColumnProfile().withName("c2").withMax(99.0).withMin(20.0).withUniqueCount(89.0).withTimestamp(timestamp);
c3Profile =
new ColumnProfile()
.withName("\"c.3\"")
.withMax(75.0)
.withMin(25.0)
.withUniqueCount(77.0)
.withTimestamp(timestamp);
c1Profile = getColumnProfile("c1", 100.0, 10.0, 100.0, timestamp);
c2Profile = getColumnProfile("c2", 99.0, 20.0, 89.0, timestamp);
c3Profile = getColumnProfile("\"c.3\"", 75.0, 25.0, 77.0, timestamp);
columnProfiles = List.of(c1Profile, c2Profile, c3Profile);
columnProfileResults.add(c1Profile);
createTableProfile = new CreateTableProfile().withTableProfile(tableProfile).withColumnProfile(columnProfiles);
@ -1256,29 +1221,14 @@ public class TableResourceTest extends EntityResourceTest<Table, CreateTable> {
// Add profiles for table1
List<TableProfile> table1ProfileList = new ArrayList<>();
List<List<ColumnProfile>> table1ColumnProfileList = new ArrayList<>();
dateStr = "2021-10-";
for (int i = 11; i <= 15; i++) {
timestamp = TestUtils.dateToTimestamp(dateStr + i);
tableProfile = new TableProfile().withRowCount(21.0).withColumnCount(3.0).withTimestamp(timestamp);
c1Profile =
new ColumnProfile()
.withName("c1")
.withMax(100.0)
.withMin(10.0)
.withUniqueCount(100.0)
.withTimestamp(timestamp);
c2Profile =
new ColumnProfile().withName("c2").withMax(99.0).withMin(20.0).withUniqueCount(89.0).withTimestamp(timestamp);
c3Profile =
new ColumnProfile()
.withName("\"c.3\"")
.withMax(75.0)
.withMin(25.0)
.withUniqueCount(77.0)
.withTimestamp(timestamp);
c1Profile = getColumnProfile("c1", 100.0, 10.0, 100.0, timestamp);
c2Profile = getColumnProfile("c2", 88.0, 20.0, 89.0, timestamp);
c3Profile = getColumnProfile("\"c.3\"", 75.0, 25.0, 77.0, timestamp);
columnProfiles = List.of(c1Profile, c2Profile, c3Profile);
table1ColumnProfileList.add(columnProfiles);
createTableProfile = new CreateTableProfile().withTableProfile(tableProfile).withColumnProfile(columnProfiles);
putTableProfileData(table1.getId(), createTableProfile, authHeaders);
table1ProfileList.add(tableProfile);
@ -1308,17 +1258,9 @@ public class TableResourceTest extends EntityResourceTest<Table, CreateTable> {
void put_tableInvalidTableProfileData_4xx(TestInfo test) throws IOException, ParseException {
Table table = createAndCheckEntity(createRequest(test), ADMIN_AUTH_HEADERS);
Long timestamp = TestUtils.dateToTimestamp("2021-09-10");
ColumnProfile c1Profile =
new ColumnProfile().withName("c1").withTimestamp(timestamp).withMax(100.0).withMin(10.0).withUniqueCount(100.0);
ColumnProfile c2Profile =
new ColumnProfile().withName("c2").withTimestamp(timestamp).withMax(99.0).withMin(20.0).withUniqueCount(89.0);
ColumnProfile c3Profile =
new ColumnProfile()
.withName("invalidColumn")
.withTimestamp(timestamp)
.withMax(75.0)
.withMin(25.0)
.withUniqueCount(77.0);
ColumnProfile c1Profile = getColumnProfile("c1", 100.0, 10.0, 100.0, timestamp);
ColumnProfile c2Profile = getColumnProfile("c2", 99.0, 20.0, 89.0, timestamp);
ColumnProfile c3Profile = getColumnProfile("invalidColumn", 75.0, 25.0, 77.0, timestamp);
List<ColumnProfile> columnProfiles = List.of(c1Profile, c2Profile, c3Profile);
TableProfile tableProfile = new TableProfile().withRowCount(6.0).withColumnCount(3.0).withTimestamp(timestamp);
CreateTableProfile createTableProfile =
@ -1517,10 +1459,10 @@ public class TableResourceTest extends EntityResourceTest<Table, CreateTable> {
assertEquals(3, getTagCategoryUsageCount("User", ADMIN_AUTH_HEADERS));
// Total 1 glossary1 tags - 1 column
assertEquals(1, getGlossaryUsageCount("g1", ADMIN_AUTH_HEADERS));
assertEquals(1, getGlossaryUsageCount("g1"));
// Total 1 glossary2 tags - 1 table
assertEquals(1, getGlossaryUsageCount("g2", ADMIN_AUTH_HEADERS));
assertEquals(1, getGlossaryUsageCount("g2"));
// Total 3 USER_ADDRESS tags - 1 table tag and 2 column tags
assertEquals(3, getTagUsageCount(USER_ADDRESS_TAG_LABEL.getTagFQN(), ADMIN_AUTH_HEADERS));
@ -1758,7 +1700,7 @@ public class TableResourceTest extends EntityResourceTest<Table, CreateTable> {
Location location = locationResourceTest.createEntity(create, ADMIN_AUTH_HEADERS);
addAndCheckLocation(table, location.getId(), OK, TEST_AUTH_HEADERS);
// Delete location and make sure it is deleted
deleteAndCheckLocation(table, TEST_AUTH_HEADERS);
deleteAndCheckLocation(table);
}
@Test
@ -1798,10 +1740,29 @@ public class TableResourceTest extends EntityResourceTest<Table, CreateTable> {
CatalogExceptionMessage.mutuallyExclusiveLabels(TIER2_TAG_LABEL, TIER1_TAG_LABEL));
}
private void deleteAndCheckLocation(Table table, Map<String, String> authHeaders) throws HttpResponseException {
@Test
void test_ownershipInheritance(TestInfo test) throws HttpResponseException {
// When a databaseSchema has no owner set, it inherits the ownership from database
// When a table has no owner set, it inherits the ownership from databaseSchema
DatabaseResourceTest dbTest = new DatabaseResourceTest();
Database db = dbTest.createEntity(dbTest.createRequest(test).withOwner(USER1_REF), ADMIN_AUTH_HEADERS);
DatabaseSchemaResourceTest schemaTest = new DatabaseSchemaResourceTest();
CreateDatabaseSchema createSchema =
schemaTest.createRequest(test).withDatabase(db.getEntityReference()).withOwner(null);
DatabaseSchema schema = schemaTest.createEntity(createSchema, ADMIN_AUTH_HEADERS);
assertEquals(USER1_REF, schema.getOwner()); // Ensure databaseSchema owner is inherited from database
Table table =
createEntity(
createRequest(test).withOwner(null).withDatabaseSchema(schema.getEntityReference()), ADMIN_AUTH_HEADERS);
assertEquals(USER1_REF, table.getOwner()); // Ensure table owner is inherited from databaseSchema
}
private void deleteAndCheckLocation(Table table) throws HttpResponseException {
WebTarget target = OpenMetadataApplicationTest.getResource(String.format("tables/%s/location", table.getId()));
TestUtils.delete(target, authHeaders);
checkLocationDeleted(table.getId(), authHeaders);
TestUtils.delete(target, TestUtils.TEST_AUTH_HEADERS);
checkLocationDeleted(table.getId(), TestUtils.TEST_AUTH_HEADERS);
}
public void checkLocationDeleted(UUID tableId, Map<String, String> authHeaders) throws HttpResponseException {
@ -2044,8 +2005,10 @@ public class TableResourceTest extends EntityResourceTest<Table, CreateTable> {
return TagResourceTest.getCategory(name, "usageCount", authHeaders).getUsageCount();
}
private static int getGlossaryUsageCount(String name, Map<String, String> authHeaders) throws HttpResponseException {
return new GlossaryResourceTest().getEntityByName(name, null, "usageCount", authHeaders).getUsageCount();
private static int getGlossaryUsageCount(String name) throws HttpResponseException {
return new GlossaryResourceTest()
.getEntityByName(name, null, "usageCount", TestUtils.ADMIN_AUTH_HEADERS)
.getUsageCount();
}
private static int getGlossaryTermUsageCount(String name, Map<String, String> authHeaders)
@ -2241,4 +2204,17 @@ public class TableResourceTest extends EntityResourceTest<Table, CreateTable> {
assertCommonFieldChange(fieldName, expected, actual);
}
}
public ColumnProfilerConfig getColumnProfilerConfig(String name, String... metrics) {
return new ColumnProfilerConfig().withColumnName(name).withMetrics(List.of(metrics));
}
public ColumnProfile getColumnProfile(String name, Object max, Object min, Double uniqueCount, Long timestamp) {
return new ColumnProfile()
.withName(name)
.withMax(max)
.withMin(min)
.withUniqueCount(uniqueCount)
.withTimestamp(timestamp);
}
}