Fixes #3968 EntityReference should contain name, displayName and FQN as separate fields (#3996)

This commit is contained in:
Suresh Srinivas 2022-04-10 08:03:28 -07:00 committed by GitHub
parent e354cdba4d
commit 2c93cd278b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
26 changed files with 84 additions and 73 deletions

View File

@ -54,7 +54,7 @@ public class ChartRepository extends EntityRepository<Chart> {
public static String getFQN(Chart chart) {
return (chart != null && chart.getService() != null)
? FullyQualifiedName.add(chart.getService().getName(), chart.getName())
? FullyQualifiedName.add(chart.getService().getFullyQualifiedName(), chart.getName())
: null;
}

View File

@ -54,7 +54,7 @@ public class DashboardRepository extends EntityRepository<Dashboard> {
public static String getFQN(Dashboard dashboard) {
return (dashboard != null && dashboard.getService() != null)
? FullyQualifiedName.add(dashboard.getService().getName(), dashboard.getName())
? FullyQualifiedName.add(dashboard.getService().getFullyQualifiedName(), dashboard.getName())
: null;
}

View File

@ -54,7 +54,7 @@ public class DatabaseRepository extends EntityRepository<Database> {
public static String getFQN(Database database) {
return (database != null && database.getService() != null)
? FullyQualifiedName.add(database.getService().getName(), database.getName())
? FullyQualifiedName.add(database.getService().getFullyQualifiedName(), database.getName())
: null;
}

View File

@ -51,7 +51,9 @@ public class DatabaseSchemaRepository extends EntityRepository<DatabaseSchema> {
}
public static String getFQN(DatabaseSchema schema) {
return (schema != null) ? FullyQualifiedName.add(schema.getDatabase().getName(), schema.getName()) : null;
return schema != null
? FullyQualifiedName.add(schema.getDatabase().getFullyQualifiedName(), schema.getName())
: null;
}
@Override

View File

@ -577,7 +577,7 @@ public abstract class EntityRepository<T> {
}
/** Validate given list of tags and add derived tags to it */
public final List<TagLabel> addDerivedTags(List<TagLabel> tagLabels) throws IOException {
public final List<TagLabel> addDerivedTags(List<TagLabel> tagLabels) {
if (tagLabels == null || tagLabels.isEmpty()) {
return tagLabels;
}
@ -605,7 +605,7 @@ public abstract class EntityRepository<T> {
}
/** Get tags associated with a given set of tags */
private List<TagLabel> getDerivedTags(TagLabel tagLabel, Tag tag) throws IOException {
private List<TagLabel> getDerivedTags(TagLabel tagLabel, Tag tag) {
List<TagLabel> derivedTags = new ArrayList<>();
for (String fqn : listOrEmpty(tag.getAssociatedTags())) {
Tag tempTag = daoCollection.tagDAO().findEntityByName(fqn);
@ -662,7 +662,7 @@ public abstract class EntityRepository<T> {
List<EntityReference> followers = findFrom(entityInterface.getId(), entityType, Relationship.FOLLOWS);
for (EntityReference follower : followers) {
User user = daoCollection.userDAO().findEntityById(follower.getId(), ALL);
follower.withName(user.getName()).withDeleted(user.getDeleted());
follower.withName(user.getName()).withDeleted(user.getDeleted()).withFullyQualifiedName(user.getName());
}
return followers;
}
@ -776,7 +776,11 @@ public abstract class EntityRepository<T> {
entityReferences.sort(EntityUtil.compareEntityReference);
for (EntityReference entityReference : entityReferences) {
EntityReference ref = daoCollection.userDAO().findEntityReferenceById(entityReference.getId());
entityReference.withType(ref.getType()).withName(ref.getName()).withDisplayName(ref.getDisplayName());
entityReference
.withType(ref.getType())
.withName(ref.getName())
.withDisplayName(ref.getDisplayName())
.withFullyQualifiedName(ref.getFullyQualifiedName());
}
}
}
@ -786,7 +790,11 @@ public abstract class EntityRepository<T> {
entityReferences.sort(EntityUtil.compareEntityReference);
for (EntityReference entityReference : entityReferences) {
EntityReference ref = daoCollection.roleDAO().findEntityReferenceById(entityReference.getId());
entityReference.withType(ref.getType()).withName(ref.getName()).withDisplayName(ref.getDisplayName());
entityReference
.withType(ref.getType())
.withName(ref.getName())
.withDisplayName(ref.getDisplayName())
.withFullyQualifiedName(ref.getFullyQualifiedName());
}
}
}

View File

@ -103,10 +103,11 @@ public class GlossaryTermRepository extends EntityRepository<GlossaryTerm> {
// Validate parent
if (entity.getParent() == null) {
entity.setFullyQualifiedName(FullyQualifiedName.add(entity.getGlossary().getName(), entity.getName()));
entity.setFullyQualifiedName(
FullyQualifiedName.add(entity.getGlossary().getFullyQualifiedName(), entity.getName()));
} else {
EntityReference parent = Entity.getEntityReference(entity.getParent());
entity.setFullyQualifiedName(FullyQualifiedName.add(parent.getName(), entity.getName()));
entity.setFullyQualifiedName(FullyQualifiedName.add(parent.getFullyQualifiedName(), entity.getName()));
entity.setParent(parent);
}

View File

@ -50,7 +50,7 @@ public class IngestionPipelineRepository extends EntityRepository<IngestionPipel
public static String getFQN(IngestionPipeline ingestionPipeline) {
return (ingestionPipeline != null && ingestionPipeline.getService() != null)
? FullyQualifiedName.add(ingestionPipeline.getService().getName(), ingestionPipeline.getName())
? FullyQualifiedName.add(ingestionPipeline.getService().getFullyQualifiedName(), ingestionPipeline.getName())
: null;
}

View File

@ -153,7 +153,7 @@ public class LocationRepository extends EntityRepository<Location> {
public static String getFQN(Location location) {
return (location != null && location.getService() != null)
? FullyQualifiedName.add(location.getService().getName(), location.getName())
? FullyQualifiedName.add(location.getService().getFullyQualifiedName(), location.getName())
: null;
}

View File

@ -49,7 +49,7 @@ public class MetricsRepository extends EntityRepository<Metrics> {
public static String getFQN(Metrics metrics) {
return (metrics != null && metrics.getService() != null)
? FullyQualifiedName.add(metrics.getService().getName(), metrics.getName())
? FullyQualifiedName.add(metrics.getService().getFullyQualifiedName(), metrics.getName())
: null;
}

View File

@ -91,7 +91,7 @@ public class MlModelRepository extends EntityRepository<MlModel> {
mlSources.forEach(
s -> {
if (s.getDataSource() != null) {
s.setFullyQualifiedName(FullyQualifiedName.add(s.getDataSource().getName(), s.getName()));
s.setFullyQualifiedName(FullyQualifiedName.add(s.getDataSource().getFullyQualifiedName(), s.getName()));
} else {
s.setFullyQualifiedName(s.getName());
}

View File

@ -63,7 +63,7 @@ public class PipelineRepository extends EntityRepository<Pipeline> {
public static String getFQN(Pipeline pipeline) {
return (pipeline != null && pipeline.getService() != null)
? FullyQualifiedName.add(pipeline.getService().getName(), pipeline.getName())
? FullyQualifiedName.add(pipeline.getService().getFullyQualifiedName(), pipeline.getName())
: null;
}

View File

@ -144,7 +144,7 @@ public class TableRepository extends EntityRepository<Table> {
public static String getFQN(Table table) {
return (table != null && table.getDatabaseSchema() != null)
? FullyQualifiedName.add(table.getDatabaseSchema().getName(), table.getName())
? FullyQualifiedName.add(table.getDatabaseSchema().getFullyQualifiedName(), table.getName())
: null;
}

View File

@ -42,7 +42,7 @@ public class TopicRepository extends EntityRepository<Topic> {
public static String getFQN(Topic topic) {
return (topic != null && topic.getService() != null)
? FullyQualifiedName.add(topic.getService().getName(), topic.getName())
? FullyQualifiedName.add(topic.getService().getFullyQualifiedName(), topic.getName())
: null;
}

View File

@ -67,7 +67,8 @@ public abstract class EntityInterface<T> {
public final EntityReference getEntityReference() {
return new EntityReference()
.withId(getId())
.withName(getFullyQualifiedName())
.withName(getName())
.withFullyQualifiedName(getFullyQualifiedName())
.withDescription(getDescription())
.withDisplayName(getDisplayName())
.withType(entityType)

View File

@ -167,7 +167,10 @@ public final class EntityUtil {
if (list != null) {
for (EntityReference ref : list) {
EntityReference ref2 = Entity.getEntityReferenceById(ref.getType(), ref.getId(), ALL);
ref.withDescription(ref2.getDescription()).withName(ref2.getName()).withDisplayName(ref2.getDisplayName());
ref.withDescription(ref2.getDescription())
.withName(ref2.getName())
.withDisplayName(ref2.getDisplayName())
.withFullyQualifiedName(ref2.getFullyQualifiedName());
}
}
return list;

View File

@ -66,7 +66,7 @@ public class FullyQualifiedName {
public static String getTableFQN(String columnFQN) {
// Split columnFQN of format databaseServiceName.databaseName.tableName.columnName
String[] split = split(columnFQN);
if (split.length < 5) {
if (split.length != 5) {
throw new IllegalArgumentException("Invalid fully qualified column name " + columnFQN);
}
// Return table FQN of format databaseService.tableName

View File

@ -24,7 +24,11 @@
"type": "string"
},
"name": {
"description": "Name of the entity instance. For entities such as tables, databases where the name is not unique, fullyQualifiedName is returned in this field.",
"description": "Name of the entity instance.",
"type": "string"
},
"fullyQualifiedName": {
"description": "Fully qualified name of the entity instance. For entities such as tables, databases fullyQualifiedName is returned in this field. For entities that don't have name hierarchy such as `user` and `team` this will be same as the `name` field.",
"type": "string"
},
"description": {

View File

@ -113,7 +113,7 @@ public class DashboardResourceTest extends EntityResourceTest<Dashboard, CreateD
ResultList<Dashboard> list = listEntities(queryParams, ADMIN_AUTH_HEADERS);
for (Dashboard db : list.getData()) {
assertEquals(service.getName(), db.getService().getName());
String expectedFQN = FullyQualifiedName.add(service.getName(), db.getName());
String expectedFQN = FullyQualifiedName.add(service.getFullyQualifiedName(), db.getName());
assertEquals(expectedFQN, db.getFullyQualifiedName());
}
}

View File

@ -77,7 +77,7 @@ public class DatabaseResourceTest extends EntityResourceTest<Database, CreateDat
CreateDatabase create = createRequest(test);
create.setService(new EntityReference().withId(SNOWFLAKE_REFERENCE.getId()).withType("databaseService"));
Database db = createAndCheckEntity(create, ADMIN_AUTH_HEADERS);
String expectedFQN = FullyQualifiedName.add(SNOWFLAKE_REFERENCE.getName(), create.getName());
String expectedFQN = FullyQualifiedName.add(SNOWFLAKE_REFERENCE.getFullyQualifiedName(), create.getName());
assertEquals(expectedFQN, db.getFullyQualifiedName());
}
@ -178,7 +178,8 @@ public class DatabaseResourceTest extends EntityResourceTest<Database, CreateDat
assertNotNull(database.getServiceType());
assertReference(createRequest.getService(), database.getService());
assertEquals(
FullyQualifiedName.add(database.getService().getName(), database.getName()), database.getFullyQualifiedName());
FullyQualifiedName.add(database.getService().getFullyQualifiedName(), database.getName()),
database.getFullyQualifiedName());
}
@Override
@ -191,7 +192,8 @@ public class DatabaseResourceTest extends EntityResourceTest<Database, CreateDat
// Validate service
assertReference(expected.getService(), updated.getService());
assertEquals(
FullyQualifiedName.add(updated.getService().getName(), updated.getName()), updated.getFullyQualifiedName());
FullyQualifiedName.add(updated.getService().getFullyQualifiedName(), updated.getName()),
updated.getFullyQualifiedName());
}
@Override

View File

@ -132,7 +132,8 @@ public class DatabaseSchemaResourceTest extends EntityResourceTest<DatabaseSchem
assertNotNull(schema.getServiceType());
assertReference(createRequest.getDatabase(), schema.getDatabase());
assertEquals(
FullyQualifiedName.add(schema.getDatabase().getName(), schema.getName()), schema.getFullyQualifiedName());
FullyQualifiedName.add(schema.getDatabase().getFullyQualifiedName(), schema.getName()),
schema.getFullyQualifiedName());
}
@Override
@ -145,7 +146,8 @@ public class DatabaseSchemaResourceTest extends EntityResourceTest<DatabaseSchem
// Validate service
assertReference(expected.getDatabase(), updated.getDatabase());
assertEquals(
FullyQualifiedName.add(updated.getDatabase().getName(), updated.getName()), updated.getFullyQualifiedName());
FullyQualifiedName.add(updated.getDatabase().getFullyQualifiedName(), updated.getName()),
updated.getFullyQualifiedName());
}
@Override

View File

@ -35,7 +35,6 @@ import static org.openmetadata.catalog.type.ColumnDataType.INT;
import static org.openmetadata.catalog.type.ColumnDataType.STRING;
import static org.openmetadata.catalog.type.ColumnDataType.STRUCT;
import static org.openmetadata.catalog.util.EntityUtil.tagLabelMatch;
import static org.openmetadata.catalog.util.FullyQualifiedName.add;
import static org.openmetadata.catalog.util.FullyQualifiedName.build;
import static org.openmetadata.catalog.util.RestUtil.DATE_FORMAT;
import static org.openmetadata.catalog.util.TestUtils.ADMIN_AUTH_HEADERS;
@ -1349,54 +1348,36 @@ public class TableResourceTest extends EntityResourceTest<Table, CreateTable> {
// GET .../tables?fields=columns,tableConstraints
final String fields = "tableConstraints";
queryParams =
new HashMap<>() {
{
put("fields", fields);
}
};
queryParams = new HashMap<>();
queryParams.put("fields", fields);
tableList = listEntities(queryParams, ADMIN_AUTH_HEADERS);
assertEquals(2, tableList.getData().size());
assertFields(tableList.getData(), fields);
// List tables with databaseFQN as filter
queryParams =
new HashMap<>() {
{
put("fields", fields);
put("database", DATABASE.getFullyQualifiedName());
}
};
queryParams = new HashMap<>();
queryParams.put("fields", fields);
queryParams.put("database", DATABASE.getFullyQualifiedName());
tableList1 = listEntities(queryParams, ADMIN_AUTH_HEADERS);
assertEquals(tableList.getData().size(), tableList1.getData().size());
assertFields(tableList1.getData(), fields);
// GET .../tables?fields=usageSummary,owner
final String fields1 = "usageSummary,owner";
queryParams =
new HashMap<>() {
{
put("fields", fields1);
}
};
queryParams = new HashMap<>();
queryParams.put("fields", fields1);
tableList = listEntities(queryParams, ADMIN_AUTH_HEADERS);
assertEquals(2, tableList.getData().size());
assertFields(tableList.getData(), fields1);
for (Table table : tableList.getData()) {
assertEquals(table.getOwner().getId(), USER_OWNER1.getId());
assertEquals(table.getOwner().getType(), USER_OWNER1.getType());
assertEquals(table.getDatabase().getId(), DATABASE.getId());
assertEquals(table.getDatabase().getName(), DATABASE.getFullyQualifiedName());
assertEquals(USER_OWNER1, table.getOwner());
assertReference(DATABASE_REFERENCE, table.getDatabase());
}
// List tables with databaseFQN as filter
queryParams =
new HashMap<>() {
{
put("fields", fields1);
put("database", DATABASE.getFullyQualifiedName());
}
};
queryParams = new HashMap<>();
queryParams.put("fields", fields1);
queryParams.put("database", DATABASE.getFullyQualifiedName());
tableList1 = listEntities(queryParams, ADMIN_AUTH_HEADERS);
assertEquals(tableList.getData().size(), tableList1.getData().size());
assertFields(tableList1.getData(), fields1);
@ -2012,7 +1993,7 @@ public class TableResourceTest extends EntityResourceTest<Table, CreateTable> {
TestUtils.validateEntityReferences(createdEntity.getFollowers());
assertListNotNull(createdEntity.getService(), createdEntity.getServiceType());
assertEquals(
add(createdEntity.getDatabaseSchema().getName(), createdEntity.getName()),
FullyQualifiedName.add(createdEntity.getDatabaseSchema().getFullyQualifiedName(), createdEntity.getName()),
createdEntity.getFullyQualifiedName());
}
@ -2050,7 +2031,7 @@ public class TableResourceTest extends EntityResourceTest<Table, CreateTable> {
TestUtils.validateTags(expected.getTags(), patched.getTags());
TestUtils.validateEntityReferences(expected.getFollowers());
assertEquals(
FullyQualifiedName.add(patched.getDatabaseSchema().getName(), patched.getName()),
FullyQualifiedName.add(patched.getDatabaseSchema().getFullyQualifiedName(), patched.getName()),
patched.getFullyQualifiedName());
}

View File

@ -143,7 +143,7 @@ public class LocationResourceTest extends EntityResourceTest<Location, CreateLoc
LocationList allLocations =
listPrefixes(
null,
FullyQualifiedName.add(AWS_STORAGE_SERVICE_REFERENCE.getName(), locationName),
FullyQualifiedName.add(AWS_STORAGE_SERVICE_REFERENCE.getFullyQualifiedName(), locationName),
1000000,
null,
null,

View File

@ -228,7 +228,7 @@ public class PipelineResourceTest extends EntityResourceTest<Pipeline, CreatePip
request.withPipelineUrl(pipelineURI).withConcurrency(pipelineConcurrency).withStartDate(startDate),
OK,
ADMIN_AUTH_HEADERS);
String expectedFQN = FullyQualifiedName.add(AIRFLOW_REFERENCE.getName(), pipeline.getName());
String expectedFQN = FullyQualifiedName.add(AIRFLOW_REFERENCE.getFullyQualifiedName(), pipeline.getName());
assertEquals(pipelineURI, pipeline.getPipelineUrl());
assertEquals(startDate, pipeline.getStartDate());
assertEquals(pipelineConcurrency, pipeline.getConcurrency());

View File

@ -170,7 +170,8 @@ public class DatabaseServiceResourceTest extends EntityResourceTest<DatabaseServ
assertEquals(1, updatedService.getPipelines().size());
EntityReference expectedPipeline = updatedService.getPipelines().get(0);
assertEquals(ingestionPipeline.getId(), expectedPipeline.getId());
assertEquals(ingestionPipeline.getFullyQualifiedName(), expectedPipeline.getName());
assertEquals(ingestionPipeline.getName(), expectedPipeline.getName());
assertEquals(ingestionPipeline.getFullyQualifiedName(), expectedPipeline.getFullyQualifiedName());
}
@Override

View File

@ -242,7 +242,7 @@ public class IngestionPipelineResourceTest extends EntityResourceTest<IngestionP
.withStartDate(startDate.toString())),
OK,
ADMIN_AUTH_HEADERS);
String expectedFQN = FullyQualifiedName.add(BIGQUERY_REFERENCE.getName(), ingestion.getName());
String expectedFQN = FullyQualifiedName.add(BIGQUERY_REFERENCE.getFullyQualifiedName(), ingestion.getName());
validateSourceConfig(DATABASE_METADATA_CONFIG, ingestion.getSource().getSourceConfig(), ingestion);
assertEquals(startDate.toString(), ingestion.getAirflowConfig().getStartDate());
assertEquals(pipelineConcurrency, ingestion.getAirflowConfig().getConcurrency());
@ -280,7 +280,7 @@ public class IngestionPipelineResourceTest extends EntityResourceTest<IngestionP
.withStartDate(startDate.toString())),
OK,
ADMIN_AUTH_HEADERS);
String expectedFQN = FullyQualifiedName.add(BIGQUERY_REFERENCE.getName(), ingestion.getName());
String expectedFQN = FullyQualifiedName.add(BIGQUERY_REFERENCE.getFullyQualifiedName(), ingestion.getName());
validateSourceConfig(queryUsageConfig, ingestion.getSource().getSourceConfig(), ingestion);
assertEquals(startDate.toString(), ingestion.getAirflowConfig().getStartDate());
assertEquals(pipelineConcurrency, ingestion.getAirflowConfig().getConcurrency());
@ -313,7 +313,7 @@ public class IngestionPipelineResourceTest extends EntityResourceTest<IngestionP
.withStartDate(startDate.toString())),
OK,
ADMIN_AUTH_HEADERS);
String expectedFQN = FullyQualifiedName.add(BIGQUERY_REFERENCE.getName(), ingestion.getName());
String expectedFQN = FullyQualifiedName.add(BIGQUERY_REFERENCE.getFullyQualifiedName(), ingestion.getName());
assertEquals(startDate.toString(), ingestion.getAirflowConfig().getStartDate());
assertEquals(pipelineConcurrency, ingestion.getAirflowConfig().getConcurrency());
assertEquals(expectedFQN, ingestion.getFullyQualifiedName());
@ -482,7 +482,7 @@ public class IngestionPipelineResourceTest extends EntityResourceTest<IngestionP
OK,
ADMIN_AUTH_HEADERS);
String expectedFQN = FullyQualifiedName.add(BIGQUERY_REFERENCE.getName(), ingestion.getName());
String expectedFQN = FullyQualifiedName.add(BIGQUERY_REFERENCE.getFullyQualifiedName(), ingestion.getName());
validateSourceConfig(DATABASE_METADATA_CONFIG, ingestion.getSource().getSourceConfig(), ingestionPipeline);
assertEquals(startDate.toString(), ingestion.getAirflowConfig().getStartDate());
assertEquals(pipelineConcurrency, ingestion.getAirflowConfig().getConcurrency());

View File

@ -263,18 +263,24 @@ public final class TestUtils {
}
public static void validateEntityReference(EntityReference ref) {
assertNotNull(ref.getId());
assertNotNull(ref.getHref());
assertNotNull(ref.getName());
assertNotNull(ref.getType());
assertNotNull(ref.getId(), invalidEntityReference(ref, "null Id"));
assertNotNull(ref.getHref(), invalidEntityReference(ref, "null href"));
assertNotNull(ref.getName(), invalidEntityReference(ref, "null name"));
assertNotNull(ref.getFullyQualifiedName(), invalidEntityReference(ref, "null fqn"));
assertNotNull(ref.getType(), invalidEntityReference(ref, "null type"));
// Ensure data entities use fully qualified name
if (List.of("table", "database", "metrics", "dashboard", "pipeline", "report", "topic", "chart", "location")
.contains(ref.getType())) {
// FullyQualifiedName has "." as separator
assertTrue(ref.getName().contains(SEPARATOR), "entity name is not fully qualified - " + ref.getName());
assertTrue(
ref.getFullyQualifiedName().contains(SEPARATOR), "entity name is not fully qualified - " + ref.getName());
}
}
public static String invalidEntityReference(EntityReference ref, String message) {
return String.format("%s:%s %s", ref.getType(), ref.getId(), message);
}
public static void validateEntityReferences(List<EntityReference> list) {
validateEntityReferences(list, false);
}