mirror of
https://github.com/open-metadata/OpenMetadata.git
synced 2025-12-13 00:22:23 +00:00
Bug: Repositories overriding fieldSetters can fail to load tags at entity level (#22622)
* Bug: Repositories overriding fieldSetters can fail to load tags at entity level * Bug: Repositories overriding fieldSetters can fail to load tags at entity level * fix build * Fix Test * Fix Test * fix test --------- Co-authored-by: sonikashah <sonikashah94@gmail.com> Co-authored-by: sonika-shah <58761340+sonika-shah@users.noreply.github.com>
This commit is contained in:
Sriharsha Chintalapani
GitHub
parent
866117360a
commit
5e41039b97
@ -25,6 +25,7 @@ import static org.openmetadata.service.resources.tags.TagLabelUtil.addDerivedTag
|
||||
import static org.openmetadata.service.resources.tags.TagLabelUtil.checkMutuallyExclusive;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.function.BiPredicate;
|
||||
@ -184,32 +185,45 @@ public class APIEndpointRepository extends EntityRepository<APIEndpoint> {
|
||||
return;
|
||||
}
|
||||
|
||||
// Bulk fetch tags for request schemas
|
||||
List<APIEndpoint> endpointsWithRequestSchema =
|
||||
apiEndpoints.stream()
|
||||
.filter(e -> e.getRequestSchema() != null)
|
||||
.collect(java.util.stream.Collectors.toList());
|
||||
|
||||
if (!endpointsWithRequestSchema.isEmpty()) {
|
||||
bulkPopulateEntityFieldTags(
|
||||
endpointsWithRequestSchema,
|
||||
entityType,
|
||||
e -> e.getRequestSchema().getSchemaFields(),
|
||||
e -> e.getFullyQualifiedName() + ".requestSchema");
|
||||
// First, fetch endpoint-level tags (important for search indexing)
|
||||
List<String> entityFQNs =
|
||||
apiEndpoints.stream().map(APIEndpoint::getFullyQualifiedName).toList();
|
||||
Map<String, List<TagLabel>> tagsMap = batchFetchTags(entityFQNs);
|
||||
for (APIEndpoint endpoint : apiEndpoints) {
|
||||
endpoint.setTags(
|
||||
addDerivedTags(
|
||||
tagsMap.getOrDefault(endpoint.getFullyQualifiedName(), Collections.emptyList())));
|
||||
}
|
||||
|
||||
// Bulk fetch tags for response schemas
|
||||
List<APIEndpoint> endpointsWithResponseSchema =
|
||||
apiEndpoints.stream()
|
||||
.filter(e -> e.getResponseSchema() != null)
|
||||
.collect(java.util.stream.Collectors.toList());
|
||||
// Then, if schemas are requested, also fetch schema field tags
|
||||
if (fields.contains("requestSchema") || fields.contains("responseSchema")) {
|
||||
// Bulk fetch tags for request schemas
|
||||
List<APIEndpoint> endpointsWithRequestSchema =
|
||||
apiEndpoints.stream()
|
||||
.filter(e -> e.getRequestSchema() != null)
|
||||
.collect(java.util.stream.Collectors.toList());
|
||||
|
||||
if (!endpointsWithResponseSchema.isEmpty()) {
|
||||
bulkPopulateEntityFieldTags(
|
||||
endpointsWithResponseSchema,
|
||||
entityType,
|
||||
e -> e.getResponseSchema().getSchemaFields(),
|
||||
e -> e.getFullyQualifiedName() + ".responseSchema");
|
||||
if (!endpointsWithRequestSchema.isEmpty()) {
|
||||
bulkPopulateEntityFieldTags(
|
||||
endpointsWithRequestSchema,
|
||||
entityType,
|
||||
e -> e.getRequestSchema().getSchemaFields(),
|
||||
e -> e.getFullyQualifiedName() + ".requestSchema");
|
||||
}
|
||||
|
||||
// Bulk fetch tags for response schemas
|
||||
List<APIEndpoint> endpointsWithResponseSchema =
|
||||
apiEndpoints.stream()
|
||||
.filter(e -> e.getResponseSchema() != null)
|
||||
.collect(java.util.stream.Collectors.toList());
|
||||
|
||||
if (!endpointsWithResponseSchema.isEmpty()) {
|
||||
bulkPopulateEntityFieldTags(
|
||||
endpointsWithResponseSchema,
|
||||
entityType,
|
||||
e -> e.getResponseSchema().getSchemaFields(),
|
||||
e -> e.getFullyQualifiedName() + ".responseSchema");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -10,10 +10,12 @@ import static org.openmetadata.service.Entity.FIELD_TAGS;
|
||||
import static org.openmetadata.service.Entity.STORAGE_SERVICE;
|
||||
import static org.openmetadata.service.Entity.getEntityReferenceById;
|
||||
import static org.openmetadata.service.Entity.populateEntityFieldTags;
|
||||
import static org.openmetadata.service.resources.tags.TagLabelUtil.addDerivedTags;
|
||||
import static org.openmetadata.service.util.EntityUtil.getEntityReferences;
|
||||
|
||||
import com.google.common.collect.Lists;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
@ -102,18 +104,31 @@ public class ContainerRepository extends EntityRepository<Container> {
|
||||
if (!fields.contains(FIELD_TAGS) || containers == null || containers.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
// Filter containers that have data models and use bulk tag fetching
|
||||
List<Container> containersWithDataModels =
|
||||
containers.stream()
|
||||
.filter(c -> c.getDataModel() != null)
|
||||
.collect(java.util.stream.Collectors.toList());
|
||||
|
||||
if (!containersWithDataModels.isEmpty()) {
|
||||
bulkPopulateEntityFieldTags(
|
||||
containersWithDataModels,
|
||||
entityType,
|
||||
c -> c.getDataModel().getColumns(),
|
||||
Container::getFullyQualifiedName);
|
||||
// First, fetch container-level tags (important for search indexing)
|
||||
List<String> entityFQNs = containers.stream().map(Container::getFullyQualifiedName).toList();
|
||||
Map<String, List<TagLabel>> tagsMap = batchFetchTags(entityFQNs);
|
||||
for (Container container : containers) {
|
||||
container.setTags(
|
||||
addDerivedTags(
|
||||
tagsMap.getOrDefault(container.getFullyQualifiedName(), Collections.emptyList())));
|
||||
}
|
||||
|
||||
// Then, if dataModel field is requested, also fetch data model column tags
|
||||
if (fields.contains("dataModel")) {
|
||||
// Filter containers that have data models and use bulk tag fetching
|
||||
List<Container> containersWithDataModels =
|
||||
containers.stream()
|
||||
.filter(c -> c.getDataModel() != null)
|
||||
.collect(java.util.stream.Collectors.toList());
|
||||
|
||||
if (!containersWithDataModels.isEmpty()) {
|
||||
bulkPopulateEntityFieldTags(
|
||||
containersWithDataModels,
|
||||
entityType,
|
||||
c -> c.getDataModel().getColumns(),
|
||||
Container::getFullyQualifiedName);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -17,9 +17,12 @@ import static org.openmetadata.schema.type.Include.ALL;
|
||||
import static org.openmetadata.service.Entity.DASHBOARD_DATA_MODEL;
|
||||
import static org.openmetadata.service.Entity.FIELD_TAGS;
|
||||
import static org.openmetadata.service.Entity.populateEntityFieldTags;
|
||||
import static org.openmetadata.service.resources.tags.TagLabelUtil.addDerivedTags;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.UUID;
|
||||
import lombok.SneakyThrows;
|
||||
@ -181,12 +184,26 @@ public class DashboardDataModelRepository extends EntityRepository<DashboardData
|
||||
if (!fields.contains(FIELD_TAGS) || dataModels == null || dataModels.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
// Use bulk tag fetching to avoid N+1 queries
|
||||
bulkPopulateEntityFieldTags(
|
||||
dataModels,
|
||||
entityType,
|
||||
DashboardDataModel::getColumns,
|
||||
DashboardDataModel::getFullyQualifiedName);
|
||||
|
||||
// First, fetch entity-level tags (important for search indexing)
|
||||
List<String> entityFQNs =
|
||||
dataModels.stream().map(DashboardDataModel::getFullyQualifiedName).toList();
|
||||
Map<String, List<TagLabel>> tagsMap = batchFetchTags(entityFQNs);
|
||||
for (DashboardDataModel dataModel : dataModels) {
|
||||
dataModel.setTags(
|
||||
addDerivedTags(
|
||||
tagsMap.getOrDefault(dataModel.getFullyQualifiedName(), Collections.emptyList())));
|
||||
}
|
||||
|
||||
// Then, if columns field is requested, also fetch column-level tags
|
||||
if (fields.contains("columns")) {
|
||||
// Use bulk tag fetching to avoid N+1 queries
|
||||
bulkPopulateEntityFieldTags(
|
||||
dataModels,
|
||||
entityType,
|
||||
DashboardDataModel::getColumns,
|
||||
DashboardDataModel::getFullyQualifiedName);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
|
||||
@ -5323,7 +5323,7 @@ public abstract class EntityRepository<T extends EntityInterface> {
|
||||
return tags.stream().map(this::createTagKey).collect(Collectors.toSet());
|
||||
}
|
||||
|
||||
private Map<String, List<TagLabel>> batchFetchTags(List<String> entityFQNs) {
|
||||
protected Map<String, List<TagLabel>> batchFetchTags(List<String> entityFQNs) {
|
||||
if (entityFQNs == null || entityFQNs.isEmpty()) {
|
||||
return Collections.emptyMap();
|
||||
}
|
||||
|
||||
@ -28,6 +28,7 @@ import static org.openmetadata.service.util.EntityUtil.taskMatch;
|
||||
|
||||
import jakarta.ws.rs.core.Response;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
@ -203,13 +204,27 @@ public class PipelineRepository extends EntityRepository<Pipeline> {
|
||||
return;
|
||||
}
|
||||
|
||||
// Use bulk tag and owner fetching for all pipeline tasks
|
||||
for (Pipeline pipeline : pipelines) {
|
||||
if (pipeline.getTasks() != null) {
|
||||
// Still need individual calls here as tasks don't have bulk fetching pattern
|
||||
// This is better than the original N+N pattern we had
|
||||
getTaskTags(fields.contains(FIELD_TAGS), pipeline.getTasks());
|
||||
getTaskOwners(fields.contains(FIELD_OWNERS), pipeline.getTasks());
|
||||
// First, if tags are requested, fetch pipeline-level tags (important for search indexing)
|
||||
if (fields.contains(FIELD_TAGS)) {
|
||||
List<String> entityFQNs = pipelines.stream().map(Pipeline::getFullyQualifiedName).toList();
|
||||
Map<String, List<TagLabel>> tagsMap = batchFetchTags(entityFQNs);
|
||||
for (Pipeline pipeline : pipelines) {
|
||||
pipeline.setTags(
|
||||
addDerivedTags(
|
||||
tagsMap.getOrDefault(pipeline.getFullyQualifiedName(), Collections.emptyList())));
|
||||
}
|
||||
}
|
||||
|
||||
// Then, if tasks field is requested, also handle task-level tags and owners
|
||||
if (fields.contains("tasks")) {
|
||||
// Use bulk tag and owner fetching for all pipeline tasks
|
||||
for (Pipeline pipeline : pipelines) {
|
||||
if (pipeline.getTasks() != null) {
|
||||
// Still need individual calls here as tasks don't have bulk fetching pattern
|
||||
// This is better than the original N+N pattern we had
|
||||
getTaskTags(fields.contains(FIELD_TAGS), pipeline.getTasks());
|
||||
getTaskOwners(fields.contains(FIELD_OWNERS), pipeline.getTasks());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -26,6 +26,7 @@ import static org.openmetadata.service.resources.tags.TagLabelUtil.checkMutually
|
||||
import static org.openmetadata.service.util.EntityUtil.getSearchIndexField;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
@ -204,9 +205,23 @@ public class SearchIndexRepository extends EntityRepository<SearchIndex> {
|
||||
if (!fields.contains(FIELD_TAGS) || searchIndexes == null || searchIndexes.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
// Use bulk tag fetching to avoid N+1 queries
|
||||
bulkPopulateEntityFieldTags(
|
||||
searchIndexes, entityType, SearchIndex::getFields, SearchIndex::getFullyQualifiedName);
|
||||
|
||||
// First, fetch searchIndex-level tags (important for search indexing)
|
||||
List<String> entityFQNs =
|
||||
searchIndexes.stream().map(SearchIndex::getFullyQualifiedName).toList();
|
||||
Map<String, List<TagLabel>> tagsMap = batchFetchTags(entityFQNs);
|
||||
for (SearchIndex searchIndex : searchIndexes) {
|
||||
searchIndex.setTags(
|
||||
addDerivedTags(
|
||||
tagsMap.getOrDefault(searchIndex.getFullyQualifiedName(), Collections.emptyList())));
|
||||
}
|
||||
|
||||
// Then, if fields are requested, also fetch field-level tags
|
||||
if (fields.contains("fields")) {
|
||||
// Use bulk tag fetching to avoid N+1 queries
|
||||
bulkPopulateEntityFieldTags(
|
||||
searchIndexes, entityType, SearchIndex::getFields, SearchIndex::getFullyQualifiedName);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
|
||||
@ -32,6 +32,7 @@ import static org.openmetadata.service.Entity.TEST_SUITE;
|
||||
import static org.openmetadata.service.Entity.getEntities;
|
||||
import static org.openmetadata.service.Entity.getEntityReferenceById;
|
||||
import static org.openmetadata.service.Entity.populateEntityFieldTags;
|
||||
import static org.openmetadata.service.resources.tags.TagLabelUtil.addDerivedTags;
|
||||
import static org.openmetadata.service.search.SearchClient.GLOBAL_SEARCH_ALIAS;
|
||||
import static org.openmetadata.service.util.EntityUtil.getLocalColumnName;
|
||||
import static org.openmetadata.service.util.FullyQualifiedName.getColumnName;
|
||||
@ -280,15 +281,21 @@ public class TableRepository extends EntityRepository<Table> {
|
||||
}
|
||||
|
||||
private void fetchAndSetColumnTags(List<Table> tables, Fields fields) {
|
||||
if (!fields.contains(FIELD_TAGS)
|
||||
|| !fields.contains(COLUMN_FIELD)
|
||||
|| tables == null
|
||||
|| tables.isEmpty()) {
|
||||
if (!fields.contains(FIELD_TAGS) || tables == null || tables.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
// Use bulk tag fetching to avoid N+1 queries
|
||||
bulkPopulateEntityFieldTags(
|
||||
tables, entityType, Table::getColumns, Table::getFullyQualifiedName);
|
||||
List<String> entityFQNs = tables.stream().map(Table::getFullyQualifiedName).toList();
|
||||
Map<String, List<TagLabel>> tagsMap = batchFetchTags(entityFQNs);
|
||||
for (Table table : tables) {
|
||||
table.setTags(
|
||||
addDerivedTags(
|
||||
tagsMap.getOrDefault(table.getFullyQualifiedName(), Collections.emptyList())));
|
||||
}
|
||||
|
||||
if (fields.contains(COLUMN_FIELD)) {
|
||||
bulkPopulateEntityFieldTags(
|
||||
tables, entityType, Table::getColumns, Table::getFullyQualifiedName);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
|
||||
@ -25,6 +25,7 @@ import static org.openmetadata.service.resources.tags.TagLabelUtil.addDerivedTag
|
||||
import static org.openmetadata.service.resources.tags.TagLabelUtil.checkMutuallyExclusive;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
@ -174,16 +175,28 @@ public class TopicRepository extends EntityRepository<Topic> {
|
||||
return;
|
||||
}
|
||||
|
||||
// Filter topics that have message schemas and use bulk tag fetching
|
||||
List<Topic> topicsWithSchemas =
|
||||
topics.stream().filter(t -> t.getMessageSchema() != null).toList();
|
||||
// First, fetch topic-level tags (important for search indexing)
|
||||
List<String> entityFQNs = topics.stream().map(Topic::getFullyQualifiedName).toList();
|
||||
Map<String, List<TagLabel>> tagsMap = batchFetchTags(entityFQNs);
|
||||
for (Topic topic : topics) {
|
||||
topic.setTags(
|
||||
addDerivedTags(
|
||||
tagsMap.getOrDefault(topic.getFullyQualifiedName(), Collections.emptyList())));
|
||||
}
|
||||
|
||||
if (!topicsWithSchemas.isEmpty()) {
|
||||
bulkPopulateEntityFieldTags(
|
||||
topicsWithSchemas,
|
||||
entityType,
|
||||
t -> t.getMessageSchema().getSchemaFields(),
|
||||
Topic::getFullyQualifiedName);
|
||||
// Then, if messageSchema field is requested, also fetch schema field tags
|
||||
if (fields.contains("messageSchema")) {
|
||||
// Filter topics that have message schemas and use bulk tag fetching
|
||||
List<Topic> topicsWithSchemas =
|
||||
topics.stream().filter(t -> t.getMessageSchema() != null).toList();
|
||||
|
||||
if (!topicsWithSchemas.isEmpty()) {
|
||||
bulkPopulateEntityFieldTags(
|
||||
topicsWithSchemas,
|
||||
entityType,
|
||||
t -> t.getMessageSchema().getSchemaFields(),
|
||||
Topic::getFullyQualifiedName);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -2,6 +2,7 @@ package org.openmetadata.service.resources.apis;
|
||||
|
||||
import static jakarta.ws.rs.core.Response.Status.BAD_REQUEST;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.openmetadata.service.Entity.FIELD_OWNERS;
|
||||
@ -27,6 +28,7 @@ import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.UUID;
|
||||
import java.util.stream.Collectors;
|
||||
import org.apache.http.client.HttpResponseException;
|
||||
import org.junit.jupiter.api.MethodOrderer;
|
||||
import org.junit.jupiter.api.Order;
|
||||
@ -382,4 +384,166 @@ public class APIEndpointResourceTest extends EntityResourceTest<APIEndpoint, Cre
|
||||
// Check the nested columns
|
||||
assertFields(expectedField.getChildren(), actualField.getChildren());
|
||||
}
|
||||
|
||||
@Test
|
||||
@Order(2)
|
||||
void test_paginationFetchesTagsAtBothEntityAndFieldLevels(TestInfo test) throws IOException {
|
||||
TagLabel endpointTagLabel = USER_ADDRESS_TAG_LABEL;
|
||||
TagLabel fieldTagLabel = PERSONAL_DATA_TAG_LABEL;
|
||||
|
||||
List<APIEndpoint> createdEndpoints = new ArrayList<>();
|
||||
for (int i = 0; i < 5; i++) {
|
||||
List<Field> requestFields =
|
||||
Arrays.asList(
|
||||
getField("requestField1_" + i, FieldDataType.STRING, fieldTagLabel),
|
||||
getField("requestField2_" + i, FieldDataType.STRING, null));
|
||||
|
||||
List<Field> responseFields =
|
||||
Arrays.asList(
|
||||
getField("responseField1_" + i, FieldDataType.STRING, PII_SENSITIVE_TAG_LABEL),
|
||||
getField("responseField2_" + i, FieldDataType.STRING, null));
|
||||
|
||||
APISchema requestSchema = new APISchema().withSchemaFields(requestFields);
|
||||
APISchema responseSchema = new APISchema().withSchemaFields(responseFields);
|
||||
|
||||
CreateAPIEndpoint createEndpoint =
|
||||
createRequest(test.getDisplayName() + "_pagination_" + i)
|
||||
.withRequestSchema(requestSchema)
|
||||
.withResponseSchema(responseSchema)
|
||||
.withTags(List.of(endpointTagLabel));
|
||||
|
||||
APIEndpoint endpoint = createEntity(createEndpoint, ADMIN_AUTH_HEADERS);
|
||||
createdEndpoints.add(endpoint);
|
||||
}
|
||||
|
||||
WebTarget target =
|
||||
getResource("apiEndpoints").queryParam("fields", "tags").queryParam("limit", "50");
|
||||
|
||||
APIEndpointResource.APIEndpointList endpointList =
|
||||
TestUtils.get(target, APIEndpointResource.APIEndpointList.class, ADMIN_AUTH_HEADERS);
|
||||
assertNotNull(endpointList.getData());
|
||||
|
||||
List<APIEndpoint> ourEndpoints =
|
||||
endpointList.getData().stream()
|
||||
.filter(e -> createdEndpoints.stream().anyMatch(ce -> ce.getId().equals(e.getId())))
|
||||
.collect(Collectors.toList());
|
||||
|
||||
assertFalse(
|
||||
ourEndpoints.isEmpty(), "Should find at least one of our created endpoints in pagination");
|
||||
|
||||
for (APIEndpoint endpoint : ourEndpoints) {
|
||||
assertNotNull(
|
||||
endpoint.getTags(),
|
||||
"Endpoint-level tags should not be null when fields=tags in pagination");
|
||||
assertEquals(1, endpoint.getTags().size(), "Should have exactly one endpoint-level tag");
|
||||
assertEquals(endpointTagLabel.getTagFQN(), endpoint.getTags().get(0).getTagFQN());
|
||||
|
||||
if (endpoint.getRequestSchema() != null
|
||||
&& endpoint.getRequestSchema().getSchemaFields() != null) {
|
||||
for (Field field : endpoint.getRequestSchema().getSchemaFields()) {
|
||||
assertTrue(
|
||||
field.getTags() == null || field.getTags().isEmpty(),
|
||||
"Request field tags should not be populated when only fields=tags is specified in pagination");
|
||||
}
|
||||
}
|
||||
if (endpoint.getResponseSchema() != null
|
||||
&& endpoint.getResponseSchema().getSchemaFields() != null) {
|
||||
for (Field field : endpoint.getResponseSchema().getSchemaFields()) {
|
||||
assertTrue(
|
||||
field.getTags() == null || field.getTags().isEmpty(),
|
||||
"Response field tags should not be populated when only fields=tags is specified in pagination");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
target =
|
||||
getResource("apiEndpoints")
|
||||
.queryParam("fields", "requestSchema,responseSchema,tags")
|
||||
.queryParam("limit", "10");
|
||||
|
||||
endpointList =
|
||||
TestUtils.get(target, APIEndpointResource.APIEndpointList.class, ADMIN_AUTH_HEADERS);
|
||||
assertNotNull(endpointList.getData());
|
||||
|
||||
ourEndpoints =
|
||||
endpointList.getData().stream()
|
||||
.filter(e -> createdEndpoints.stream().anyMatch(ce -> ce.getId().equals(e.getId())))
|
||||
.collect(Collectors.toList());
|
||||
|
||||
assertFalse(
|
||||
ourEndpoints.isEmpty(), "Should find at least one of our created endpoints in pagination");
|
||||
|
||||
// Verify both endpoint-level and field-level tags are fetched
|
||||
for (APIEndpoint endpoint : ourEndpoints) {
|
||||
// Verify endpoint-level tags
|
||||
assertNotNull(
|
||||
endpoint.getTags(),
|
||||
"Endpoint-level tags should not be null in pagination with schemas,tags");
|
||||
assertEquals(1, endpoint.getTags().size(), "Should have exactly one endpoint-level tag");
|
||||
assertEquals(endpointTagLabel.getTagFQN(), endpoint.getTags().get(0).getTagFQN());
|
||||
|
||||
// Verify request field-level tags
|
||||
assertNotNull(
|
||||
endpoint.getRequestSchema(),
|
||||
"RequestSchema should not be null when fields includes requestSchema");
|
||||
assertNotNull(
|
||||
endpoint.getRequestSchema().getSchemaFields(),
|
||||
"Request schema fields should not be null");
|
||||
|
||||
Field requestField1 =
|
||||
endpoint.getRequestSchema().getSchemaFields().stream()
|
||||
.filter(f -> f.getName().startsWith("requestField1_"))
|
||||
.findFirst()
|
||||
.orElseThrow(() -> new AssertionError("Should find requestField1 field"));
|
||||
|
||||
assertNotNull(
|
||||
requestField1.getTags(),
|
||||
"Request field tags should not be null when fields=requestSchema,responseSchema,tags in pagination");
|
||||
assertEquals(1, requestField1.getTags().size(), "Request field should have exactly one tag");
|
||||
assertEquals(fieldTagLabel.getTagFQN(), requestField1.getTags().get(0).getTagFQN());
|
||||
|
||||
// Verify response field-level tags
|
||||
assertNotNull(
|
||||
endpoint.getResponseSchema(),
|
||||
"ResponseSchema should not be null when fields includes responseSchema");
|
||||
assertNotNull(
|
||||
endpoint.getResponseSchema().getSchemaFields(),
|
||||
"Response schema fields should not be null");
|
||||
|
||||
Field responseField1 =
|
||||
endpoint.getResponseSchema().getSchemaFields().stream()
|
||||
.filter(f -> f.getName().startsWith("responseField1_"))
|
||||
.findFirst()
|
||||
.orElseThrow(() -> new AssertionError("Should find responseField1 field"));
|
||||
|
||||
assertNotNull(
|
||||
responseField1.getTags(),
|
||||
"Response field tags should not be null when fields=requestSchema,responseSchema,tags in pagination");
|
||||
assertEquals(
|
||||
1, responseField1.getTags().size(), "Response field should have exactly one tag");
|
||||
assertEquals(
|
||||
PII_SENSITIVE_TAG_LABEL.getTagFQN(), responseField1.getTags().get(0).getTagFQN());
|
||||
|
||||
// Fields without tags should remain empty
|
||||
Field requestField2 =
|
||||
endpoint.getRequestSchema().getSchemaFields().stream()
|
||||
.filter(f -> f.getName().startsWith("requestField2_"))
|
||||
.findFirst()
|
||||
.orElseThrow(() -> new AssertionError("Should find requestField2 field"));
|
||||
|
||||
assertTrue(
|
||||
requestField2.getTags() == null || requestField2.getTags().isEmpty(),
|
||||
"requestField2 should not have tags");
|
||||
|
||||
Field responseField2 =
|
||||
endpoint.getResponseSchema().getSchemaFields().stream()
|
||||
.filter(f -> f.getName().startsWith("responseField2_"))
|
||||
.findFirst()
|
||||
.orElseThrow(() -> new AssertionError("Should find responseField2 field"));
|
||||
|
||||
assertTrue(
|
||||
responseField2.getTags() == null || responseField2.getTags().isEmpty(),
|
||||
"responseField2 should not have tags");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -5752,6 +5752,116 @@ public class TableResourceTest extends EntityResourceTest<Table, CreateTable> {
|
||||
&& e.getRelatedEntity().getId().equals(tableC.getId())));
|
||||
}
|
||||
|
||||
@Test
|
||||
@Order(2)
|
||||
void test_paginationFetchesTagsAtBothEntityAndFieldLevels(TestInfo test) throws IOException {
|
||||
TagLabel tableTagLabel = USER_ADDRESS_TAG_LABEL;
|
||||
TagLabel columnTagLabel = GLOSSARY1_TERM1_LABEL;
|
||||
|
||||
List<Table> createdTables = new ArrayList<>();
|
||||
for (int i = 0; i < 5; i++) {
|
||||
List<Column> columns =
|
||||
Arrays.asList(
|
||||
getColumn("col1_" + i, BIGINT, null).withTags(List.of(columnTagLabel)),
|
||||
getColumn("col2_" + i, VARCHAR, null).withDataLength(50));
|
||||
|
||||
CreateTable createTable =
|
||||
createRequest(test.getDisplayName() + "_pagination_" + i)
|
||||
.withColumns(columns)
|
||||
.withTags(List.of(tableTagLabel))
|
||||
.withTableConstraints(null);
|
||||
|
||||
Table table = createEntity(createTable, ADMIN_AUTH_HEADERS);
|
||||
createdTables.add(table);
|
||||
}
|
||||
|
||||
// Test pagination with fields=tags (should fetch table-level tags only)
|
||||
WebTarget target =
|
||||
getResource("tables")
|
||||
.queryParam("fields", "tags")
|
||||
.queryParam("limit", "50")
|
||||
.queryParam(
|
||||
"databaseSchema",
|
||||
DATABASE_SCHEMA.getFullyQualifiedName()); // Filter by schema to get our tables
|
||||
|
||||
TableList tableList = TestUtils.get(target, TableList.class, ADMIN_AUTH_HEADERS);
|
||||
assertNotNull(tableList.getData());
|
||||
|
||||
List<Table> ourTables =
|
||||
tableList.getData().stream()
|
||||
.filter(t -> createdTables.stream().anyMatch(ct -> ct.getId().equals(t.getId())))
|
||||
.collect(Collectors.toList());
|
||||
|
||||
assertFalse(
|
||||
ourTables.isEmpty(), "Should find at least one of our created tables in pagination");
|
||||
|
||||
for (Table table : ourTables) {
|
||||
assertNotNull(
|
||||
table.getTags(), "Table-level tags should not be null when fields=tags in pagination");
|
||||
assertEquals(1, table.getTags().size(), "Should have exactly one table-level tag");
|
||||
assertEquals(tableTagLabel.getTagFQN(), table.getTags().get(0).getTagFQN());
|
||||
|
||||
if (table.getColumns() != null) {
|
||||
for (Column col : table.getColumns()) {
|
||||
assertTrue(
|
||||
col.getTags() == null || col.getTags().isEmpty(),
|
||||
"Column tags should not be populated when only fields=tags is specified in pagination");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
target =
|
||||
getResource("tables")
|
||||
.queryParam("fields", "columns,tags")
|
||||
.queryParam("limit", "50")
|
||||
.queryParam(
|
||||
"databaseSchema",
|
||||
DATABASE_SCHEMA.getFullyQualifiedName()); // Filter by schema to get our tables
|
||||
|
||||
tableList = TestUtils.get(target, TableList.class, ADMIN_AUTH_HEADERS);
|
||||
assertNotNull(tableList.getData());
|
||||
|
||||
ourTables =
|
||||
tableList.getData().stream()
|
||||
.filter(t -> createdTables.stream().anyMatch(ct -> ct.getId().equals(t.getId())))
|
||||
.collect(Collectors.toList());
|
||||
|
||||
assertFalse(
|
||||
ourTables.isEmpty(), "Should find at least one of our created tables in pagination");
|
||||
|
||||
for (Table table : ourTables) {
|
||||
assertNotNull(
|
||||
table.getTags(), "Table-level tags should not be null in pagination with columns,tags");
|
||||
assertEquals(1, table.getTags().size(), "Should have exactly one table-level tag");
|
||||
assertEquals(tableTagLabel.getTagFQN(), table.getTags().get(0).getTagFQN());
|
||||
|
||||
assertNotNull(table.getColumns(), "Columns should not be null when fields includes columns");
|
||||
Column col1 =
|
||||
table.getColumns().stream()
|
||||
.filter(c -> c.getName().startsWith("col1_"))
|
||||
.findFirst()
|
||||
.orElseThrow(() -> new AssertionError("Should find col1 column"));
|
||||
|
||||
assertNotNull(
|
||||
col1.getTags(), "Column tags should not be null when fields=columns,tags in pagination");
|
||||
assertTrue(!col1.getTags().isEmpty(), "Column should have at least one tag");
|
||||
// Check that our expected tag is present
|
||||
boolean hasExpectedTag =
|
||||
col1.getTags().stream()
|
||||
.anyMatch(tag -> tag.getTagFQN().equals(columnTagLabel.getTagFQN()));
|
||||
assertTrue(
|
||||
hasExpectedTag, "Column should have the expected tag: " + columnTagLabel.getTagFQN());
|
||||
|
||||
Column col2 =
|
||||
table.getColumns().stream()
|
||||
.filter(c -> c.getName().startsWith("col2_"))
|
||||
.findFirst()
|
||||
.orElseThrow(() -> new AssertionError("Should find col2 column"));
|
||||
|
||||
assertTrue(col2.getTags() == null || col2.getTags().isEmpty(), "col2 should not have tags");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_compositeKeyConstraintIndexOutOfBounds_fixed(TestInfo test) throws IOException {
|
||||
// Create a schema for this test to avoid conflicts
|
||||
|
||||
@ -16,6 +16,7 @@ package org.openmetadata.service.resources.datamodels;
|
||||
import static jakarta.ws.rs.core.Response.Status.BAD_REQUEST;
|
||||
import static jakarta.ws.rs.core.Response.Status.NOT_FOUND;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.openmetadata.common.utils.CommonUtil.listOf;
|
||||
@ -31,15 +32,21 @@ import static org.openmetadata.service.util.TestUtils.assertListNotNull;
|
||||
import static org.openmetadata.service.util.TestUtils.assertListNull;
|
||||
import static org.openmetadata.service.util.TestUtils.assertResponse;
|
||||
|
||||
import jakarta.ws.rs.client.WebTarget;
|
||||
import jakarta.ws.rs.core.Response.Status;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.http.client.HttpResponseException;
|
||||
import org.junit.jupiter.api.MethodOrderer;
|
||||
import org.junit.jupiter.api.Order;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.TestInfo;
|
||||
import org.junit.jupiter.api.TestMethodOrder;
|
||||
import org.junit.jupiter.api.parallel.Execution;
|
||||
import org.junit.jupiter.api.parallel.ExecutionMode;
|
||||
import org.openmetadata.schema.api.data.CreateDashboardDataModel;
|
||||
@ -56,8 +63,10 @@ import org.openmetadata.service.exception.CatalogExceptionMessage;
|
||||
import org.openmetadata.service.resources.EntityResourceTest;
|
||||
import org.openmetadata.service.resources.services.DashboardServiceResourceTest;
|
||||
import org.openmetadata.service.util.ResultList;
|
||||
import org.openmetadata.service.util.TestUtils;
|
||||
|
||||
@Slf4j
|
||||
@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
|
||||
public class DashboardDataModelResourceTest
|
||||
extends EntityResourceTest<DashboardDataModel, CreateDashboardDataModel> {
|
||||
|
||||
@ -289,4 +298,130 @@ public class DashboardDataModelResourceTest
|
||||
8, mixedFieldsDataModel.getColumns().size(), "Should return all columns in mixed request");
|
||||
assertNotNull(mixedFieldsDataModel.getOwners(), "Should also return other requested fields");
|
||||
}
|
||||
|
||||
@Test
|
||||
@Order(1)
|
||||
void test_paginationFetchesTagsAtBothEntityAndFieldLevels(TestInfo test) throws IOException {
|
||||
TagLabel dataModelTagLabel = USER_ADDRESS_TAG_LABEL;
|
||||
TagLabel columnTagLabel = PERSONAL_DATA_TAG_LABEL;
|
||||
|
||||
List<DashboardDataModel> createdDataModels = new ArrayList<>();
|
||||
for (int i = 0; i < 5; i++) {
|
||||
List<Column> columns =
|
||||
Arrays.asList(
|
||||
getColumn("column1_" + i, BIGINT, columnTagLabel),
|
||||
getColumn("column2_" + i, BIGINT, null),
|
||||
getColumn("column3_" + i, INT, null));
|
||||
|
||||
CreateDashboardDataModel createDataModel =
|
||||
createRequest(test.getDisplayName() + "_pagination_" + i)
|
||||
.withColumns(columns)
|
||||
.withTags(List.of(dataModelTagLabel));
|
||||
|
||||
DashboardDataModel dataModel = createEntity(createDataModel, ADMIN_AUTH_HEADERS);
|
||||
createdDataModels.add(dataModel);
|
||||
}
|
||||
|
||||
// Test pagination with fields=tags (should fetch data model-level tags only)
|
||||
WebTarget target =
|
||||
getResource("dashboard/datamodels").queryParam("fields", "tags").queryParam("limit", "10");
|
||||
|
||||
DashboardDataModelResource.DashboardDataModelList dataModelList =
|
||||
TestUtils.get(
|
||||
target, DashboardDataModelResource.DashboardDataModelList.class, ADMIN_AUTH_HEADERS);
|
||||
assertNotNull(dataModelList.getData());
|
||||
|
||||
// Verify at least one of our created data models is in the response
|
||||
List<DashboardDataModel> ourDataModels =
|
||||
dataModelList.getData().stream()
|
||||
.filter(
|
||||
dm -> createdDataModels.stream().anyMatch(cdm -> cdm.getId().equals(dm.getId())))
|
||||
.collect(java.util.stream.Collectors.toList());
|
||||
|
||||
assertFalse(
|
||||
ourDataModels.isEmpty(),
|
||||
"Should find at least one of our created data models in pagination");
|
||||
|
||||
// Verify data model-level tags are fetched
|
||||
for (DashboardDataModel dataModel : ourDataModels) {
|
||||
assertNotNull(
|
||||
dataModel.getTags(),
|
||||
"Data model-level tags should not be null when fields=tags in pagination");
|
||||
assertEquals(1, dataModel.getTags().size(), "Should have exactly one data model-level tag");
|
||||
assertEquals(dataModelTagLabel.getTagFQN(), dataModel.getTags().get(0).getTagFQN());
|
||||
|
||||
// DashboardDataModel returns columns by default even when not explicitly requested
|
||||
// The columns retain their tags from creation. This is different from Table behavior
|
||||
// but is the expected behavior for DashboardDataModel.
|
||||
// The important part is that the entity-level tags are properly fetched.
|
||||
}
|
||||
|
||||
// Test pagination with fields=columns,tags (should fetch both data model and column tags)
|
||||
target =
|
||||
getResource("dashboard/datamodels")
|
||||
.queryParam("fields", "columns,tags")
|
||||
.queryParam("limit", "10");
|
||||
|
||||
dataModelList =
|
||||
TestUtils.get(
|
||||
target, DashboardDataModelResource.DashboardDataModelList.class, ADMIN_AUTH_HEADERS);
|
||||
assertNotNull(dataModelList.getData());
|
||||
|
||||
// Verify at least one of our created data models is in the response
|
||||
ourDataModels =
|
||||
dataModelList.getData().stream()
|
||||
.filter(
|
||||
dm -> createdDataModels.stream().anyMatch(cdm -> cdm.getId().equals(dm.getId())))
|
||||
.collect(java.util.stream.Collectors.toList());
|
||||
|
||||
assertFalse(
|
||||
ourDataModels.isEmpty(),
|
||||
"Should find at least one of our created data models in pagination");
|
||||
|
||||
// Verify both data model-level and column-level tags are fetched
|
||||
for (DashboardDataModel dataModel : ourDataModels) {
|
||||
// Verify data model-level tags
|
||||
assertNotNull(
|
||||
dataModel.getTags(),
|
||||
"Data model-level tags should not be null in pagination with columns,tags");
|
||||
assertEquals(1, dataModel.getTags().size(), "Should have exactly one data model-level tag");
|
||||
assertEquals(dataModelTagLabel.getTagFQN(), dataModel.getTags().get(0).getTagFQN());
|
||||
|
||||
// Verify column-level tags
|
||||
assertNotNull(
|
||||
dataModel.getColumns(), "Columns should not be null when fields includes columns");
|
||||
assertFalse(dataModel.getColumns().isEmpty(), "Columns should not be empty");
|
||||
|
||||
Column column1 =
|
||||
dataModel.getColumns().stream()
|
||||
.filter(c -> c.getName().startsWith("column1_"))
|
||||
.findFirst()
|
||||
.orElseThrow(() -> new AssertionError("Should find column1 column"));
|
||||
|
||||
assertNotNull(
|
||||
column1.getTags(),
|
||||
"Column tags should not be null when fields=columns,tags in pagination");
|
||||
assertEquals(1, column1.getTags().size(), "Column should have exactly one tag");
|
||||
assertEquals(columnTagLabel.getTagFQN(), column1.getTags().get(0).getTagFQN());
|
||||
|
||||
// column2 and column3 should not have tags
|
||||
Column column2 =
|
||||
dataModel.getColumns().stream()
|
||||
.filter(c -> c.getName().startsWith("column2_"))
|
||||
.findFirst()
|
||||
.orElseThrow(() -> new AssertionError("Should find column2 column"));
|
||||
|
||||
assertTrue(
|
||||
column2.getTags() == null || column2.getTags().isEmpty(), "column2 should not have tags");
|
||||
|
||||
Column column3 =
|
||||
dataModel.getColumns().stream()
|
||||
.filter(c -> c.getName().startsWith("column3_"))
|
||||
.findFirst()
|
||||
.orElseThrow(() -> new AssertionError("Should find column3 column"));
|
||||
|
||||
assertTrue(
|
||||
column3.getTags() == null || column3.getTags().isEmpty(), "column3 should not have tags");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -17,6 +17,7 @@ import static jakarta.ws.rs.core.Response.Status.BAD_REQUEST;
|
||||
import static jakarta.ws.rs.core.Response.Status.NOT_FOUND;
|
||||
import static jakarta.ws.rs.core.Response.Status.OK;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.openmetadata.common.utils.CommonUtil.listOf;
|
||||
@ -53,8 +54,11 @@ import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.http.client.HttpResponseException;
|
||||
import org.joda.time.DateTime;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.MethodOrderer;
|
||||
import org.junit.jupiter.api.Order;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.TestInfo;
|
||||
import org.junit.jupiter.api.TestMethodOrder;
|
||||
import org.openmetadata.schema.api.data.CreatePipeline;
|
||||
import org.openmetadata.schema.api.services.CreatePipelineService;
|
||||
import org.openmetadata.schema.entity.data.Pipeline;
|
||||
@ -78,6 +82,7 @@ import org.openmetadata.service.util.ResultList;
|
||||
import org.openmetadata.service.util.TestUtils;
|
||||
|
||||
@Slf4j
|
||||
@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
|
||||
public class PipelineResourceTest extends EntityResourceTest<Pipeline, CreatePipeline> {
|
||||
public static List<Task> TASKS;
|
||||
|
||||
@ -839,4 +844,124 @@ public class PipelineResourceTest extends EntityResourceTest<Pipeline, CreatePip
|
||||
private void verifyPipelineStatus(PipelineStatus actualStatus, PipelineStatus expectedStatus) {
|
||||
assertEquals(actualStatus, expectedStatus);
|
||||
}
|
||||
|
||||
@Order(1)
|
||||
@Test
|
||||
void test_paginationFetchesTagsAtBothEntityAndFieldLevels(TestInfo test) throws IOException {
|
||||
// Use existing tags that are already set up in the test environment
|
||||
TagLabel pipelineTagLabel = USER_ADDRESS_TAG_LABEL;
|
||||
TagLabel taskTagLabel = PERSONAL_DATA_TAG_LABEL;
|
||||
|
||||
// Create multiple pipelines with tags at both pipeline and task levels
|
||||
List<Pipeline> createdPipelines = new ArrayList<>();
|
||||
for (int i = 0; i < 5; i++) {
|
||||
List<Task> tasks = new ArrayList<>();
|
||||
for (int j = 0; j < 3; j++) {
|
||||
Task task =
|
||||
new Task()
|
||||
.withName("task" + j + "_" + i)
|
||||
.withDescription("description")
|
||||
.withDisplayName("displayName")
|
||||
.withSourceUrl("http://localhost:0");
|
||||
|
||||
if (j == 0) {
|
||||
// Add tag to first task only
|
||||
task.withTags(List.of(taskTagLabel));
|
||||
}
|
||||
tasks.add(task);
|
||||
}
|
||||
|
||||
CreatePipeline createPipeline =
|
||||
createRequest(test.getDisplayName() + "_pagination_" + i)
|
||||
.withTasks(tasks)
|
||||
.withTags(List.of(pipelineTagLabel));
|
||||
|
||||
Pipeline pipeline = createEntity(createPipeline, ADMIN_AUTH_HEADERS);
|
||||
createdPipelines.add(pipeline);
|
||||
}
|
||||
|
||||
// Test pagination with fields=tags (should fetch pipeline-level tags only)
|
||||
WebTarget target =
|
||||
getResource("pipelines").queryParam("fields", "tags").queryParam("limit", "10");
|
||||
|
||||
PipelineList pipelineList = TestUtils.get(target, PipelineList.class, ADMIN_AUTH_HEADERS);
|
||||
assertNotNull(pipelineList.getData());
|
||||
|
||||
// Verify at least one of our created pipelines is in the response
|
||||
List<Pipeline> ourPipelines =
|
||||
pipelineList.getData().stream()
|
||||
.filter(p -> createdPipelines.stream().anyMatch(cp -> cp.getId().equals(p.getId())))
|
||||
.collect(Collectors.toList());
|
||||
|
||||
assertFalse(
|
||||
ourPipelines.isEmpty(), "Should find at least one of our created pipelines in pagination");
|
||||
|
||||
// Verify pipeline-level tags are fetched
|
||||
for (Pipeline pipeline : ourPipelines) {
|
||||
assertNotNull(
|
||||
pipeline.getTags(),
|
||||
"Pipeline-level tags should not be null when fields=tags in pagination");
|
||||
assertEquals(1, pipeline.getTags().size(), "Should have exactly one pipeline-level tag");
|
||||
assertEquals(pipelineTagLabel.getTagFQN(), pipeline.getTags().get(0).getTagFQN());
|
||||
|
||||
// Tasks should not have tags when only fields=tags is specified
|
||||
if (pipeline.getTasks() != null) {
|
||||
for (Task task : pipeline.getTasks()) {
|
||||
assertTrue(
|
||||
task.getTags() == null || task.getTags().isEmpty(),
|
||||
"Task tags should not be populated when only fields=tags is specified in pagination");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Test pagination with fields=tasks,tags (should fetch both pipeline and task tags)
|
||||
target = getResource("pipelines").queryParam("fields", "tasks,tags").queryParam("limit", "10");
|
||||
|
||||
pipelineList = TestUtils.get(target, PipelineList.class, ADMIN_AUTH_HEADERS);
|
||||
assertNotNull(pipelineList.getData());
|
||||
|
||||
// Verify at least one of our created pipelines is in the response
|
||||
ourPipelines =
|
||||
pipelineList.getData().stream()
|
||||
.filter(p -> createdPipelines.stream().anyMatch(cp -> cp.getId().equals(p.getId())))
|
||||
.collect(Collectors.toList());
|
||||
|
||||
assertFalse(
|
||||
ourPipelines.isEmpty(), "Should find at least one of our created pipelines in pagination");
|
||||
|
||||
// Verify both pipeline-level and task-level tags are fetched
|
||||
for (Pipeline pipeline : ourPipelines) {
|
||||
// Verify pipeline-level tags
|
||||
assertNotNull(
|
||||
pipeline.getTags(),
|
||||
"Pipeline-level tags should not be null in pagination with tasks,tags");
|
||||
assertEquals(1, pipeline.getTags().size(), "Should have exactly one pipeline-level tag");
|
||||
assertEquals(pipelineTagLabel.getTagFQN(), pipeline.getTags().get(0).getTagFQN());
|
||||
|
||||
// Verify task-level tags
|
||||
assertNotNull(pipeline.getTasks(), "Tasks should not be null when fields includes tasks");
|
||||
assertFalse(pipeline.getTasks().isEmpty(), "Tasks should not be empty");
|
||||
|
||||
// Find the first task which should have a tag
|
||||
Task task0 =
|
||||
pipeline.getTasks().stream()
|
||||
.filter(t -> t.getName().startsWith("task0_"))
|
||||
.findFirst()
|
||||
.orElseThrow(() -> new AssertionError("Should find task0 task"));
|
||||
|
||||
assertNotNull(
|
||||
task0.getTags(), "Task tags should not be null when fields=tasks,tags in pagination");
|
||||
assertEquals(1, task0.getTags().size(), "Task should have exactly one tag");
|
||||
assertEquals(taskTagLabel.getTagFQN(), task0.getTags().get(0).getTagFQN());
|
||||
|
||||
// Other tasks should not have tags
|
||||
for (Task task : pipeline.getTasks()) {
|
||||
if (!task.getName().startsWith("task0_")) {
|
||||
assertTrue(
|
||||
task.getTags() == null || task.getTags().isEmpty(),
|
||||
"Other tasks should not have tags");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -19,6 +19,7 @@ import static jakarta.ws.rs.core.Response.Status.OK;
|
||||
import static java.util.Collections.singletonList;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.openmetadata.common.utils.CommonUtil.listOf;
|
||||
@ -50,6 +51,7 @@ import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.UUID;
|
||||
import java.util.stream.Collectors;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.http.client.HttpResponseException;
|
||||
import org.junit.jupiter.api.Test;
|
||||
@ -949,4 +951,118 @@ public class SearchIndexResourceTest extends EntityResourceTest<SearchIndex, Cre
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_paginationFetchesTagsAtBothEntityAndFieldLevels(TestInfo test) throws IOException {
|
||||
TagLabel searchIndexTagLabel = USER_ADDRESS_TAG_LABEL;
|
||||
TagLabel fieldTagLabel = GLOSSARY1_TERM1_LABEL;
|
||||
|
||||
List<SearchIndex> createdSearchIndexes = new ArrayList<>();
|
||||
for (int i = 0; i < 5; i++) {
|
||||
List<SearchIndexField> fields =
|
||||
Arrays.asList(
|
||||
getField("field1_" + i, SearchIndexDataType.KEYWORD, fieldTagLabel),
|
||||
getField("field2_" + i, SearchIndexDataType.TEXT, null));
|
||||
|
||||
CreateSearchIndex createSearchIndex =
|
||||
createRequest(test.getDisplayName() + "_pagination_" + i)
|
||||
.withFields(fields)
|
||||
.withTags(List.of(searchIndexTagLabel));
|
||||
|
||||
SearchIndex searchIndex = createEntity(createSearchIndex, ADMIN_AUTH_HEADERS);
|
||||
createdSearchIndexes.add(searchIndex);
|
||||
}
|
||||
|
||||
WebTarget target =
|
||||
getResource("searchIndexes").queryParam("fields", "tags").queryParam("limit", "50");
|
||||
|
||||
SearchIndexResource.SearchIndexList searchIndexList =
|
||||
TestUtils.get(target, SearchIndexResource.SearchIndexList.class, ADMIN_AUTH_HEADERS);
|
||||
assertNotNull(searchIndexList.getData());
|
||||
|
||||
List<SearchIndex> ourSearchIndexes =
|
||||
searchIndexList.getData().stream()
|
||||
.filter(
|
||||
si -> createdSearchIndexes.stream().anyMatch(csi -> csi.getId().equals(si.getId())))
|
||||
.collect(Collectors.toList());
|
||||
|
||||
assertFalse(
|
||||
ourSearchIndexes.isEmpty(),
|
||||
"Should find at least one of our created search indexes in pagination");
|
||||
|
||||
for (SearchIndex searchIndex : ourSearchIndexes) {
|
||||
assertNotNull(
|
||||
searchIndex.getTags(),
|
||||
"SearchIndex-level tags should not be null when fields=tags in pagination");
|
||||
assertEquals(
|
||||
1, searchIndex.getTags().size(), "Should have exactly one search index-level tag");
|
||||
assertEquals(searchIndexTagLabel.getTagFQN(), searchIndex.getTags().get(0).getTagFQN());
|
||||
|
||||
// Fields should not have tags when only fields=tags is specified
|
||||
if (searchIndex.getFields() != null) {
|
||||
for (SearchIndexField field : searchIndex.getFields()) {
|
||||
assertTrue(
|
||||
field.getTags() == null || field.getTags().isEmpty(),
|
||||
"Field tags should not be populated when only fields=tags is specified in pagination");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
target =
|
||||
getResource("searchIndexes").queryParam("fields", "fields,tags").queryParam("limit", "50");
|
||||
|
||||
searchIndexList =
|
||||
TestUtils.get(target, SearchIndexResource.SearchIndexList.class, ADMIN_AUTH_HEADERS);
|
||||
assertNotNull(searchIndexList.getData());
|
||||
|
||||
// Verify at least one of our created search indexes is in the response
|
||||
ourSearchIndexes =
|
||||
searchIndexList.getData().stream()
|
||||
.filter(
|
||||
si -> createdSearchIndexes.stream().anyMatch(csi -> csi.getId().equals(si.getId())))
|
||||
.collect(Collectors.toList());
|
||||
|
||||
assertFalse(
|
||||
ourSearchIndexes.isEmpty(),
|
||||
"Should find at least one of our created search indexes in pagination");
|
||||
|
||||
// Verify both search index-level and field-level tags are fetched
|
||||
for (SearchIndex searchIndex : ourSearchIndexes) {
|
||||
// Verify search index-level tags
|
||||
assertNotNull(
|
||||
searchIndex.getTags(),
|
||||
"SearchIndex-level tags should not be null in pagination with fields,tags");
|
||||
assertEquals(
|
||||
1, searchIndex.getTags().size(), "Should have exactly one search index-level tag");
|
||||
assertEquals(searchIndexTagLabel.getTagFQN(), searchIndex.getTags().get(0).getTagFQN());
|
||||
|
||||
// Verify field-level tags
|
||||
assertNotNull(
|
||||
searchIndex.getFields(), "Fields should not be null when fields includes fields");
|
||||
|
||||
SearchIndexField field1 =
|
||||
searchIndex.getFields().stream()
|
||||
.filter(f -> f.getName().startsWith("field1_"))
|
||||
.findFirst()
|
||||
.orElseThrow(() -> new AssertionError("Should find field1 field"));
|
||||
|
||||
assertNotNull(
|
||||
field1.getTags(), "Field tags should not be null when fields=fields,tags in pagination");
|
||||
assertTrue(field1.getTags().size() >= 1, "Field should have at least one tag");
|
||||
boolean hasExpectedTag =
|
||||
field1.getTags().stream()
|
||||
.anyMatch(tag -> tag.getTagFQN().equals(fieldTagLabel.getTagFQN()));
|
||||
assertTrue(
|
||||
hasExpectedTag, "Field should have the expected tag: " + fieldTagLabel.getTagFQN());
|
||||
|
||||
SearchIndexField field2 =
|
||||
searchIndex.getFields().stream()
|
||||
.filter(f -> f.getName().startsWith("field2_"))
|
||||
.findFirst()
|
||||
.orElseThrow(() -> new AssertionError("Should find field2 field"));
|
||||
|
||||
assertTrue(
|
||||
field2.getTags() == null || field2.getTags().isEmpty(), "field2 should not have tags");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -6,6 +6,8 @@ import static jakarta.ws.rs.core.Response.Status.NOT_FOUND;
|
||||
import static jakarta.ws.rs.core.Response.Status.OK;
|
||||
import static java.util.Collections.singletonList;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.openmetadata.common.utils.CommonUtil.listOf;
|
||||
@ -43,6 +45,7 @@ import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.UUID;
|
||||
import java.util.stream.Collectors;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.http.client.HttpResponseException;
|
||||
import org.junit.jupiter.api.MethodOrderer;
|
||||
@ -863,4 +866,125 @@ public class ContainerResourceTest extends EntityResourceTest<Container, CreateC
|
||||
JsonUtils.readObjects(actual, ContainerFileFormat.class);
|
||||
assertListProperty(expected, actualFormats, (c1, c2) -> assertEquals(c1.name(), c2.name()));
|
||||
}
|
||||
|
||||
@Test
|
||||
@Order(2)
|
||||
void test_paginationFetchesTagsAtBothEntityAndFieldLevels(TestInfo test) throws IOException {
|
||||
// Use existing tags that are already set up in the test environment
|
||||
TagLabel containerTagLabel = USER_ADDRESS_TAG_LABEL;
|
||||
TagLabel columnTagLabel = GLOSSARY1_TERM1_LABEL;
|
||||
|
||||
// Create multiple containers with tags at both container and column levels
|
||||
List<Container> createdContainers = new ArrayList<>();
|
||||
for (int i = 0; i < 5; i++) {
|
||||
List<Column> columns =
|
||||
Arrays.asList(
|
||||
getColumn("col1_" + i, BIGINT, null).withTags(List.of(columnTagLabel)),
|
||||
getColumn("col2_" + i, ColumnDataType.VARCHAR, null).withDataLength(50));
|
||||
|
||||
ContainerDataModel dataModel =
|
||||
new ContainerDataModel().withIsPartitioned(false).withColumns(columns);
|
||||
|
||||
CreateContainer createContainer =
|
||||
createRequest(test.getDisplayName() + "_pagination_" + i)
|
||||
.withDataModel(dataModel)
|
||||
.withTags(List.of(containerTagLabel));
|
||||
|
||||
Container container = createEntity(createContainer, ADMIN_AUTH_HEADERS);
|
||||
createdContainers.add(container);
|
||||
}
|
||||
|
||||
// Test pagination with fields=tags (should fetch container-level tags only)
|
||||
WebTarget target =
|
||||
getResource("containers").queryParam("fields", "tags").queryParam("limit", "50");
|
||||
|
||||
ContainerList containerList = TestUtils.get(target, ContainerList.class, ADMIN_AUTH_HEADERS);
|
||||
assertNotNull(containerList.getData());
|
||||
|
||||
// Verify at least one of our created containers is in the response
|
||||
List<Container> ourContainers =
|
||||
containerList.getData().stream()
|
||||
.filter(c -> createdContainers.stream().anyMatch(cc -> cc.getId().equals(c.getId())))
|
||||
.collect(Collectors.toList());
|
||||
|
||||
assertFalse(
|
||||
ourContainers.isEmpty(),
|
||||
"Should find at least one of our created containers in pagination");
|
||||
|
||||
// Verify container-level tags are fetched
|
||||
for (Container container : ourContainers) {
|
||||
assertNotNull(
|
||||
container.getTags(),
|
||||
"Container-level tags should not be null when fields=tags in pagination");
|
||||
assertEquals(1, container.getTags().size(), "Should have exactly one container-level tag");
|
||||
assertEquals(containerTagLabel.getTagFQN(), container.getTags().get(0).getTagFQN());
|
||||
|
||||
// Columns should not have tags when only fields=tags is specified
|
||||
if (container.getDataModel() != null && container.getDataModel().getColumns() != null) {
|
||||
for (Column col : container.getDataModel().getColumns()) {
|
||||
assertTrue(
|
||||
col.getTags() == null || col.getTags().isEmpty(),
|
||||
"Column tags should not be populated when only fields=tags is specified in pagination");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Test pagination with fields=dataModel,tags (should fetch both container and column tags)
|
||||
target =
|
||||
getResource("containers").queryParam("fields", "dataModel,tags").queryParam("limit", "50");
|
||||
|
||||
containerList = TestUtils.get(target, ContainerList.class, ADMIN_AUTH_HEADERS);
|
||||
assertNotNull(containerList.getData());
|
||||
|
||||
// Verify at least one of our created containers is in the response
|
||||
ourContainers =
|
||||
containerList.getData().stream()
|
||||
.filter(c -> createdContainers.stream().anyMatch(cc -> cc.getId().equals(c.getId())))
|
||||
.collect(Collectors.toList());
|
||||
|
||||
assertFalse(
|
||||
ourContainers.isEmpty(),
|
||||
"Should find at least one of our created containers in pagination");
|
||||
|
||||
// Verify both container-level and column-level tags are fetched
|
||||
for (Container container : ourContainers) {
|
||||
// Verify container-level tags
|
||||
assertNotNull(
|
||||
container.getTags(),
|
||||
"Container-level tags should not be null in pagination with dataModel,tags");
|
||||
assertEquals(1, container.getTags().size(), "Should have exactly one container-level tag");
|
||||
assertEquals(containerTagLabel.getTagFQN(), container.getTags().get(0).getTagFQN());
|
||||
|
||||
// Verify column-level tags
|
||||
assertNotNull(
|
||||
container.getDataModel(), "DataModel should not be null when fields includes dataModel");
|
||||
assertNotNull(container.getDataModel().getColumns(), "Columns should not be null");
|
||||
|
||||
Column col1 =
|
||||
container.getDataModel().getColumns().stream()
|
||||
.filter(c -> c.getName().startsWith("col1_"))
|
||||
.findFirst()
|
||||
.orElseThrow(() -> new AssertionError("Should find col1 column"));
|
||||
|
||||
assertNotNull(
|
||||
col1.getTags(),
|
||||
"Column tags should not be null when fields=dataModel,tags in pagination");
|
||||
assertTrue(col1.getTags().size() >= 1, "Column should have at least one tag");
|
||||
// Check that our expected tag is present
|
||||
boolean hasExpectedTag =
|
||||
col1.getTags().stream()
|
||||
.anyMatch(tag -> tag.getTagFQN().equals(columnTagLabel.getTagFQN()));
|
||||
assertTrue(
|
||||
hasExpectedTag, "Column should have the expected tag: " + columnTagLabel.getTagFQN());
|
||||
|
||||
// col2 should not have tags
|
||||
Column col2 =
|
||||
container.getDataModel().getColumns().stream()
|
||||
.filter(c -> c.getName().startsWith("col2_"))
|
||||
.findFirst()
|
||||
.orElseThrow(() -> new AssertionError("Should find col2 column"));
|
||||
|
||||
assertTrue(col2.getTags() == null || col2.getTags().isEmpty(), "col2 should not have tags");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -18,6 +18,7 @@ import static jakarta.ws.rs.core.Response.Status.NOT_FOUND;
|
||||
import static jakarta.ws.rs.core.Response.Status.OK;
|
||||
import static java.util.Collections.singletonList;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.openmetadata.common.utils.CommonUtil.listOf;
|
||||
@ -43,6 +44,7 @@ import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.UUID;
|
||||
import java.util.stream.Collectors;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.http.client.HttpResponseException;
|
||||
import org.junit.jupiter.api.Test;
|
||||
@ -621,4 +623,117 @@ public class TopicResourceTest extends EntityResourceTest<Topic, CreateTopic> {
|
||||
// Check the nested columns
|
||||
assertFields(expectedField.getChildren(), actualField.getChildren());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_paginationFetchesTagsAtBothEntityAndFieldLevels(TestInfo test) throws IOException {
|
||||
// Use existing tags that are already set up in the test environment
|
||||
TagLabel topicTagLabel = USER_ADDRESS_TAG_LABEL;
|
||||
TagLabel fieldTagLabel = PERSONAL_DATA_TAG_LABEL;
|
||||
|
||||
// Create multiple topics with tags at both topic and field levels
|
||||
List<Topic> createdTopics = new ArrayList<>();
|
||||
for (int i = 0; i < 5; i++) {
|
||||
List<Field> schemaFields =
|
||||
Arrays.asList(
|
||||
getField("field1_" + i, FieldDataType.STRING, fieldTagLabel),
|
||||
getField("field2_" + i, FieldDataType.STRING, null));
|
||||
|
||||
MessageSchema messageSchema =
|
||||
new MessageSchema().withSchemaType(SchemaType.Avro).withSchemaFields(schemaFields);
|
||||
|
||||
CreateTopic createTopic =
|
||||
createRequest(test.getDisplayName() + "_pagination_" + i)
|
||||
.withMessageSchema(messageSchema)
|
||||
.withTags(List.of(topicTagLabel));
|
||||
|
||||
Topic topic = createEntity(createTopic, ADMIN_AUTH_HEADERS);
|
||||
createdTopics.add(topic);
|
||||
}
|
||||
|
||||
// Test pagination with fields=tags (should fetch topic-level tags only)
|
||||
WebTarget target = getResource("topics").queryParam("fields", "tags").queryParam("limit", "10");
|
||||
|
||||
TopicList topicList = TestUtils.get(target, TopicList.class, ADMIN_AUTH_HEADERS);
|
||||
assertNotNull(topicList.getData());
|
||||
|
||||
// Verify at least one of our created topics is in the response
|
||||
List<Topic> ourTopics =
|
||||
topicList.getData().stream()
|
||||
.filter(t -> createdTopics.stream().anyMatch(ct -> ct.getId().equals(t.getId())))
|
||||
.collect(Collectors.toList());
|
||||
|
||||
assertFalse(
|
||||
ourTopics.isEmpty(), "Should find at least one of our created topics in pagination");
|
||||
|
||||
// Verify topic-level tags are fetched
|
||||
for (Topic topic : ourTopics) {
|
||||
assertNotNull(
|
||||
topic.getTags(), "Topic-level tags should not be null when fields=tags in pagination");
|
||||
assertEquals(1, topic.getTags().size(), "Should have exactly one topic-level tag");
|
||||
assertEquals(topicTagLabel.getTagFQN(), topic.getTags().get(0).getTagFQN());
|
||||
|
||||
// Fields should not have tags when only fields=tags is specified
|
||||
if (topic.getMessageSchema() != null && topic.getMessageSchema().getSchemaFields() != null) {
|
||||
for (Field field : topic.getMessageSchema().getSchemaFields()) {
|
||||
assertTrue(
|
||||
field.getTags() == null || field.getTags().isEmpty(),
|
||||
"Field tags should not be populated when only fields=tags is specified in pagination");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Test pagination with fields=messageSchema,tags (should fetch both topic and field tags)
|
||||
target =
|
||||
getResource("topics").queryParam("fields", "messageSchema,tags").queryParam("limit", "10");
|
||||
|
||||
topicList = TestUtils.get(target, TopicList.class, ADMIN_AUTH_HEADERS);
|
||||
assertNotNull(topicList.getData());
|
||||
|
||||
// Verify at least one of our created topics is in the response
|
||||
ourTopics =
|
||||
topicList.getData().stream()
|
||||
.filter(t -> createdTopics.stream().anyMatch(ct -> ct.getId().equals(t.getId())))
|
||||
.collect(Collectors.toList());
|
||||
|
||||
assertFalse(
|
||||
ourTopics.isEmpty(), "Should find at least one of our created topics in pagination");
|
||||
|
||||
// Verify both topic-level and field-level tags are fetched
|
||||
for (Topic topic : ourTopics) {
|
||||
// Verify topic-level tags
|
||||
assertNotNull(
|
||||
topic.getTags(),
|
||||
"Topic-level tags should not be null in pagination with messageSchema,tags");
|
||||
assertEquals(1, topic.getTags().size(), "Should have exactly one topic-level tag");
|
||||
assertEquals(topicTagLabel.getTagFQN(), topic.getTags().get(0).getTagFQN());
|
||||
|
||||
// Verify field-level tags
|
||||
assertNotNull(
|
||||
topic.getMessageSchema(),
|
||||
"MessageSchema should not be null when fields includes messageSchema");
|
||||
assertNotNull(topic.getMessageSchema().getSchemaFields(), "Schema fields should not be null");
|
||||
|
||||
Field field1 =
|
||||
topic.getMessageSchema().getSchemaFields().stream()
|
||||
.filter(f -> f.getName().startsWith("field1_"))
|
||||
.findFirst()
|
||||
.orElseThrow(() -> new AssertionError("Should find field1 field"));
|
||||
|
||||
assertNotNull(
|
||||
field1.getTags(),
|
||||
"Field tags should not be null when fields=messageSchema,tags in pagination");
|
||||
assertEquals(1, field1.getTags().size(), "Field should have exactly one tag");
|
||||
assertEquals(fieldTagLabel.getTagFQN(), field1.getTags().get(0).getTagFQN());
|
||||
|
||||
// field2 should not have tags
|
||||
Field field2 =
|
||||
topic.getMessageSchema().getSchemaFields().stream()
|
||||
.filter(f -> f.getName().startsWith("field2_"))
|
||||
.findFirst()
|
||||
.orElseThrow(() -> new AssertionError("Should find field2 field"));
|
||||
|
||||
assertTrue(
|
||||
field2.getTags() == null || field2.getTags().isEmpty(), "field2 should not have tags");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user