From 5e41039b97d0f5338d99fb25e548730d4ce154eb Mon Sep 17 00:00:00 2001 From: Sriharsha Chintalapani Date: Mon, 4 Aug 2025 02:04:18 -0700 Subject: [PATCH] Bug: Repositories overriding fieldSetters can fail to load tags at entity level (#22622) * Bug: Repositories overriding fieldSetters can fail to load tags at entity level * Bug: Repositories overriding fieldSetters can fail to load tags at entity level * fix build * Fix Test * Fix Test * fix test --------- Co-authored-by: sonikashah Co-authored-by: sonika-shah <58761340+sonika-shah@users.noreply.github.com> --- .../service/jdbi3/APIEndpointRepository.java | 60 ++++--- .../service/jdbi3/ContainerRepository.java | 37 ++-- .../jdbi3/DashboardDataModelRepository.java | 29 +++- .../service/jdbi3/EntityRepository.java | 2 +- .../service/jdbi3/PipelineRepository.java | 29 +++- .../service/jdbi3/SearchIndexRepository.java | 21 ++- .../service/jdbi3/TableRepository.java | 21 ++- .../service/jdbi3/TopicRepository.java | 31 +++- .../apis/APIEndpointResourceTest.java | 164 ++++++++++++++++++ .../databases/TableResourceTest.java | 110 ++++++++++++ .../DashboardDataModelResourceTest.java | 135 ++++++++++++++ .../pipelines/PipelineResourceTest.java | 125 +++++++++++++ .../searchindex/SearchIndexResourceTest.java | 116 +++++++++++++ .../storages/ContainerResourceTest.java | 124 +++++++++++++ .../resources/topics/TopicResourceTest.java | 115 ++++++++++++ 15 files changed, 1052 insertions(+), 67 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/APIEndpointRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/APIEndpointRepository.java index f129e258a7f..dd2112ee319 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/APIEndpointRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/APIEndpointRepository.java @@ -25,6 +25,7 @@ import static org.openmetadata.service.resources.tags.TagLabelUtil.addDerivedTag import static org.openmetadata.service.resources.tags.TagLabelUtil.checkMutuallyExclusive; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.function.BiPredicate; @@ -184,32 +185,45 @@ public class APIEndpointRepository extends EntityRepository { return; } - // Bulk fetch tags for request schemas - List endpointsWithRequestSchema = - apiEndpoints.stream() - .filter(e -> e.getRequestSchema() != null) - .collect(java.util.stream.Collectors.toList()); - - if (!endpointsWithRequestSchema.isEmpty()) { - bulkPopulateEntityFieldTags( - endpointsWithRequestSchema, - entityType, - e -> e.getRequestSchema().getSchemaFields(), - e -> e.getFullyQualifiedName() + ".requestSchema"); + // First, fetch endpoint-level tags (important for search indexing) + List entityFQNs = + apiEndpoints.stream().map(APIEndpoint::getFullyQualifiedName).toList(); + Map> tagsMap = batchFetchTags(entityFQNs); + for (APIEndpoint endpoint : apiEndpoints) { + endpoint.setTags( + addDerivedTags( + tagsMap.getOrDefault(endpoint.getFullyQualifiedName(), Collections.emptyList()))); } - // Bulk fetch tags for response schemas - List endpointsWithResponseSchema = - apiEndpoints.stream() - .filter(e -> e.getResponseSchema() != null) - .collect(java.util.stream.Collectors.toList()); + // Then, if schemas are requested, also fetch schema field tags + if (fields.contains("requestSchema") || fields.contains("responseSchema")) { + // Bulk fetch tags for request schemas + List endpointsWithRequestSchema = + apiEndpoints.stream() + .filter(e -> e.getRequestSchema() != null) + .collect(java.util.stream.Collectors.toList()); - if (!endpointsWithResponseSchema.isEmpty()) { - bulkPopulateEntityFieldTags( - endpointsWithResponseSchema, - entityType, - e -> e.getResponseSchema().getSchemaFields(), - e -> e.getFullyQualifiedName() + ".responseSchema"); + if (!endpointsWithRequestSchema.isEmpty()) { + bulkPopulateEntityFieldTags( + endpointsWithRequestSchema, + entityType, + e -> e.getRequestSchema().getSchemaFields(), + e -> e.getFullyQualifiedName() + ".requestSchema"); + } + + // Bulk fetch tags for response schemas + List endpointsWithResponseSchema = + apiEndpoints.stream() + .filter(e -> e.getResponseSchema() != null) + .collect(java.util.stream.Collectors.toList()); + + if (!endpointsWithResponseSchema.isEmpty()) { + bulkPopulateEntityFieldTags( + endpointsWithResponseSchema, + entityType, + e -> e.getResponseSchema().getSchemaFields(), + e -> e.getFullyQualifiedName() + ".responseSchema"); + } } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ContainerRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ContainerRepository.java index 8bbbb95ca3a..8074d5fe837 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ContainerRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ContainerRepository.java @@ -10,10 +10,12 @@ import static org.openmetadata.service.Entity.FIELD_TAGS; import static org.openmetadata.service.Entity.STORAGE_SERVICE; import static org.openmetadata.service.Entity.getEntityReferenceById; import static org.openmetadata.service.Entity.populateEntityFieldTags; +import static org.openmetadata.service.resources.tags.TagLabelUtil.addDerivedTags; import static org.openmetadata.service.util.EntityUtil.getEntityReferences; import com.google.common.collect.Lists; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -102,18 +104,31 @@ public class ContainerRepository extends EntityRepository { if (!fields.contains(FIELD_TAGS) || containers == null || containers.isEmpty()) { return; } - // Filter containers that have data models and use bulk tag fetching - List containersWithDataModels = - containers.stream() - .filter(c -> c.getDataModel() != null) - .collect(java.util.stream.Collectors.toList()); - if (!containersWithDataModels.isEmpty()) { - bulkPopulateEntityFieldTags( - containersWithDataModels, - entityType, - c -> c.getDataModel().getColumns(), - Container::getFullyQualifiedName); + // First, fetch container-level tags (important for search indexing) + List entityFQNs = containers.stream().map(Container::getFullyQualifiedName).toList(); + Map> tagsMap = batchFetchTags(entityFQNs); + for (Container container : containers) { + container.setTags( + addDerivedTags( + tagsMap.getOrDefault(container.getFullyQualifiedName(), Collections.emptyList()))); + } + + // Then, if dataModel field is requested, also fetch data model column tags + if (fields.contains("dataModel")) { + // Filter containers that have data models and use bulk tag fetching + List containersWithDataModels = + containers.stream() + .filter(c -> c.getDataModel() != null) + .collect(java.util.stream.Collectors.toList()); + + if (!containersWithDataModels.isEmpty()) { + bulkPopulateEntityFieldTags( + containersWithDataModels, + entityType, + c -> c.getDataModel().getColumns(), + Container::getFullyQualifiedName); + } } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DashboardDataModelRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DashboardDataModelRepository.java index 7d27ee5fed4..7faab1e44b1 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DashboardDataModelRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DashboardDataModelRepository.java @@ -17,9 +17,12 @@ import static org.openmetadata.schema.type.Include.ALL; import static org.openmetadata.service.Entity.DASHBOARD_DATA_MODEL; import static org.openmetadata.service.Entity.FIELD_TAGS; import static org.openmetadata.service.Entity.populateEntityFieldTags; +import static org.openmetadata.service.resources.tags.TagLabelUtil.addDerivedTags; import java.util.ArrayList; +import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.UUID; import lombok.SneakyThrows; @@ -181,12 +184,26 @@ public class DashboardDataModelRepository extends EntityRepository entityFQNs = + dataModels.stream().map(DashboardDataModel::getFullyQualifiedName).toList(); + Map> tagsMap = batchFetchTags(entityFQNs); + for (DashboardDataModel dataModel : dataModels) { + dataModel.setTags( + addDerivedTags( + tagsMap.getOrDefault(dataModel.getFullyQualifiedName(), Collections.emptyList()))); + } + + // Then, if columns field is requested, also fetch column-level tags + if (fields.contains("columns")) { + // Use bulk tag fetching to avoid N+1 queries + bulkPopulateEntityFieldTags( + dataModels, + entityType, + DashboardDataModel::getColumns, + DashboardDataModel::getFullyQualifiedName); + } } @Override diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java index 57b42bf69a8..bb1509c69a4 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java @@ -5323,7 +5323,7 @@ public abstract class EntityRepository { return tags.stream().map(this::createTagKey).collect(Collectors.toSet()); } - private Map> batchFetchTags(List entityFQNs) { + protected Map> batchFetchTags(List entityFQNs) { if (entityFQNs == null || entityFQNs.isEmpty()) { return Collections.emptyMap(); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/PipelineRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/PipelineRepository.java index ce0e6c40abd..efcd8473b93 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/PipelineRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/PipelineRepository.java @@ -28,6 +28,7 @@ import static org.openmetadata.service.util.EntityUtil.taskMatch; import jakarta.ws.rs.core.Response; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -203,13 +204,27 @@ public class PipelineRepository extends EntityRepository { return; } - // Use bulk tag and owner fetching for all pipeline tasks - for (Pipeline pipeline : pipelines) { - if (pipeline.getTasks() != null) { - // Still need individual calls here as tasks don't have bulk fetching pattern - // This is better than the original N+N pattern we had - getTaskTags(fields.contains(FIELD_TAGS), pipeline.getTasks()); - getTaskOwners(fields.contains(FIELD_OWNERS), pipeline.getTasks()); + // First, if tags are requested, fetch pipeline-level tags (important for search indexing) + if (fields.contains(FIELD_TAGS)) { + List entityFQNs = pipelines.stream().map(Pipeline::getFullyQualifiedName).toList(); + Map> tagsMap = batchFetchTags(entityFQNs); + for (Pipeline pipeline : pipelines) { + pipeline.setTags( + addDerivedTags( + tagsMap.getOrDefault(pipeline.getFullyQualifiedName(), Collections.emptyList()))); + } + } + + // Then, if tasks field is requested, also handle task-level tags and owners + if (fields.contains("tasks")) { + // Use bulk tag and owner fetching for all pipeline tasks + for (Pipeline pipeline : pipelines) { + if (pipeline.getTasks() != null) { + // Still need individual calls here as tasks don't have bulk fetching pattern + // This is better than the original N+N pattern we had + getTaskTags(fields.contains(FIELD_TAGS), pipeline.getTasks()); + getTaskOwners(fields.contains(FIELD_OWNERS), pipeline.getTasks()); + } } } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SearchIndexRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SearchIndexRepository.java index 2a9a196ae1d..f9bad6574da 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SearchIndexRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SearchIndexRepository.java @@ -26,6 +26,7 @@ import static org.openmetadata.service.resources.tags.TagLabelUtil.checkMutually import static org.openmetadata.service.util.EntityUtil.getSearchIndexField; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -204,9 +205,23 @@ public class SearchIndexRepository extends EntityRepository { if (!fields.contains(FIELD_TAGS) || searchIndexes == null || searchIndexes.isEmpty()) { return; } - // Use bulk tag fetching to avoid N+1 queries - bulkPopulateEntityFieldTags( - searchIndexes, entityType, SearchIndex::getFields, SearchIndex::getFullyQualifiedName); + + // First, fetch searchIndex-level tags (important for search indexing) + List entityFQNs = + searchIndexes.stream().map(SearchIndex::getFullyQualifiedName).toList(); + Map> tagsMap = batchFetchTags(entityFQNs); + for (SearchIndex searchIndex : searchIndexes) { + searchIndex.setTags( + addDerivedTags( + tagsMap.getOrDefault(searchIndex.getFullyQualifiedName(), Collections.emptyList()))); + } + + // Then, if fields are requested, also fetch field-level tags + if (fields.contains("fields")) { + // Use bulk tag fetching to avoid N+1 queries + bulkPopulateEntityFieldTags( + searchIndexes, entityType, SearchIndex::getFields, SearchIndex::getFullyQualifiedName); + } } @Override diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java index 50cb2ec1e8b..e53093577db 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java @@ -32,6 +32,7 @@ import static org.openmetadata.service.Entity.TEST_SUITE; import static org.openmetadata.service.Entity.getEntities; import static org.openmetadata.service.Entity.getEntityReferenceById; import static org.openmetadata.service.Entity.populateEntityFieldTags; +import static org.openmetadata.service.resources.tags.TagLabelUtil.addDerivedTags; import static org.openmetadata.service.search.SearchClient.GLOBAL_SEARCH_ALIAS; import static org.openmetadata.service.util.EntityUtil.getLocalColumnName; import static org.openmetadata.service.util.FullyQualifiedName.getColumnName; @@ -280,15 +281,21 @@ public class TableRepository extends EntityRepository { } private void fetchAndSetColumnTags(List
tables, Fields fields) { - if (!fields.contains(FIELD_TAGS) - || !fields.contains(COLUMN_FIELD) - || tables == null - || tables.isEmpty()) { + if (!fields.contains(FIELD_TAGS) || tables == null || tables.isEmpty()) { return; } - // Use bulk tag fetching to avoid N+1 queries - bulkPopulateEntityFieldTags( - tables, entityType, Table::getColumns, Table::getFullyQualifiedName); + List entityFQNs = tables.stream().map(Table::getFullyQualifiedName).toList(); + Map> tagsMap = batchFetchTags(entityFQNs); + for (Table table : tables) { + table.setTags( + addDerivedTags( + tagsMap.getOrDefault(table.getFullyQualifiedName(), Collections.emptyList()))); + } + + if (fields.contains(COLUMN_FIELD)) { + bulkPopulateEntityFieldTags( + tables, entityType, Table::getColumns, Table::getFullyQualifiedName); + } } @Override diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TopicRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TopicRepository.java index ae706eac925..f1d8e34a364 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TopicRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TopicRepository.java @@ -25,6 +25,7 @@ import static org.openmetadata.service.resources.tags.TagLabelUtil.addDerivedTag import static org.openmetadata.service.resources.tags.TagLabelUtil.checkMutuallyExclusive; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -174,16 +175,28 @@ public class TopicRepository extends EntityRepository { return; } - // Filter topics that have message schemas and use bulk tag fetching - List topicsWithSchemas = - topics.stream().filter(t -> t.getMessageSchema() != null).toList(); + // First, fetch topic-level tags (important for search indexing) + List entityFQNs = topics.stream().map(Topic::getFullyQualifiedName).toList(); + Map> tagsMap = batchFetchTags(entityFQNs); + for (Topic topic : topics) { + topic.setTags( + addDerivedTags( + tagsMap.getOrDefault(topic.getFullyQualifiedName(), Collections.emptyList()))); + } - if (!topicsWithSchemas.isEmpty()) { - bulkPopulateEntityFieldTags( - topicsWithSchemas, - entityType, - t -> t.getMessageSchema().getSchemaFields(), - Topic::getFullyQualifiedName); + // Then, if messageSchema field is requested, also fetch schema field tags + if (fields.contains("messageSchema")) { + // Filter topics that have message schemas and use bulk tag fetching + List topicsWithSchemas = + topics.stream().filter(t -> t.getMessageSchema() != null).toList(); + + if (!topicsWithSchemas.isEmpty()) { + bulkPopulateEntityFieldTags( + topicsWithSchemas, + entityType, + t -> t.getMessageSchema().getSchemaFields(), + Topic::getFullyQualifiedName); + } } } diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/apis/APIEndpointResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/apis/APIEndpointResourceTest.java index 82682f5d6ac..2efa9129a84 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/apis/APIEndpointResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/apis/APIEndpointResourceTest.java @@ -2,6 +2,7 @@ package org.openmetadata.service.resources.apis; import static jakarta.ws.rs.core.Response.Status.BAD_REQUEST; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.openmetadata.service.Entity.FIELD_OWNERS; @@ -27,6 +28,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.stream.Collectors; import org.apache.http.client.HttpResponseException; import org.junit.jupiter.api.MethodOrderer; import org.junit.jupiter.api.Order; @@ -382,4 +384,166 @@ public class APIEndpointResourceTest extends EntityResourceTest createdEndpoints = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + List requestFields = + Arrays.asList( + getField("requestField1_" + i, FieldDataType.STRING, fieldTagLabel), + getField("requestField2_" + i, FieldDataType.STRING, null)); + + List responseFields = + Arrays.asList( + getField("responseField1_" + i, FieldDataType.STRING, PII_SENSITIVE_TAG_LABEL), + getField("responseField2_" + i, FieldDataType.STRING, null)); + + APISchema requestSchema = new APISchema().withSchemaFields(requestFields); + APISchema responseSchema = new APISchema().withSchemaFields(responseFields); + + CreateAPIEndpoint createEndpoint = + createRequest(test.getDisplayName() + "_pagination_" + i) + .withRequestSchema(requestSchema) + .withResponseSchema(responseSchema) + .withTags(List.of(endpointTagLabel)); + + APIEndpoint endpoint = createEntity(createEndpoint, ADMIN_AUTH_HEADERS); + createdEndpoints.add(endpoint); + } + + WebTarget target = + getResource("apiEndpoints").queryParam("fields", "tags").queryParam("limit", "50"); + + APIEndpointResource.APIEndpointList endpointList = + TestUtils.get(target, APIEndpointResource.APIEndpointList.class, ADMIN_AUTH_HEADERS); + assertNotNull(endpointList.getData()); + + List ourEndpoints = + endpointList.getData().stream() + .filter(e -> createdEndpoints.stream().anyMatch(ce -> ce.getId().equals(e.getId()))) + .collect(Collectors.toList()); + + assertFalse( + ourEndpoints.isEmpty(), "Should find at least one of our created endpoints in pagination"); + + for (APIEndpoint endpoint : ourEndpoints) { + assertNotNull( + endpoint.getTags(), + "Endpoint-level tags should not be null when fields=tags in pagination"); + assertEquals(1, endpoint.getTags().size(), "Should have exactly one endpoint-level tag"); + assertEquals(endpointTagLabel.getTagFQN(), endpoint.getTags().get(0).getTagFQN()); + + if (endpoint.getRequestSchema() != null + && endpoint.getRequestSchema().getSchemaFields() != null) { + for (Field field : endpoint.getRequestSchema().getSchemaFields()) { + assertTrue( + field.getTags() == null || field.getTags().isEmpty(), + "Request field tags should not be populated when only fields=tags is specified in pagination"); + } + } + if (endpoint.getResponseSchema() != null + && endpoint.getResponseSchema().getSchemaFields() != null) { + for (Field field : endpoint.getResponseSchema().getSchemaFields()) { + assertTrue( + field.getTags() == null || field.getTags().isEmpty(), + "Response field tags should not be populated when only fields=tags is specified in pagination"); + } + } + } + + target = + getResource("apiEndpoints") + .queryParam("fields", "requestSchema,responseSchema,tags") + .queryParam("limit", "10"); + + endpointList = + TestUtils.get(target, APIEndpointResource.APIEndpointList.class, ADMIN_AUTH_HEADERS); + assertNotNull(endpointList.getData()); + + ourEndpoints = + endpointList.getData().stream() + .filter(e -> createdEndpoints.stream().anyMatch(ce -> ce.getId().equals(e.getId()))) + .collect(Collectors.toList()); + + assertFalse( + ourEndpoints.isEmpty(), "Should find at least one of our created endpoints in pagination"); + + // Verify both endpoint-level and field-level tags are fetched + for (APIEndpoint endpoint : ourEndpoints) { + // Verify endpoint-level tags + assertNotNull( + endpoint.getTags(), + "Endpoint-level tags should not be null in pagination with schemas,tags"); + assertEquals(1, endpoint.getTags().size(), "Should have exactly one endpoint-level tag"); + assertEquals(endpointTagLabel.getTagFQN(), endpoint.getTags().get(0).getTagFQN()); + + // Verify request field-level tags + assertNotNull( + endpoint.getRequestSchema(), + "RequestSchema should not be null when fields includes requestSchema"); + assertNotNull( + endpoint.getRequestSchema().getSchemaFields(), + "Request schema fields should not be null"); + + Field requestField1 = + endpoint.getRequestSchema().getSchemaFields().stream() + .filter(f -> f.getName().startsWith("requestField1_")) + .findFirst() + .orElseThrow(() -> new AssertionError("Should find requestField1 field")); + + assertNotNull( + requestField1.getTags(), + "Request field tags should not be null when fields=requestSchema,responseSchema,tags in pagination"); + assertEquals(1, requestField1.getTags().size(), "Request field should have exactly one tag"); + assertEquals(fieldTagLabel.getTagFQN(), requestField1.getTags().get(0).getTagFQN()); + + // Verify response field-level tags + assertNotNull( + endpoint.getResponseSchema(), + "ResponseSchema should not be null when fields includes responseSchema"); + assertNotNull( + endpoint.getResponseSchema().getSchemaFields(), + "Response schema fields should not be null"); + + Field responseField1 = + endpoint.getResponseSchema().getSchemaFields().stream() + .filter(f -> f.getName().startsWith("responseField1_")) + .findFirst() + .orElseThrow(() -> new AssertionError("Should find responseField1 field")); + + assertNotNull( + responseField1.getTags(), + "Response field tags should not be null when fields=requestSchema,responseSchema,tags in pagination"); + assertEquals( + 1, responseField1.getTags().size(), "Response field should have exactly one tag"); + assertEquals( + PII_SENSITIVE_TAG_LABEL.getTagFQN(), responseField1.getTags().get(0).getTagFQN()); + + // Fields without tags should remain empty + Field requestField2 = + endpoint.getRequestSchema().getSchemaFields().stream() + .filter(f -> f.getName().startsWith("requestField2_")) + .findFirst() + .orElseThrow(() -> new AssertionError("Should find requestField2 field")); + + assertTrue( + requestField2.getTags() == null || requestField2.getTags().isEmpty(), + "requestField2 should not have tags"); + + Field responseField2 = + endpoint.getResponseSchema().getSchemaFields().stream() + .filter(f -> f.getName().startsWith("responseField2_")) + .findFirst() + .orElseThrow(() -> new AssertionError("Should find responseField2 field")); + + assertTrue( + responseField2.getTags() == null || responseField2.getTags().isEmpty(), + "responseField2 should not have tags"); + } + } } diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/TableResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/TableResourceTest.java index b30d9d76910..de20928fa4d 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/TableResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/TableResourceTest.java @@ -5752,6 +5752,116 @@ public class TableResourceTest extends EntityResourceTest { && e.getRelatedEntity().getId().equals(tableC.getId()))); } + @Test + @Order(2) + void test_paginationFetchesTagsAtBothEntityAndFieldLevels(TestInfo test) throws IOException { + TagLabel tableTagLabel = USER_ADDRESS_TAG_LABEL; + TagLabel columnTagLabel = GLOSSARY1_TERM1_LABEL; + + List
createdTables = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + List columns = + Arrays.asList( + getColumn("col1_" + i, BIGINT, null).withTags(List.of(columnTagLabel)), + getColumn("col2_" + i, VARCHAR, null).withDataLength(50)); + + CreateTable createTable = + createRequest(test.getDisplayName() + "_pagination_" + i) + .withColumns(columns) + .withTags(List.of(tableTagLabel)) + .withTableConstraints(null); + + Table table = createEntity(createTable, ADMIN_AUTH_HEADERS); + createdTables.add(table); + } + + // Test pagination with fields=tags (should fetch table-level tags only) + WebTarget target = + getResource("tables") + .queryParam("fields", "tags") + .queryParam("limit", "50") + .queryParam( + "databaseSchema", + DATABASE_SCHEMA.getFullyQualifiedName()); // Filter by schema to get our tables + + TableList tableList = TestUtils.get(target, TableList.class, ADMIN_AUTH_HEADERS); + assertNotNull(tableList.getData()); + + List
ourTables = + tableList.getData().stream() + .filter(t -> createdTables.stream().anyMatch(ct -> ct.getId().equals(t.getId()))) + .collect(Collectors.toList()); + + assertFalse( + ourTables.isEmpty(), "Should find at least one of our created tables in pagination"); + + for (Table table : ourTables) { + assertNotNull( + table.getTags(), "Table-level tags should not be null when fields=tags in pagination"); + assertEquals(1, table.getTags().size(), "Should have exactly one table-level tag"); + assertEquals(tableTagLabel.getTagFQN(), table.getTags().get(0).getTagFQN()); + + if (table.getColumns() != null) { + for (Column col : table.getColumns()) { + assertTrue( + col.getTags() == null || col.getTags().isEmpty(), + "Column tags should not be populated when only fields=tags is specified in pagination"); + } + } + } + + target = + getResource("tables") + .queryParam("fields", "columns,tags") + .queryParam("limit", "50") + .queryParam( + "databaseSchema", + DATABASE_SCHEMA.getFullyQualifiedName()); // Filter by schema to get our tables + + tableList = TestUtils.get(target, TableList.class, ADMIN_AUTH_HEADERS); + assertNotNull(tableList.getData()); + + ourTables = + tableList.getData().stream() + .filter(t -> createdTables.stream().anyMatch(ct -> ct.getId().equals(t.getId()))) + .collect(Collectors.toList()); + + assertFalse( + ourTables.isEmpty(), "Should find at least one of our created tables in pagination"); + + for (Table table : ourTables) { + assertNotNull( + table.getTags(), "Table-level tags should not be null in pagination with columns,tags"); + assertEquals(1, table.getTags().size(), "Should have exactly one table-level tag"); + assertEquals(tableTagLabel.getTagFQN(), table.getTags().get(0).getTagFQN()); + + assertNotNull(table.getColumns(), "Columns should not be null when fields includes columns"); + Column col1 = + table.getColumns().stream() + .filter(c -> c.getName().startsWith("col1_")) + .findFirst() + .orElseThrow(() -> new AssertionError("Should find col1 column")); + + assertNotNull( + col1.getTags(), "Column tags should not be null when fields=columns,tags in pagination"); + assertTrue(!col1.getTags().isEmpty(), "Column should have at least one tag"); + // Check that our expected tag is present + boolean hasExpectedTag = + col1.getTags().stream() + .anyMatch(tag -> tag.getTagFQN().equals(columnTagLabel.getTagFQN())); + assertTrue( + hasExpectedTag, "Column should have the expected tag: " + columnTagLabel.getTagFQN()); + + Column col2 = + table.getColumns().stream() + .filter(c -> c.getName().startsWith("col2_")) + .findFirst() + .orElseThrow(() -> new AssertionError("Should find col2 column")); + + assertTrue(col2.getTags() == null || col2.getTags().isEmpty(), "col2 should not have tags"); + } + } + @Test void test_compositeKeyConstraintIndexOutOfBounds_fixed(TestInfo test) throws IOException { // Create a schema for this test to avoid conflicts diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/datamodels/DashboardDataModelResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/datamodels/DashboardDataModelResourceTest.java index a9cd9ab24ff..f1f76a3c29d 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/datamodels/DashboardDataModelResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/datamodels/DashboardDataModelResourceTest.java @@ -16,6 +16,7 @@ package org.openmetadata.service.resources.datamodels; import static jakarta.ws.rs.core.Response.Status.BAD_REQUEST; import static jakarta.ws.rs.core.Response.Status.NOT_FOUND; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.openmetadata.common.utils.CommonUtil.listOf; @@ -31,15 +32,21 @@ import static org.openmetadata.service.util.TestUtils.assertListNotNull; import static org.openmetadata.service.util.TestUtils.assertListNull; import static org.openmetadata.service.util.TestUtils.assertResponse; +import jakarta.ws.rs.client.WebTarget; import jakarta.ws.rs.core.Response.Status; import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; import lombok.extern.slf4j.Slf4j; import org.apache.http.client.HttpResponseException; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.api.TestMethodOrder; import org.junit.jupiter.api.parallel.Execution; import org.junit.jupiter.api.parallel.ExecutionMode; import org.openmetadata.schema.api.data.CreateDashboardDataModel; @@ -56,8 +63,10 @@ import org.openmetadata.service.exception.CatalogExceptionMessage; import org.openmetadata.service.resources.EntityResourceTest; import org.openmetadata.service.resources.services.DashboardServiceResourceTest; import org.openmetadata.service.util.ResultList; +import org.openmetadata.service.util.TestUtils; @Slf4j +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) public class DashboardDataModelResourceTest extends EntityResourceTest { @@ -289,4 +298,130 @@ public class DashboardDataModelResourceTest 8, mixedFieldsDataModel.getColumns().size(), "Should return all columns in mixed request"); assertNotNull(mixedFieldsDataModel.getOwners(), "Should also return other requested fields"); } + + @Test + @Order(1) + void test_paginationFetchesTagsAtBothEntityAndFieldLevels(TestInfo test) throws IOException { + TagLabel dataModelTagLabel = USER_ADDRESS_TAG_LABEL; + TagLabel columnTagLabel = PERSONAL_DATA_TAG_LABEL; + + List createdDataModels = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + List columns = + Arrays.asList( + getColumn("column1_" + i, BIGINT, columnTagLabel), + getColumn("column2_" + i, BIGINT, null), + getColumn("column3_" + i, INT, null)); + + CreateDashboardDataModel createDataModel = + createRequest(test.getDisplayName() + "_pagination_" + i) + .withColumns(columns) + .withTags(List.of(dataModelTagLabel)); + + DashboardDataModel dataModel = createEntity(createDataModel, ADMIN_AUTH_HEADERS); + createdDataModels.add(dataModel); + } + + // Test pagination with fields=tags (should fetch data model-level tags only) + WebTarget target = + getResource("dashboard/datamodels").queryParam("fields", "tags").queryParam("limit", "10"); + + DashboardDataModelResource.DashboardDataModelList dataModelList = + TestUtils.get( + target, DashboardDataModelResource.DashboardDataModelList.class, ADMIN_AUTH_HEADERS); + assertNotNull(dataModelList.getData()); + + // Verify at least one of our created data models is in the response + List ourDataModels = + dataModelList.getData().stream() + .filter( + dm -> createdDataModels.stream().anyMatch(cdm -> cdm.getId().equals(dm.getId()))) + .collect(java.util.stream.Collectors.toList()); + + assertFalse( + ourDataModels.isEmpty(), + "Should find at least one of our created data models in pagination"); + + // Verify data model-level tags are fetched + for (DashboardDataModel dataModel : ourDataModels) { + assertNotNull( + dataModel.getTags(), + "Data model-level tags should not be null when fields=tags in pagination"); + assertEquals(1, dataModel.getTags().size(), "Should have exactly one data model-level tag"); + assertEquals(dataModelTagLabel.getTagFQN(), dataModel.getTags().get(0).getTagFQN()); + + // DashboardDataModel returns columns by default even when not explicitly requested + // The columns retain their tags from creation. This is different from Table behavior + // but is the expected behavior for DashboardDataModel. + // The important part is that the entity-level tags are properly fetched. + } + + // Test pagination with fields=columns,tags (should fetch both data model and column tags) + target = + getResource("dashboard/datamodels") + .queryParam("fields", "columns,tags") + .queryParam("limit", "10"); + + dataModelList = + TestUtils.get( + target, DashboardDataModelResource.DashboardDataModelList.class, ADMIN_AUTH_HEADERS); + assertNotNull(dataModelList.getData()); + + // Verify at least one of our created data models is in the response + ourDataModels = + dataModelList.getData().stream() + .filter( + dm -> createdDataModels.stream().anyMatch(cdm -> cdm.getId().equals(dm.getId()))) + .collect(java.util.stream.Collectors.toList()); + + assertFalse( + ourDataModels.isEmpty(), + "Should find at least one of our created data models in pagination"); + + // Verify both data model-level and column-level tags are fetched + for (DashboardDataModel dataModel : ourDataModels) { + // Verify data model-level tags + assertNotNull( + dataModel.getTags(), + "Data model-level tags should not be null in pagination with columns,tags"); + assertEquals(1, dataModel.getTags().size(), "Should have exactly one data model-level tag"); + assertEquals(dataModelTagLabel.getTagFQN(), dataModel.getTags().get(0).getTagFQN()); + + // Verify column-level tags + assertNotNull( + dataModel.getColumns(), "Columns should not be null when fields includes columns"); + assertFalse(dataModel.getColumns().isEmpty(), "Columns should not be empty"); + + Column column1 = + dataModel.getColumns().stream() + .filter(c -> c.getName().startsWith("column1_")) + .findFirst() + .orElseThrow(() -> new AssertionError("Should find column1 column")); + + assertNotNull( + column1.getTags(), + "Column tags should not be null when fields=columns,tags in pagination"); + assertEquals(1, column1.getTags().size(), "Column should have exactly one tag"); + assertEquals(columnTagLabel.getTagFQN(), column1.getTags().get(0).getTagFQN()); + + // column2 and column3 should not have tags + Column column2 = + dataModel.getColumns().stream() + .filter(c -> c.getName().startsWith("column2_")) + .findFirst() + .orElseThrow(() -> new AssertionError("Should find column2 column")); + + assertTrue( + column2.getTags() == null || column2.getTags().isEmpty(), "column2 should not have tags"); + + Column column3 = + dataModel.getColumns().stream() + .filter(c -> c.getName().startsWith("column3_")) + .findFirst() + .orElseThrow(() -> new AssertionError("Should find column3 column")); + + assertTrue( + column3.getTags() == null || column3.getTags().isEmpty(), "column3 should not have tags"); + } + } } diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/pipelines/PipelineResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/pipelines/PipelineResourceTest.java index d3cfdd99b15..3b5bff3a180 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/pipelines/PipelineResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/pipelines/PipelineResourceTest.java @@ -17,6 +17,7 @@ import static jakarta.ws.rs.core.Response.Status.BAD_REQUEST; import static jakarta.ws.rs.core.Response.Status.NOT_FOUND; import static jakarta.ws.rs.core.Response.Status.OK; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.openmetadata.common.utils.CommonUtil.listOf; @@ -53,8 +54,11 @@ import lombok.extern.slf4j.Slf4j; import org.apache.http.client.HttpResponseException; import org.joda.time.DateTime; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.api.TestMethodOrder; import org.openmetadata.schema.api.data.CreatePipeline; import org.openmetadata.schema.api.services.CreatePipelineService; import org.openmetadata.schema.entity.data.Pipeline; @@ -78,6 +82,7 @@ import org.openmetadata.service.util.ResultList; import org.openmetadata.service.util.TestUtils; @Slf4j +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) public class PipelineResourceTest extends EntityResourceTest { public static List TASKS; @@ -839,4 +844,124 @@ public class PipelineResourceTest extends EntityResourceTest createdPipelines = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + List tasks = new ArrayList<>(); + for (int j = 0; j < 3; j++) { + Task task = + new Task() + .withName("task" + j + "_" + i) + .withDescription("description") + .withDisplayName("displayName") + .withSourceUrl("http://localhost:0"); + + if (j == 0) { + // Add tag to first task only + task.withTags(List.of(taskTagLabel)); + } + tasks.add(task); + } + + CreatePipeline createPipeline = + createRequest(test.getDisplayName() + "_pagination_" + i) + .withTasks(tasks) + .withTags(List.of(pipelineTagLabel)); + + Pipeline pipeline = createEntity(createPipeline, ADMIN_AUTH_HEADERS); + createdPipelines.add(pipeline); + } + + // Test pagination with fields=tags (should fetch pipeline-level tags only) + WebTarget target = + getResource("pipelines").queryParam("fields", "tags").queryParam("limit", "10"); + + PipelineList pipelineList = TestUtils.get(target, PipelineList.class, ADMIN_AUTH_HEADERS); + assertNotNull(pipelineList.getData()); + + // Verify at least one of our created pipelines is in the response + List ourPipelines = + pipelineList.getData().stream() + .filter(p -> createdPipelines.stream().anyMatch(cp -> cp.getId().equals(p.getId()))) + .collect(Collectors.toList()); + + assertFalse( + ourPipelines.isEmpty(), "Should find at least one of our created pipelines in pagination"); + + // Verify pipeline-level tags are fetched + for (Pipeline pipeline : ourPipelines) { + assertNotNull( + pipeline.getTags(), + "Pipeline-level tags should not be null when fields=tags in pagination"); + assertEquals(1, pipeline.getTags().size(), "Should have exactly one pipeline-level tag"); + assertEquals(pipelineTagLabel.getTagFQN(), pipeline.getTags().get(0).getTagFQN()); + + // Tasks should not have tags when only fields=tags is specified + if (pipeline.getTasks() != null) { + for (Task task : pipeline.getTasks()) { + assertTrue( + task.getTags() == null || task.getTags().isEmpty(), + "Task tags should not be populated when only fields=tags is specified in pagination"); + } + } + } + + // Test pagination with fields=tasks,tags (should fetch both pipeline and task tags) + target = getResource("pipelines").queryParam("fields", "tasks,tags").queryParam("limit", "10"); + + pipelineList = TestUtils.get(target, PipelineList.class, ADMIN_AUTH_HEADERS); + assertNotNull(pipelineList.getData()); + + // Verify at least one of our created pipelines is in the response + ourPipelines = + pipelineList.getData().stream() + .filter(p -> createdPipelines.stream().anyMatch(cp -> cp.getId().equals(p.getId()))) + .collect(Collectors.toList()); + + assertFalse( + ourPipelines.isEmpty(), "Should find at least one of our created pipelines in pagination"); + + // Verify both pipeline-level and task-level tags are fetched + for (Pipeline pipeline : ourPipelines) { + // Verify pipeline-level tags + assertNotNull( + pipeline.getTags(), + "Pipeline-level tags should not be null in pagination with tasks,tags"); + assertEquals(1, pipeline.getTags().size(), "Should have exactly one pipeline-level tag"); + assertEquals(pipelineTagLabel.getTagFQN(), pipeline.getTags().get(0).getTagFQN()); + + // Verify task-level tags + assertNotNull(pipeline.getTasks(), "Tasks should not be null when fields includes tasks"); + assertFalse(pipeline.getTasks().isEmpty(), "Tasks should not be empty"); + + // Find the first task which should have a tag + Task task0 = + pipeline.getTasks().stream() + .filter(t -> t.getName().startsWith("task0_")) + .findFirst() + .orElseThrow(() -> new AssertionError("Should find task0 task")); + + assertNotNull( + task0.getTags(), "Task tags should not be null when fields=tasks,tags in pagination"); + assertEquals(1, task0.getTags().size(), "Task should have exactly one tag"); + assertEquals(taskTagLabel.getTagFQN(), task0.getTags().get(0).getTagFQN()); + + // Other tasks should not have tags + for (Task task : pipeline.getTasks()) { + if (!task.getName().startsWith("task0_")) { + assertTrue( + task.getTags() == null || task.getTags().isEmpty(), + "Other tasks should not have tags"); + } + } + } + } } diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/searchindex/SearchIndexResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/searchindex/SearchIndexResourceTest.java index 1cc5a544bae..355f3a1cc21 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/searchindex/SearchIndexResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/searchindex/SearchIndexResourceTest.java @@ -19,6 +19,7 @@ import static jakarta.ws.rs.core.Response.Status.OK; import static java.util.Collections.singletonList; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.openmetadata.common.utils.CommonUtil.listOf; @@ -50,6 +51,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.apache.http.client.HttpResponseException; import org.junit.jupiter.api.Test; @@ -949,4 +951,118 @@ public class SearchIndexResourceTest extends EntityResourceTest createdSearchIndexes = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + List fields = + Arrays.asList( + getField("field1_" + i, SearchIndexDataType.KEYWORD, fieldTagLabel), + getField("field2_" + i, SearchIndexDataType.TEXT, null)); + + CreateSearchIndex createSearchIndex = + createRequest(test.getDisplayName() + "_pagination_" + i) + .withFields(fields) + .withTags(List.of(searchIndexTagLabel)); + + SearchIndex searchIndex = createEntity(createSearchIndex, ADMIN_AUTH_HEADERS); + createdSearchIndexes.add(searchIndex); + } + + WebTarget target = + getResource("searchIndexes").queryParam("fields", "tags").queryParam("limit", "50"); + + SearchIndexResource.SearchIndexList searchIndexList = + TestUtils.get(target, SearchIndexResource.SearchIndexList.class, ADMIN_AUTH_HEADERS); + assertNotNull(searchIndexList.getData()); + + List ourSearchIndexes = + searchIndexList.getData().stream() + .filter( + si -> createdSearchIndexes.stream().anyMatch(csi -> csi.getId().equals(si.getId()))) + .collect(Collectors.toList()); + + assertFalse( + ourSearchIndexes.isEmpty(), + "Should find at least one of our created search indexes in pagination"); + + for (SearchIndex searchIndex : ourSearchIndexes) { + assertNotNull( + searchIndex.getTags(), + "SearchIndex-level tags should not be null when fields=tags in pagination"); + assertEquals( + 1, searchIndex.getTags().size(), "Should have exactly one search index-level tag"); + assertEquals(searchIndexTagLabel.getTagFQN(), searchIndex.getTags().get(0).getTagFQN()); + + // Fields should not have tags when only fields=tags is specified + if (searchIndex.getFields() != null) { + for (SearchIndexField field : searchIndex.getFields()) { + assertTrue( + field.getTags() == null || field.getTags().isEmpty(), + "Field tags should not be populated when only fields=tags is specified in pagination"); + } + } + } + + target = + getResource("searchIndexes").queryParam("fields", "fields,tags").queryParam("limit", "50"); + + searchIndexList = + TestUtils.get(target, SearchIndexResource.SearchIndexList.class, ADMIN_AUTH_HEADERS); + assertNotNull(searchIndexList.getData()); + + // Verify at least one of our created search indexes is in the response + ourSearchIndexes = + searchIndexList.getData().stream() + .filter( + si -> createdSearchIndexes.stream().anyMatch(csi -> csi.getId().equals(si.getId()))) + .collect(Collectors.toList()); + + assertFalse( + ourSearchIndexes.isEmpty(), + "Should find at least one of our created search indexes in pagination"); + + // Verify both search index-level and field-level tags are fetched + for (SearchIndex searchIndex : ourSearchIndexes) { + // Verify search index-level tags + assertNotNull( + searchIndex.getTags(), + "SearchIndex-level tags should not be null in pagination with fields,tags"); + assertEquals( + 1, searchIndex.getTags().size(), "Should have exactly one search index-level tag"); + assertEquals(searchIndexTagLabel.getTagFQN(), searchIndex.getTags().get(0).getTagFQN()); + + // Verify field-level tags + assertNotNull( + searchIndex.getFields(), "Fields should not be null when fields includes fields"); + + SearchIndexField field1 = + searchIndex.getFields().stream() + .filter(f -> f.getName().startsWith("field1_")) + .findFirst() + .orElseThrow(() -> new AssertionError("Should find field1 field")); + + assertNotNull( + field1.getTags(), "Field tags should not be null when fields=fields,tags in pagination"); + assertTrue(field1.getTags().size() >= 1, "Field should have at least one tag"); + boolean hasExpectedTag = + field1.getTags().stream() + .anyMatch(tag -> tag.getTagFQN().equals(fieldTagLabel.getTagFQN())); + assertTrue( + hasExpectedTag, "Field should have the expected tag: " + fieldTagLabel.getTagFQN()); + + SearchIndexField field2 = + searchIndex.getFields().stream() + .filter(f -> f.getName().startsWith("field2_")) + .findFirst() + .orElseThrow(() -> new AssertionError("Should find field2 field")); + + assertTrue( + field2.getTags() == null || field2.getTags().isEmpty(), "field2 should not have tags"); + } + } } diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/storages/ContainerResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/storages/ContainerResourceTest.java index b45007e53e4..e81b3ba937c 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/storages/ContainerResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/storages/ContainerResourceTest.java @@ -6,6 +6,8 @@ import static jakarta.ws.rs.core.Response.Status.NOT_FOUND; import static jakarta.ws.rs.core.Response.Status.OK; import static java.util.Collections.singletonList; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.openmetadata.common.utils.CommonUtil.listOf; @@ -43,6 +45,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.apache.http.client.HttpResponseException; import org.junit.jupiter.api.MethodOrderer; @@ -863,4 +866,125 @@ public class ContainerResourceTest extends EntityResourceTest assertEquals(c1.name(), c2.name())); } + + @Test + @Order(2) + void test_paginationFetchesTagsAtBothEntityAndFieldLevels(TestInfo test) throws IOException { + // Use existing tags that are already set up in the test environment + TagLabel containerTagLabel = USER_ADDRESS_TAG_LABEL; + TagLabel columnTagLabel = GLOSSARY1_TERM1_LABEL; + + // Create multiple containers with tags at both container and column levels + List createdContainers = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + List columns = + Arrays.asList( + getColumn("col1_" + i, BIGINT, null).withTags(List.of(columnTagLabel)), + getColumn("col2_" + i, ColumnDataType.VARCHAR, null).withDataLength(50)); + + ContainerDataModel dataModel = + new ContainerDataModel().withIsPartitioned(false).withColumns(columns); + + CreateContainer createContainer = + createRequest(test.getDisplayName() + "_pagination_" + i) + .withDataModel(dataModel) + .withTags(List.of(containerTagLabel)); + + Container container = createEntity(createContainer, ADMIN_AUTH_HEADERS); + createdContainers.add(container); + } + + // Test pagination with fields=tags (should fetch container-level tags only) + WebTarget target = + getResource("containers").queryParam("fields", "tags").queryParam("limit", "50"); + + ContainerList containerList = TestUtils.get(target, ContainerList.class, ADMIN_AUTH_HEADERS); + assertNotNull(containerList.getData()); + + // Verify at least one of our created containers is in the response + List ourContainers = + containerList.getData().stream() + .filter(c -> createdContainers.stream().anyMatch(cc -> cc.getId().equals(c.getId()))) + .collect(Collectors.toList()); + + assertFalse( + ourContainers.isEmpty(), + "Should find at least one of our created containers in pagination"); + + // Verify container-level tags are fetched + for (Container container : ourContainers) { + assertNotNull( + container.getTags(), + "Container-level tags should not be null when fields=tags in pagination"); + assertEquals(1, container.getTags().size(), "Should have exactly one container-level tag"); + assertEquals(containerTagLabel.getTagFQN(), container.getTags().get(0).getTagFQN()); + + // Columns should not have tags when only fields=tags is specified + if (container.getDataModel() != null && container.getDataModel().getColumns() != null) { + for (Column col : container.getDataModel().getColumns()) { + assertTrue( + col.getTags() == null || col.getTags().isEmpty(), + "Column tags should not be populated when only fields=tags is specified in pagination"); + } + } + } + + // Test pagination with fields=dataModel,tags (should fetch both container and column tags) + target = + getResource("containers").queryParam("fields", "dataModel,tags").queryParam("limit", "50"); + + containerList = TestUtils.get(target, ContainerList.class, ADMIN_AUTH_HEADERS); + assertNotNull(containerList.getData()); + + // Verify at least one of our created containers is in the response + ourContainers = + containerList.getData().stream() + .filter(c -> createdContainers.stream().anyMatch(cc -> cc.getId().equals(c.getId()))) + .collect(Collectors.toList()); + + assertFalse( + ourContainers.isEmpty(), + "Should find at least one of our created containers in pagination"); + + // Verify both container-level and column-level tags are fetched + for (Container container : ourContainers) { + // Verify container-level tags + assertNotNull( + container.getTags(), + "Container-level tags should not be null in pagination with dataModel,tags"); + assertEquals(1, container.getTags().size(), "Should have exactly one container-level tag"); + assertEquals(containerTagLabel.getTagFQN(), container.getTags().get(0).getTagFQN()); + + // Verify column-level tags + assertNotNull( + container.getDataModel(), "DataModel should not be null when fields includes dataModel"); + assertNotNull(container.getDataModel().getColumns(), "Columns should not be null"); + + Column col1 = + container.getDataModel().getColumns().stream() + .filter(c -> c.getName().startsWith("col1_")) + .findFirst() + .orElseThrow(() -> new AssertionError("Should find col1 column")); + + assertNotNull( + col1.getTags(), + "Column tags should not be null when fields=dataModel,tags in pagination"); + assertTrue(col1.getTags().size() >= 1, "Column should have at least one tag"); + // Check that our expected tag is present + boolean hasExpectedTag = + col1.getTags().stream() + .anyMatch(tag -> tag.getTagFQN().equals(columnTagLabel.getTagFQN())); + assertTrue( + hasExpectedTag, "Column should have the expected tag: " + columnTagLabel.getTagFQN()); + + // col2 should not have tags + Column col2 = + container.getDataModel().getColumns().stream() + .filter(c -> c.getName().startsWith("col2_")) + .findFirst() + .orElseThrow(() -> new AssertionError("Should find col2 column")); + + assertTrue(col2.getTags() == null || col2.getTags().isEmpty(), "col2 should not have tags"); + } + } } diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/topics/TopicResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/topics/TopicResourceTest.java index 26d07a8fd87..31b812d8225 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/topics/TopicResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/topics/TopicResourceTest.java @@ -18,6 +18,7 @@ import static jakarta.ws.rs.core.Response.Status.NOT_FOUND; import static jakarta.ws.rs.core.Response.Status.OK; import static java.util.Collections.singletonList; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.openmetadata.common.utils.CommonUtil.listOf; @@ -43,6 +44,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.apache.http.client.HttpResponseException; import org.junit.jupiter.api.Test; @@ -621,4 +623,117 @@ public class TopicResourceTest extends EntityResourceTest { // Check the nested columns assertFields(expectedField.getChildren(), actualField.getChildren()); } + + @Test + void test_paginationFetchesTagsAtBothEntityAndFieldLevels(TestInfo test) throws IOException { + // Use existing tags that are already set up in the test environment + TagLabel topicTagLabel = USER_ADDRESS_TAG_LABEL; + TagLabel fieldTagLabel = PERSONAL_DATA_TAG_LABEL; + + // Create multiple topics with tags at both topic and field levels + List createdTopics = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + List schemaFields = + Arrays.asList( + getField("field1_" + i, FieldDataType.STRING, fieldTagLabel), + getField("field2_" + i, FieldDataType.STRING, null)); + + MessageSchema messageSchema = + new MessageSchema().withSchemaType(SchemaType.Avro).withSchemaFields(schemaFields); + + CreateTopic createTopic = + createRequest(test.getDisplayName() + "_pagination_" + i) + .withMessageSchema(messageSchema) + .withTags(List.of(topicTagLabel)); + + Topic topic = createEntity(createTopic, ADMIN_AUTH_HEADERS); + createdTopics.add(topic); + } + + // Test pagination with fields=tags (should fetch topic-level tags only) + WebTarget target = getResource("topics").queryParam("fields", "tags").queryParam("limit", "10"); + + TopicList topicList = TestUtils.get(target, TopicList.class, ADMIN_AUTH_HEADERS); + assertNotNull(topicList.getData()); + + // Verify at least one of our created topics is in the response + List ourTopics = + topicList.getData().stream() + .filter(t -> createdTopics.stream().anyMatch(ct -> ct.getId().equals(t.getId()))) + .collect(Collectors.toList()); + + assertFalse( + ourTopics.isEmpty(), "Should find at least one of our created topics in pagination"); + + // Verify topic-level tags are fetched + for (Topic topic : ourTopics) { + assertNotNull( + topic.getTags(), "Topic-level tags should not be null when fields=tags in pagination"); + assertEquals(1, topic.getTags().size(), "Should have exactly one topic-level tag"); + assertEquals(topicTagLabel.getTagFQN(), topic.getTags().get(0).getTagFQN()); + + // Fields should not have tags when only fields=tags is specified + if (topic.getMessageSchema() != null && topic.getMessageSchema().getSchemaFields() != null) { + for (Field field : topic.getMessageSchema().getSchemaFields()) { + assertTrue( + field.getTags() == null || field.getTags().isEmpty(), + "Field tags should not be populated when only fields=tags is specified in pagination"); + } + } + } + + // Test pagination with fields=messageSchema,tags (should fetch both topic and field tags) + target = + getResource("topics").queryParam("fields", "messageSchema,tags").queryParam("limit", "10"); + + topicList = TestUtils.get(target, TopicList.class, ADMIN_AUTH_HEADERS); + assertNotNull(topicList.getData()); + + // Verify at least one of our created topics is in the response + ourTopics = + topicList.getData().stream() + .filter(t -> createdTopics.stream().anyMatch(ct -> ct.getId().equals(t.getId()))) + .collect(Collectors.toList()); + + assertFalse( + ourTopics.isEmpty(), "Should find at least one of our created topics in pagination"); + + // Verify both topic-level and field-level tags are fetched + for (Topic topic : ourTopics) { + // Verify topic-level tags + assertNotNull( + topic.getTags(), + "Topic-level tags should not be null in pagination with messageSchema,tags"); + assertEquals(1, topic.getTags().size(), "Should have exactly one topic-level tag"); + assertEquals(topicTagLabel.getTagFQN(), topic.getTags().get(0).getTagFQN()); + + // Verify field-level tags + assertNotNull( + topic.getMessageSchema(), + "MessageSchema should not be null when fields includes messageSchema"); + assertNotNull(topic.getMessageSchema().getSchemaFields(), "Schema fields should not be null"); + + Field field1 = + topic.getMessageSchema().getSchemaFields().stream() + .filter(f -> f.getName().startsWith("field1_")) + .findFirst() + .orElseThrow(() -> new AssertionError("Should find field1 field")); + + assertNotNull( + field1.getTags(), + "Field tags should not be null when fields=messageSchema,tags in pagination"); + assertEquals(1, field1.getTags().size(), "Field should have exactly one tag"); + assertEquals(fieldTagLabel.getTagFQN(), field1.getTags().get(0).getTagFQN()); + + // field2 should not have tags + Field field2 = + topic.getMessageSchema().getSchemaFields().stream() + .filter(f -> f.getName().startsWith("field2_")) + .findFirst() + .orElseThrow(() -> new AssertionError("Should find field2 field")); + + assertTrue( + field2.getTags() == null || field2.getTags().isEmpty(), "field2 should not have tags"); + } + } }