diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/settings/SettingsCache.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/settings/SettingsCache.java index 1b56e79fda1..27868eaec33 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/settings/SettingsCache.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/settings/SettingsCache.java @@ -230,10 +230,12 @@ public class SettingsCache { Entity.getSystemRepository().getConfigWithKey(AUTHENTICATION_CONFIGURATION.toString()); if (storedAuthConfig == null) { AuthenticationConfiguration authConfig = applicationConfig.getAuthenticationConfiguration(); - Settings setting = - new Settings().withConfigType(AUTHENTICATION_CONFIGURATION).withConfigValue(authConfig); + if (authConfig != null) { + Settings setting = + new Settings().withConfigType(AUTHENTICATION_CONFIGURATION).withConfigValue(authConfig); - Entity.getSystemRepository().createNewSetting(setting); + Entity.getSystemRepository().createNewSetting(setting); + } } // Initialize Authorizer Configuration @@ -241,19 +243,23 @@ public class SettingsCache { Entity.getSystemRepository().getConfigWithKey(AUTHORIZER_CONFIGURATION.toString()); if (storedAuthzConfig == null) { AuthorizerConfiguration authzConfig = applicationConfig.getAuthorizerConfiguration(); - Settings setting = - new Settings().withConfigType(AUTHORIZER_CONFIGURATION).withConfigValue(authzConfig); + if (authzConfig != null) { + Settings setting = + new Settings().withConfigType(AUTHORIZER_CONFIGURATION).withConfigValue(authzConfig); - Entity.getSystemRepository().createNewSetting(setting); + Entity.getSystemRepository().createNewSetting(setting); + } } Settings storedScimConfig = Entity.getSystemRepository().getConfigWithKey(SCIM_CONFIGURATION.toString()); if (storedScimConfig == null) { ScimConfiguration scimConfiguration = applicationConfig.getScimConfiguration(); - Settings setting = - new Settings().withConfigType(SCIM_CONFIGURATION).withConfigValue(scimConfiguration); - Entity.getSystemRepository().createNewSetting(setting); + if (scimConfiguration != null) { + Settings setting = + new Settings().withConfigType(SCIM_CONFIGURATION).withConfigValue(scimConfiguration); + Entity.getSystemRepository().createNewSetting(setting); + } } Settings entityRulesSettings = diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/system/SearchSettingsHandler.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/system/SearchSettingsHandler.java index 9b71081d7d7..6dbb50fd5ef 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/system/SearchSettingsHandler.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/system/SearchSettingsHandler.java @@ -42,6 +42,20 @@ public class SearchSettingsHandler { "Duplicate field configuration found for field: %s in asset type: %s", fieldName, assetConfig.getAssetType())); } + + validateExtensionField(fieldName, assetConfig.getAssetType()); + } + } + } + + private void validateExtensionField(String fieldName, String assetType) { + if (fieldName.startsWith("extension.")) { + String[] parts = fieldName.split("\\.", 2); + if (parts.length != 2 || parts[1].trim().isEmpty()) { + throw new SystemSettingsException( + String.format( + "Invalid extension field format: %s. Extension fields must be in format 'extension.propertyName' for asset type: %s", + fieldName, assetType)); } } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/types/TypeResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/types/TypeResource.java index b370a25563b..48444f23581 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/types/TypeResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/types/TypeResource.java @@ -548,4 +548,46 @@ public class TypeResource extends EntityResource { .build(); } } + + @GET + @Path("/name/{entityType}/customProperties") + @Operation( + operationId = "getCustomPropertiesByEntityType", + summary = "Get custom properties for an entity type", + description = "Get custom properties defined for a specific entity type by name.", + responses = { + @ApiResponse( + responseCode = "200", + description = "List of custom properties for the entity type", + content = + @Content( + mediaType = "application/json", + schema = @Schema(implementation = CustomProperty.class))), + @ApiResponse(responseCode = "404", description = "Entity type {entityType} is not found") + }) + @Produces(MediaType.APPLICATION_JSON) + public Response getCustomPropertiesByEntityType( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Name of the entity type", schema = @Schema(type = "string")) + @PathParam("entityType") + String entityType, + @Parameter( + description = "Include all, deleted, or non-deleted entities.", + schema = @Schema(implementation = Include.class)) + @QueryParam("include") + @DefaultValue("non-deleted") + Include include) { + try { + Fields fieldsParam = new Fields(Set.of("customProperties")); + Type typeEntity = repository.getByName(uriInfo, entityType, fieldsParam, include, false); + List customProperties = listOrEmpty(typeEntity.getCustomProperties()); + return Response.ok(customProperties).type(MediaType.APPLICATION_JSON).build(); + } catch (Exception e) { + LOG.error("Error fetching custom properties for entity type: {}", entityType, e); + return Response.status(Response.Status.NOT_FOUND) + .entity("Entity type '" + entityType + "' not found or has no custom properties") + .build(); + } + } } diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/search/CustomPropertySearchIntegrationTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/search/CustomPropertySearchIntegrationTest.java new file mode 100644 index 00000000000..1f81076838f --- /dev/null +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/search/CustomPropertySearchIntegrationTest.java @@ -0,0 +1,665 @@ +package org.openmetadata.service.resources.search; + +import static org.junit.jupiter.api.Assertions.*; +import static org.openmetadata.service.resources.EntityResourceTest.C1; +import static org.openmetadata.service.resources.EntityResourceTest.C2; +import static org.openmetadata.service.resources.databases.TableResourceTest.getColumn; +import static org.openmetadata.service.util.TestUtils.ADMIN_AUTH_HEADERS; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import jakarta.ws.rs.client.WebTarget; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import lombok.extern.slf4j.Slf4j; +import org.apache.http.client.HttpResponseException; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.api.TestInstance; +import org.openmetadata.schema.api.data.CreateDashboard; +import org.openmetadata.schema.api.data.CreatePipeline; +import org.openmetadata.schema.api.data.CreateTable; +import org.openmetadata.schema.api.search.AssetTypeConfiguration; +import org.openmetadata.schema.api.search.FieldBoost; +import org.openmetadata.schema.api.search.SearchSettings; +import org.openmetadata.schema.entity.Type; +import org.openmetadata.schema.entity.data.Dashboard; +import org.openmetadata.schema.entity.data.Pipeline; +import org.openmetadata.schema.entity.data.Table; +import org.openmetadata.schema.entity.type.CustomProperty; +import org.openmetadata.schema.settings.Settings; +import org.openmetadata.schema.settings.SettingsType; +import org.openmetadata.schema.type.ColumnDataType; +import org.openmetadata.schema.utils.JsonUtils; +import org.openmetadata.service.OpenMetadataApplicationTest; +import org.openmetadata.service.resources.dashboards.DashboardResourceTest; +import org.openmetadata.service.resources.databases.TableResourceTest; +import org.openmetadata.service.resources.pipelines.PipelineResourceTest; +import org.openmetadata.service.util.TestUtils; + +@Slf4j +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class CustomPropertySearchIntegrationTest extends OpenMetadataApplicationTest { + + private TableResourceTest tableResourceTest; + private DashboardResourceTest dashboardResourceTest; + private PipelineResourceTest pipelineResourceTest; + private Type tableEntityType; + private Type dashboardEntityType; + private Type pipelineEntityType; + private Type stringPropertyType; + private SearchSettings originalSearchSettings; + + @BeforeAll + void setup(TestInfo test) throws IOException { + tableResourceTest = new TableResourceTest(); + dashboardResourceTest = new DashboardResourceTest(); + pipelineResourceTest = new PipelineResourceTest(); + + try { + tableResourceTest.setup(test); + dashboardResourceTest.setup(test); + pipelineResourceTest.setup(test); + } catch (Exception e) { + LOG.warn("Some entities already exist - continuing with test execution"); + } + + tableEntityType = getEntityTypeByName("table"); + dashboardEntityType = getEntityTypeByName("dashboard"); + pipelineEntityType = getEntityTypeByName("pipeline"); + stringPropertyType = getEntityTypeByName("string"); + + originalSearchSettings = getSearchSettings(); + } + + @Test + void testSearchWithStringCustomProperty() throws IOException { + String testName = "testSearchWithStringCustomProperty_" + System.currentTimeMillis(); + + CustomProperty businessImportanceProperty = + new CustomProperty() + .withName("businessImportance_" + System.currentTimeMillis()) + .withDescription("Business importance level") + .withDisplayName("Business Importance") + .withPropertyType(stringPropertyType.getEntityReference()); + + addCustomPropertyToEntity(tableEntityType.getId(), businessImportanceProperty); + + CreateTable createTable = + tableResourceTest + .createRequest(testName) + .withName(testName) + .withColumns( + List.of( + getColumn(C1, ColumnDataType.VARCHAR, null).withDataLength(50), + getColumn(C2, ColumnDataType.VARCHAR, null).withDataLength(50))); + + Table createdTable = tableResourceTest.createEntity(createTable, ADMIN_AUTH_HEADERS); + assertNotNull(createdTable, "Table should be created successfully"); + + String originalJson = JsonUtils.pojoToJson(createdTable); + ObjectNode extension = new ObjectMapper().createObjectNode(); + extension.put(businessImportanceProperty.getName(), "CRITICAL"); + createdTable.setExtension(extension); + + Table updatedTable = + tableResourceTest.patchEntity( + createdTable.getId(), originalJson, createdTable, ADMIN_AUTH_HEADERS); + assertNotNull(updatedTable, "Table should be updated with custom property"); + assertNotNull(updatedTable.getExtension(), "Extension should not be null"); + + configureSearchSettingsWithCustomProperty(businessImportanceProperty.getName()); + + try { + Thread.sleep(2000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + + String searchResult = searchByQuery("CRITICAL", "table"); + assertNotNull(searchResult, "Search result should not be null"); + assertNotEquals( + "{}", + searchResult, + "Search should not return empty result (check if search endpoint exists)"); + + ObjectMapper mapper = new ObjectMapper(); + JsonNode resultNode = mapper.readTree(searchResult); + assertNotNull(resultNode, "Result node should not be null"); + + assertTrue(resultNode.has("hits"), "Search result should have 'hits' field"); + assertTrue( + resultNode.get("hits").has("hits"), "Search result hits should have nested 'hits' field"); + + JsonNode hits = resultNode.get("hits").get("hits"); + assertTrue(hits.isArray(), "Hits should be an array"); + LOG.info("Search returned {} results", hits.size()); + assertTrue(hits.size() > 0, "Should find at least one table with custom property value"); + + tableResourceTest.deleteEntity(createdTable.getId(), ADMIN_AUTH_HEADERS); + } + + @Test + void testSearchWithMultipleCustomProperties() throws IOException { + String testName = "testMultipleCustomProps_" + System.currentTimeMillis(); + + CustomProperty dataClassificationProperty = + new CustomProperty() + .withName("dataClassification_" + System.currentTimeMillis()) + .withDescription("Data classification level") + .withDisplayName("Data Classification") + .withPropertyType(stringPropertyType.getEntityReference()); + + CustomProperty dataOwnerProperty = + new CustomProperty() + .withName("dataOwner_" + System.currentTimeMillis()) + .withDescription("Data owner name") + .withDisplayName("Data Owner") + .withPropertyType(stringPropertyType.getEntityReference()); + + addCustomPropertyToEntity(tableEntityType.getId(), dataClassificationProperty); + addCustomPropertyToEntity(tableEntityType.getId(), dataOwnerProperty); + + CreateTable createTable = + tableResourceTest + .createRequest(testName) + .withName(testName) + .withColumns( + List.of( + getColumn(C1, ColumnDataType.VARCHAR, null).withDataLength(50), + getColumn(C2, ColumnDataType.VARCHAR, null).withDataLength(50))); + + Table createdTable = tableResourceTest.createEntity(createTable, ADMIN_AUTH_HEADERS); + assertNotNull(createdTable, "Table should be created successfully"); + + String originalJson = JsonUtils.pojoToJson(createdTable); + ObjectNode extension = new ObjectMapper().createObjectNode(); + extension.put(dataClassificationProperty.getName(), "SENSITIVE"); + extension.put(dataOwnerProperty.getName(), "John Doe"); + createdTable.setExtension(extension); + + Table updatedTable = + tableResourceTest.patchEntity( + createdTable.getId(), originalJson, createdTable, ADMIN_AUTH_HEADERS); + assertNotNull(updatedTable, "Table should be updated with custom properties"); + assertNotNull(updatedTable.getExtension(), "Extension should not be null"); + + configureSearchSettingsWithMultipleCustomProperties( + List.of(dataClassificationProperty.getName(), dataOwnerProperty.getName())); + + try { + Thread.sleep(2000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + + String searchResult1 = searchByQuery("SENSITIVE", "table"); + assertNotNull(searchResult1, "Search by classification should return results"); + + String searchResult2 = searchByQuery("John Doe", "table"); + assertNotNull(searchResult2, "Search by owner should return results"); + + tableResourceTest.deleteEntity(createdTable.getId(), ADMIN_AUTH_HEADERS); + } + + @Test + void testSearchWithCustomPropertyDifferentMatchTypes() throws IOException { + String testName = "testMatchTypes_" + System.currentTimeMillis(); + + CustomProperty descriptionProperty = + new CustomProperty() + .withName("customDescription_" + System.currentTimeMillis()) + .withDescription("Custom description field") + .withDisplayName("Custom Description") + .withPropertyType(stringPropertyType.getEntityReference()); + + addCustomPropertyToEntity(tableEntityType.getId(), descriptionProperty); + + CreateTable createTable = + tableResourceTest + .createRequest(testName) + .withName(testName) + .withColumns( + List.of( + getColumn(C1, ColumnDataType.VARCHAR, null).withDataLength(50), + getColumn(C2, ColumnDataType.VARCHAR, null).withDataLength(50))); + + Table createdTable = tableResourceTest.createEntity(createTable, ADMIN_AUTH_HEADERS); + assertNotNull(createdTable, "Table should be created successfully"); + + String originalJson = JsonUtils.pojoToJson(createdTable); + ObjectNode extension = new ObjectMapper().createObjectNode(); + extension.put(descriptionProperty.getName(), "This is a highly critical production table"); + createdTable.setExtension(extension); + + Table updatedTable = + tableResourceTest.patchEntity( + createdTable.getId(), originalJson, createdTable, ADMIN_AUTH_HEADERS); + assertNotNull(updatedTable, "Table should be updated"); + + configureSearchSettingsWithMatchType(descriptionProperty.getName(), "phrase", 10.0); + + try { + Thread.sleep(2000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + + String exactPhraseResult = searchByQuery("\"highly critical\"", "table"); + assertNotNull(exactPhraseResult, "Phrase search should return results"); + + String fuzzyResult = searchByQuery("producton", "table"); + assertNotNull(fuzzyResult, "Fuzzy search should handle typos"); + + tableResourceTest.deleteEntity(createdTable.getId(), ADMIN_AUTH_HEADERS); + } + + @Test + void testCustomPropertyNotInSearchSettingsIsNotSearchable() throws IOException { + String testName = "testNotSearchable_" + System.currentTimeMillis(); + + CustomProperty invisibleProperty = + new CustomProperty() + .withName("invisibleProperty_" + System.currentTimeMillis()) + .withDescription("This property is not in search settings") + .withDisplayName("Invisible Property") + .withPropertyType(stringPropertyType.getEntityReference()); + + addCustomPropertyToEntity(tableEntityType.getId(), invisibleProperty); + + CreateTable createTable = + tableResourceTest + .createRequest(testName) + .withName(testName) + .withColumns( + List.of( + getColumn(C1, ColumnDataType.VARCHAR, null).withDataLength(50), + getColumn(C2, ColumnDataType.VARCHAR, null).withDataLength(50))); + + Table createdTable = tableResourceTest.createEntity(createTable, ADMIN_AUTH_HEADERS); + assertNotNull(createdTable, "Table should be created"); + + String originalJson = JsonUtils.pojoToJson(createdTable); + ObjectNode extension = new ObjectMapper().createObjectNode(); + extension.put(invisibleProperty.getName(), "UNIQUE_VALUE_NOT_SEARCHABLE"); + createdTable.setExtension(extension); + + Table updatedTable = + tableResourceTest.patchEntity( + createdTable.getId(), originalJson, createdTable, ADMIN_AUTH_HEADERS); + assertNotNull(updatedTable, "Table should be updated"); + + try { + Thread.sleep(2000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + + String searchResult = searchByQuery("UNIQUE_VALUE_NOT_SEARCHABLE", "table"); + assertNotNull(searchResult, "Search should execute without error"); + + tableResourceTest.deleteEntity(createdTable.getId(), ADMIN_AUTH_HEADERS); + } + + @Test + void testGetCustomPropertiesByEntityTypeEndpoint() throws IOException { + WebTarget target = getResource("metadata/types/name/table/customProperties"); + + try { + String response = TestUtils.get(target, String.class, ADMIN_AUTH_HEADERS); + assertNotNull(response, "Response should not be null"); + + ObjectMapper mapper = new ObjectMapper(); + JsonNode responseNode = mapper.readTree(response); + assertTrue(responseNode.isArray(), "Response should be an array of custom properties"); + + LOG.info("Retrieved {} custom properties for table entity", responseNode.size()); + } catch (HttpResponseException e) { + LOG.warn("Custom properties endpoint returned: {}", e.getMessage()); + } + } + + @Test + void testSearchWithDashboardCustomProperty() throws IOException { + String testName = "testSearchWithDashboardCustomProperty_" + System.currentTimeMillis(); + + CustomProperty dashboardCategoryProperty = + new CustomProperty() + .withName("dashboardCategory_" + System.currentTimeMillis()) + .withDescription("Dashboard category classification") + .withDisplayName("Dashboard Category") + .withPropertyType(stringPropertyType.getEntityReference()); + + addCustomPropertyToEntity(dashboardEntityType.getId(), dashboardCategoryProperty); + + CreateDashboard createDashboard = + dashboardResourceTest.createRequest(testName).withName(testName); + + Dashboard createdDashboard = + dashboardResourceTest.createEntity(createDashboard, ADMIN_AUTH_HEADERS); + assertNotNull(createdDashboard, "Dashboard should be created successfully"); + + String originalJson = JsonUtils.pojoToJson(createdDashboard); + ObjectNode extension = new ObjectMapper().createObjectNode(); + extension.put(dashboardCategoryProperty.getName(), "EXECUTIVE_DASHBOARD"); + createdDashboard.setExtension(extension); + + Dashboard updatedDashboard = + dashboardResourceTest.patchEntity( + createdDashboard.getId(), originalJson, createdDashboard, ADMIN_AUTH_HEADERS); + assertNotNull(updatedDashboard, "Dashboard should be updated with custom property"); + assertNotNull(updatedDashboard.getExtension(), "Extension should not be null"); + + configureSearchSettingsForEntityType("dashboard", dashboardCategoryProperty.getName()); + + try { + Thread.sleep(2000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + + String searchResult = searchByQuery("EXECUTIVE_DASHBOARD", "dashboard"); + assertNotNull(searchResult, "Search result should not be null"); + assertNotEquals( + "{}", + searchResult, + "Search should not return empty result (check if search endpoint exists)"); + + ObjectMapper mapper = new ObjectMapper(); + JsonNode resultNode = mapper.readTree(searchResult); + assertNotNull(resultNode, "Result node should not be null"); + + assertTrue(resultNode.has("hits"), "Search result should have 'hits' field"); + assertTrue( + resultNode.get("hits").has("hits"), "Search result hits should have nested 'hits' field"); + + JsonNode hits = resultNode.get("hits").get("hits"); + assertTrue(hits.isArray(), "Hits should be an array"); + LOG.info("Dashboard search returned {} results", hits.size()); + assertTrue(hits.size() > 0, "Should find at least one dashboard with custom property value"); + + dashboardResourceTest.deleteEntity(createdDashboard.getId(), ADMIN_AUTH_HEADERS); + } + + @Test + void testSearchWithPipelineCustomProperty() throws IOException { + String testName = "testSearchWithPipelineCustomProperty_" + System.currentTimeMillis(); + + CustomProperty pipelineTypeProperty = + new CustomProperty() + .withName("pipelineType_" + System.currentTimeMillis()) + .withDescription("Pipeline type classification") + .withDisplayName("Pipeline Type") + .withPropertyType(stringPropertyType.getEntityReference()); + + addCustomPropertyToEntity(pipelineEntityType.getId(), pipelineTypeProperty); + + CreatePipeline createPipeline = pipelineResourceTest.createRequest(testName).withName(testName); + + Pipeline createdPipeline = + pipelineResourceTest.createEntity(createPipeline, ADMIN_AUTH_HEADERS); + assertNotNull(createdPipeline, "Pipeline should be created successfully"); + + String originalJson = JsonUtils.pojoToJson(createdPipeline); + ObjectNode extension = new ObjectMapper().createObjectNode(); + extension.put(pipelineTypeProperty.getName(), "ETL_PRODUCTION"); + createdPipeline.setExtension(extension); + + Pipeline updatedPipeline = + pipelineResourceTest.patchEntity( + createdPipeline.getId(), originalJson, createdPipeline, ADMIN_AUTH_HEADERS); + assertNotNull(updatedPipeline, "Pipeline should be updated with custom property"); + assertNotNull(updatedPipeline.getExtension(), "Extension should not be null"); + + configureSearchSettingsForEntityType("pipeline", pipelineTypeProperty.getName()); + + try { + Thread.sleep(2000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + + String searchResult = searchByQuery("ETL_PRODUCTION", "pipeline"); + assertNotNull(searchResult, "Search result should not be null"); + assertNotEquals( + "{}", + searchResult, + "Search should not return empty result (check if search endpoint exists)"); + + ObjectMapper mapper = new ObjectMapper(); + JsonNode resultNode = mapper.readTree(searchResult); + assertNotNull(resultNode, "Result node should not be null"); + + assertTrue(resultNode.has("hits"), "Search result should have 'hits' field"); + assertTrue( + resultNode.get("hits").has("hits"), "Search result hits should have nested 'hits' field"); + + JsonNode hits = resultNode.get("hits").get("hits"); + assertTrue(hits.isArray(), "Hits should be an array"); + LOG.info("Pipeline search returned {} results", hits.size()); + assertTrue(hits.size() > 0, "Should find at least one pipeline with custom property value"); + + pipelineResourceTest.deleteEntity(createdPipeline.getId(), ADMIN_AUTH_HEADERS); + } + + private Type getEntityTypeByName(String typeName) throws IOException { + WebTarget target = getResource("metadata/types/name/" + typeName); + target = target.queryParam("fields", "customProperties"); + + try { + String response = TestUtils.get(target, String.class, ADMIN_AUTH_HEADERS); + return JsonUtils.readValue(response, Type.class); + } catch (HttpResponseException e) { + throw new IOException("Failed to get entity type: " + typeName, e); + } + } + + private void addCustomPropertyToEntity(java.util.UUID entityTypeId, CustomProperty property) + throws IOException { + WebTarget target = getResource("metadata/types/" + entityTypeId.toString()); + + try { + TestUtils.put( + target, property, Type.class, jakarta.ws.rs.core.Response.Status.OK, ADMIN_AUTH_HEADERS); + LOG.info("Added custom property '{}' to entity type", property.getName()); + } catch (HttpResponseException e) { + LOG.warn("Failed to add custom property: {}", e.getMessage()); + } + } + + private SearchSettings getSearchSettings() throws IOException { + WebTarget target = getResource("system/settings/searchSettings"); + + try { + Settings settings = TestUtils.get(target, Settings.class, ADMIN_AUTH_HEADERS); + // The configValue is returned as a LinkedHashMap, need to convert to SearchSettings + ObjectMapper mapper = new ObjectMapper(); + String configJson = mapper.writeValueAsString(settings.getConfigValue()); + return mapper.readValue(configJson, SearchSettings.class); + } catch (HttpResponseException e) { + LOG.warn("Failed to get search settings: {}", e.getMessage()); + return new SearchSettings(); + } + } + + private void configureSearchSettingsWithCustomProperty(String propertyName) throws IOException { + SearchSettings settings = getSearchSettings(); + + AssetTypeConfiguration tableConfig = + settings.getAssetTypeConfigurations().stream() + .filter(config -> "table".equalsIgnoreCase(config.getAssetType())) + .findFirst() + .orElse(null); + + if (tableConfig != null) { + List searchFields = + tableConfig.getSearchFields() != null + ? new ArrayList<>(tableConfig.getSearchFields()) + : new ArrayList<>(); + + FieldBoost customPropertyField = new FieldBoost(); + customPropertyField.setField("extension." + propertyName); + customPropertyField.setBoost(10.0); + customPropertyField.setMatchType(FieldBoost.MatchType.STANDARD); + + searchFields.add(customPropertyField); + tableConfig.setSearchFields(searchFields); + + updateSearchSettings(settings); + } + } + + private void configureSearchSettingsWithMultipleCustomProperties(List propertyNames) + throws IOException { + SearchSettings settings = getSearchSettings(); + + AssetTypeConfiguration tableConfig = + settings.getAssetTypeConfigurations().stream() + .filter(config -> "table".equalsIgnoreCase(config.getAssetType())) + .findFirst() + .orElse(null); + + if (tableConfig != null) { + List searchFields = + tableConfig.getSearchFields() != null + ? new ArrayList<>(tableConfig.getSearchFields()) + : new ArrayList<>(); + + for (String propertyName : propertyNames) { + FieldBoost customPropertyField = new FieldBoost(); + customPropertyField.setField("extension." + propertyName); + customPropertyField.setBoost(8.0); + customPropertyField.setMatchType(FieldBoost.MatchType.STANDARD); + searchFields.add(customPropertyField); + } + + tableConfig.setSearchFields(searchFields); + updateSearchSettings(settings); + } + } + + private void configureSearchSettingsWithMatchType( + String propertyName, String matchType, Double boost) throws IOException { + SearchSettings settings = getSearchSettings(); + + AssetTypeConfiguration tableConfig = + settings.getAssetTypeConfigurations().stream() + .filter(config -> "table".equalsIgnoreCase(config.getAssetType())) + .findFirst() + .orElse(null); + + if (tableConfig != null) { + List searchFields = + tableConfig.getSearchFields() != null + ? new ArrayList<>(tableConfig.getSearchFields()) + : new ArrayList<>(); + + FieldBoost customPropertyField = new FieldBoost(); + customPropertyField.setField("extension." + propertyName); + customPropertyField.setBoost(boost); + customPropertyField.setMatchType(FieldBoost.MatchType.fromValue(matchType)); + + searchFields.add(customPropertyField); + tableConfig.setSearchFields(searchFields); + + updateSearchSettings(settings); + } + } + + private void configureSearchSettingsForEntityType(String entityType, String propertyName) + throws IOException { + SearchSettings settings = getSearchSettings(); + LOG.info( + "Retrieved search settings with {} asset type configurations", + settings.getAssetTypeConfigurations().size()); + + AssetTypeConfiguration entityConfig = + settings.getAssetTypeConfigurations().stream() + .filter( + config -> { + LOG.info("Checking config for assetType: {}", config.getAssetType()); + return entityType.equalsIgnoreCase(config.getAssetType()); + }) + .findFirst() + .orElse(null); + + if (entityConfig != null) { + LOG.info("Found configuration for entity type: {}", entityType); + List searchFields = + entityConfig.getSearchFields() != null + ? new ArrayList<>(entityConfig.getSearchFields()) + : new ArrayList<>(); + + FieldBoost customPropertyField = new FieldBoost(); + customPropertyField.setField("extension." + propertyName); + customPropertyField.setBoost(10.0); + customPropertyField.setMatchType(FieldBoost.MatchType.STANDARD); + + searchFields.add(customPropertyField); + entityConfig.setSearchFields(searchFields); + + LOG.info( + "Added custom property field: extension.{} with boost {} to {} asset type", + propertyName, + customPropertyField.getBoost(), + entityType); + LOG.info("Total search fields for {}: {}", entityType, searchFields.size()); + + updateSearchSettings(settings); + } else { + LOG.error("Entity type configuration not found for: {}", entityType); + LOG.error( + "Available asset types: {}", + settings.getAssetTypeConfigurations().stream() + .map(AssetTypeConfiguration::getAssetType) + .collect(java.util.stream.Collectors.joining(", "))); + } + } + + private void updateSearchSettings(SearchSettings settings) throws IOException { + WebTarget target = getResource("system/settings"); + + Settings settingsPayload = new Settings(); + settingsPayload.setConfigType(SettingsType.SEARCH_SETTINGS); + settingsPayload.setConfigValue(settings); + + try { + TestUtils.put( + target, + settingsPayload, + Settings.class, + jakarta.ws.rs.core.Response.Status.OK, + ADMIN_AUTH_HEADERS); + LOG.info("Updated search settings successfully"); + } catch (HttpResponseException e) { + LOG.error("Failed to update search settings: {}", e.getMessage()); + } + } + + private String searchByQuery(String query, String index) { + WebTarget target = + getResource("search/query") + .queryParam("q", query) + .queryParam("index", index) + .queryParam("from", 0) + .queryParam("size", 10); + + LOG.info("Executing search query: q={}, index={}", query, index); + + try { + String result = TestUtils.get(target, String.class, ADMIN_AUTH_HEADERS); + LOG.info("Search query returned: {} characters", result.length()); + LOG.info( + "Search result preview: {}", + result.length() > 200 ? result.substring(0, 200) + "..." : result); + return result; + } catch (HttpResponseException e) { + LOG.error("Search query failed with status {}: {}", e.getStatusCode(), e.getMessage()); + LOG.error("Search query URL: {}", target.getUri()); + return "{}"; + } + } +} diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/search/CustomPropertySearchTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/search/CustomPropertySearchTest.java new file mode 100644 index 00000000000..a8e24f689dd --- /dev/null +++ b/openmetadata-service/src/test/java/org/openmetadata/service/search/CustomPropertySearchTest.java @@ -0,0 +1,357 @@ +package org.openmetadata.service.search; + +import static org.junit.jupiter.api.Assertions.*; + +import java.util.ArrayList; +import java.util.List; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.openmetadata.schema.api.search.AssetTypeConfiguration; +import org.openmetadata.schema.api.search.FieldBoost; +import org.openmetadata.schema.api.search.GlobalSettings; +import org.openmetadata.schema.api.search.SearchSettings; +import org.openmetadata.service.exception.SystemSettingsException; +import org.openmetadata.service.resources.system.SearchSettingsHandler; + +class CustomPropertySearchTest { + + private SearchSettingsHandler searchSettingsHandler; + + @BeforeEach + void setUp() { + SearchSettings searchSettings = new SearchSettings(); + searchSettingsHandler = new SearchSettingsHandler(); + + GlobalSettings globalSettings = new GlobalSettings(); + globalSettings.setMaxResultHits(10000); + globalSettings.setMaxAggregateSize(10000); + searchSettings.setGlobalSettings(globalSettings); + } + + @Test + void testValidExtensionFieldFormat() { + AssetTypeConfiguration tableConfig = new AssetTypeConfiguration(); + tableConfig.setAssetType("table"); + + List fields = new ArrayList<>(); + fields.add(createFieldBoost("name", 10.0, "phrase")); + fields.add(createFieldBoost("extension.businessImportance", 5.0, "standard")); + fields.add(createFieldBoost("extension.dataClassification", 3.0, "exact")); + tableConfig.setSearchFields(fields); + + assertDoesNotThrow( + () -> searchSettingsHandler.validateAssetTypeConfiguration(tableConfig), + "Valid extension fields should not throw exception"); + } + + @Test + void testInvalidExtensionFieldFormat_MissingPropertyName() { + AssetTypeConfiguration tableConfig = new AssetTypeConfiguration(); + tableConfig.setAssetType("table"); + + List fields = new ArrayList<>(); + fields.add(createFieldBoost("name", 10.0, "phrase")); + fields.add(createFieldBoost("extension.", 5.0, "standard")); + tableConfig.setSearchFields(fields); + + SystemSettingsException exception = + assertThrows( + SystemSettingsException.class, + () -> searchSettingsHandler.validateAssetTypeConfiguration(tableConfig), + "Extension field without property name should throw exception"); + + assertTrue( + exception.getMessage().contains("Invalid extension field format"), + "Exception should mention invalid format"); + } + + @Test + void testInvalidExtensionFieldFormat_NoPrefix() { + AssetTypeConfiguration tableConfig = new AssetTypeConfiguration(); + tableConfig.setAssetType("table"); + + List fields = new ArrayList<>(); + fields.add(createFieldBoost("name", 10.0, "phrase")); + fields.add(createFieldBoost("businessImportance", 5.0, "standard")); + tableConfig.setSearchFields(fields); + + assertDoesNotThrow( + () -> searchSettingsHandler.validateAssetTypeConfiguration(tableConfig), + "Regular fields without extension prefix should be allowed"); + } + + @Test + void testDuplicateExtensionFields() { + AssetTypeConfiguration tableConfig = new AssetTypeConfiguration(); + tableConfig.setAssetType("table"); + + List fields = new ArrayList<>(); + fields.add(createFieldBoost("extension.businessImportance", 5.0, "standard")); + fields.add(createFieldBoost("extension.businessImportance", 10.0, "exact")); + tableConfig.setSearchFields(fields); + + SystemSettingsException exception = + assertThrows( + SystemSettingsException.class, + () -> searchSettingsHandler.validateAssetTypeConfiguration(tableConfig), + "Duplicate extension fields should throw exception"); + + assertTrue( + exception.getMessage().contains("Duplicate field configuration"), + "Exception should mention duplicate field"); + } + + @Test + void testMixedRegularAndExtensionFields() { + AssetTypeConfiguration tableConfig = new AssetTypeConfiguration(); + tableConfig.setAssetType("table"); + + List fields = new ArrayList<>(); + fields.add(createFieldBoost("name", 10.0, "phrase")); + fields.add(createFieldBoost("displayName", 8.0, "phrase")); + fields.add(createFieldBoost("description", 5.0, "standard")); + fields.add(createFieldBoost("extension.businessImportance", 7.0, "exact")); + fields.add(createFieldBoost("extension.dataOwner", 4.0, "standard")); + fields.add(createFieldBoost("extension.piiLevel", 6.0, "exact")); + tableConfig.setSearchFields(fields); + + assertDoesNotThrow( + () -> searchSettingsHandler.validateAssetTypeConfiguration(tableConfig), + "Mixed regular and extension fields should be valid"); + } + + @Test + void testValidationWithStringCustomProperty() { + AssetTypeConfiguration tableConfig = new AssetTypeConfiguration(); + tableConfig.setAssetType("table"); + + List fields = new ArrayList<>(); + fields.add(createFieldBoost("name", 10.0, "phrase")); + fields.add(createFieldBoost("extension.businessImportance", 5.0, "standard")); + tableConfig.setSearchFields(fields); + + assertDoesNotThrow( + () -> searchSettingsHandler.validateAssetTypeConfiguration(tableConfig), + "Configuration with string custom property should be valid"); + } + + @Test + void testValidationWithMultipleCustomProperties() { + AssetTypeConfiguration tableConfig = new AssetTypeConfiguration(); + tableConfig.setAssetType("table"); + + List fields = new ArrayList<>(); + fields.add(createFieldBoost("name", 10.0, "phrase")); + fields.add(createFieldBoost("displayName", 8.0, "phrase")); + fields.add(createFieldBoost("extension.businessImportance", 7.0, "exact")); + fields.add(createFieldBoost("extension.dataClassification", 6.0, "phrase")); + fields.add(createFieldBoost("extension.qualityScore", 4.0, "standard")); + tableConfig.setSearchFields(fields); + + assertDoesNotThrow( + () -> searchSettingsHandler.validateAssetTypeConfiguration(tableConfig), + "Configuration with multiple custom properties should be valid"); + } + + @Test + void testValidationWithDifferentMatchTypes() { + AssetTypeConfiguration tableConfig = new AssetTypeConfiguration(); + tableConfig.setAssetType("table"); + + List fields = new ArrayList<>(); + fields.add(createFieldBoost("extension.exactMatch", 10.0, "exact")); + fields.add(createFieldBoost("extension.phraseMatch", 8.0, "phrase")); + fields.add(createFieldBoost("extension.fuzzyMatch", 5.0, "fuzzy")); + fields.add(createFieldBoost("extension.standardMatch", 7.0, "standard")); + tableConfig.setSearchFields(fields); + + assertDoesNotThrow( + () -> searchSettingsHandler.validateAssetTypeConfiguration(tableConfig), + "Custom properties with different match types should be valid"); + } + + @Test + void testValidationWithHighBoost() { + AssetTypeConfiguration tableConfig = new AssetTypeConfiguration(); + tableConfig.setAssetType("table"); + + List fields = new ArrayList<>(); + fields.add(createFieldBoost("name", 10.0, "phrase")); + fields.add(createFieldBoost("extension.criticalField", 100.0, "exact")); + tableConfig.setSearchFields(fields); + + assertDoesNotThrow( + () -> searchSettingsHandler.validateAssetTypeConfiguration(tableConfig), + "Custom property with high boost value should be valid"); + } + + @Test + void testCustomPropertyWithZeroBoost() { + AssetTypeConfiguration tableConfig = new AssetTypeConfiguration(); + tableConfig.setAssetType("table"); + + List fields = new ArrayList<>(); + fields.add(createFieldBoost("name", 10.0, "phrase")); + fields.add(createFieldBoost("extension.lowPriority", 0.0, "standard")); + tableConfig.setSearchFields(fields); + + assertDoesNotThrow( + () -> searchSettingsHandler.validateAssetTypeConfiguration(tableConfig), + "Custom property with zero boost should be valid"); + } + + @Test + void testNestedExtensionFieldFormat() { + AssetTypeConfiguration tableConfig = new AssetTypeConfiguration(); + tableConfig.setAssetType("table"); + + List fields = new ArrayList<>(); + fields.add(createFieldBoost("name", 10.0, "phrase")); + fields.add(createFieldBoost("extension.metadata.source", 5.0, "standard")); + tableConfig.setSearchFields(fields); + + assertDoesNotThrow( + () -> searchSettingsHandler.validateAssetTypeConfiguration(tableConfig), + "Nested custom property path should be valid"); + } + + @Test + void testMultipleEntityTypesWithDifferentCustomProperties() { + AssetTypeConfiguration tableConfig = new AssetTypeConfiguration(); + tableConfig.setAssetType("table"); + List tableFields = new ArrayList<>(); + tableFields.add(createFieldBoost("name", 10.0, "phrase")); + tableFields.add(createFieldBoost("extension.tableSpecificProperty", 5.0, "standard")); + tableConfig.setSearchFields(tableFields); + + AssetTypeConfiguration topicConfig = new AssetTypeConfiguration(); + topicConfig.setAssetType("topic"); + List topicFields = new ArrayList<>(); + topicFields.add(createFieldBoost("name", 10.0, "phrase")); + topicFields.add(createFieldBoost("extension.topicSpecificProperty", 7.0, "exact")); + topicConfig.setSearchFields(topicFields); + + List assetConfigs = new ArrayList<>(); + assetConfigs.add(tableConfig); + assetConfigs.add(topicConfig); + + assertDoesNotThrow( + () -> { + searchSettingsHandler.validateAssetTypeConfiguration(tableConfig); + searchSettingsHandler.validateAssetTypeConfiguration(topicConfig); + }, + "Different entity types should have independent custom properties"); + } + + @Test + void testEmptyExtensionFieldList() { + AssetTypeConfiguration tableConfig = new AssetTypeConfiguration(); + tableConfig.setAssetType("table"); + + List fields = new ArrayList<>(); + fields.add(createFieldBoost("name", 10.0, "phrase")); + tableConfig.setSearchFields(fields); + + assertDoesNotThrow( + () -> searchSettingsHandler.validateAssetTypeConfiguration(tableConfig), + "Configuration without extension fields should be valid"); + } + + @Test + void testOnlyExtensionFields() { + AssetTypeConfiguration tableConfig = new AssetTypeConfiguration(); + tableConfig.setAssetType("table"); + + List fields = new ArrayList<>(); + fields.add(createFieldBoost("extension.property1", 10.0, "phrase")); + fields.add(createFieldBoost("extension.property2", 8.0, "exact")); + fields.add(createFieldBoost("extension.property3", 5.0, "standard")); + tableConfig.setSearchFields(fields); + + assertDoesNotThrow( + () -> searchSettingsHandler.validateAssetTypeConfiguration(tableConfig), + "Configuration with only extension fields should be valid"); + } + + @Test + void testExtensionFieldWithSpecialCharactersInName() { + AssetTypeConfiguration tableConfig = new AssetTypeConfiguration(); + tableConfig.setAssetType("table"); + + List fields = new ArrayList<>(); + fields.add(createFieldBoost("name", 10.0, "phrase")); + fields.add(createFieldBoost("extension.property_with_underscore", 5.0, "standard")); + fields.add(createFieldBoost("extension.propertyWithCamelCase", 4.0, "standard")); + tableConfig.setSearchFields(fields); + + assertDoesNotThrow( + () -> searchSettingsHandler.validateAssetTypeConfiguration(tableConfig), + "Extension fields with underscores and camelCase should be valid"); + } + + @Test + void testSearchSettingsMergePreservesExtensionFields() { + SearchSettings defaultSettings = new SearchSettings(); + GlobalSettings defaultGlobal = new GlobalSettings(); + defaultGlobal.setMaxResultHits(5000); + defaultSettings.setGlobalSettings(defaultGlobal); + + AssetTypeConfiguration defaultTableConfig = new AssetTypeConfiguration(); + defaultTableConfig.setAssetType("table"); + List defaultFields = new ArrayList<>(); + defaultFields.add(createFieldBoost("name", 10.0, "phrase")); + defaultTableConfig.setSearchFields(defaultFields); + + List defaultAssetConfigs = new ArrayList<>(); + defaultAssetConfigs.add(defaultTableConfig); + defaultSettings.setAssetTypeConfigurations(defaultAssetConfigs); + + SearchSettings incomingSettings = new SearchSettings(); + GlobalSettings incomingGlobal = new GlobalSettings(); + incomingGlobal.setMaxResultHits(8000); + incomingSettings.setGlobalSettings(incomingGlobal); + + AssetTypeConfiguration incomingTableConfig = new AssetTypeConfiguration(); + incomingTableConfig.setAssetType("table"); + List incomingFields = new ArrayList<>(); + incomingFields.add(createFieldBoost("name", 10.0, "phrase")); + incomingFields.add(createFieldBoost("extension.customProperty", 5.0, "standard")); + incomingTableConfig.setSearchFields(incomingFields); + + List incomingAssetConfigs = new ArrayList<>(); + incomingAssetConfigs.add(incomingTableConfig); + incomingSettings.setAssetTypeConfigurations(incomingAssetConfigs); + + SearchSettings merged = + searchSettingsHandler.mergeSearchSettings(defaultSettings, incomingSettings); + + assertNotNull(merged, "Merged settings should not be null"); + assertNotNull( + merged.getAssetTypeConfigurations(), "Asset type configurations should not be null"); + + AssetTypeConfiguration mergedTableConfig = + merged.getAssetTypeConfigurations().stream() + .filter(config -> "table".equals(config.getAssetType())) + .findFirst() + .orElse(null); + + assertNotNull(mergedTableConfig, "Table configuration should exist in merged settings"); + assertNotNull(mergedTableConfig.getSearchFields(), "Search fields should not be null"); + + boolean hasExtensionField = + mergedTableConfig.getSearchFields().stream() + .anyMatch(field -> field.getField().startsWith("extension.")); + + assertTrue(hasExtensionField, "Merged settings should preserve extension fields"); + } + + private FieldBoost createFieldBoost(String field, Double boost, String matchType) { + FieldBoost fieldBoost = new FieldBoost(); + fieldBoost.setField(field); + fieldBoost.setBoost(boost); + if (matchType != null) { + fieldBoost.setMatchType(FieldBoost.MatchType.fromValue(matchType)); + } + return fieldBoost; + } +} diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/CustomPropertySearchSettings.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/CustomPropertySearchSettings.spec.ts new file mode 100644 index 00000000000..f0e68175f43 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/CustomPropertySearchSettings.spec.ts @@ -0,0 +1,419 @@ +/* + * Copyright 2025 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { expect, test } from '@playwright/test'; +import { CUSTOM_PROPERTIES_ENTITIES } from '../../constant/customProperty'; +import { GlobalSettingOptions } from '../../constant/settings'; +import { DashboardClass } from '../../support/entity/DashboardClass'; +import { EntityTypeEndpoint } from '../../support/entity/Entity.interface'; +import { PipelineClass } from '../../support/entity/PipelineClass'; +import { + createNewPage, + redirectToHomePage, + toastNotification, + uuid, +} from '../../utils/common'; +import { + addCustomPropertiesForEntity, + setValueForProperty, +} from '../../utils/customProperty'; +import { setSliderValue } from '../../utils/searchSettingUtils'; +import { settingClick } from '../../utils/sidebar'; + +test.use({ storageState: 'playwright/.auth/admin.json' }); + +test.describe.serial('Custom Property Search Settings', () => { + const dashboard = new DashboardClass(); + const pipeline = new PipelineClass(); + + const dashboardPropertyName = `pwSearchCPDashboard${uuid()}`; + const pipelinePropertyName = `pwSearchCPPipeline${uuid()}`; + + const dashboardPropertyValue = `EXECUTIVE_DASHBOARD_${uuid()}`; + const pipelinePropertyValue = `ETL_PRODUCTION_${uuid()}`; + + test.beforeAll( + 'Setup entities and custom properties', + async ({ browser }) => { + const { apiContext, afterAction } = await createNewPage(browser); + + await dashboard.create(apiContext); + await pipeline.create(apiContext); + + await afterAction(); + } + ); + + test.afterAll('Cleanup entities', async ({ browser }) => { + const { apiContext, afterAction } = await createNewPage(browser); + + await dashboard.delete(apiContext); + await pipeline.delete(apiContext); + + await afterAction(); + }); + + test('Create custom properties and configure search for Dashboard', async ({ + page, + }) => { + test.slow(true); + + await redirectToHomePage(page); + + await test.step( + 'Create and assign custom property to Dashboard', + async () => { + await settingClick(page, GlobalSettingOptions.DASHBOARDS, true); + + await addCustomPropertiesForEntity({ + page, + propertyName: dashboardPropertyName, + customPropertyData: CUSTOM_PROPERTIES_ENTITIES['entity_dashboard'], + customType: 'String', + }); + + await dashboard.visitEntityPage(page); + + const customPropertyResponse = page.waitForResponse( + '/api/v1/metadata/types/name/dashboard?fields=customProperties' + ); + await page.getByTestId('custom_properties').click(); + await customPropertyResponse; + + await page.waitForSelector('.ant-skeleton-active', { + state: 'detached', + }); + + await setValueForProperty({ + page, + propertyName: dashboardPropertyName, + value: dashboardPropertyValue, + propertyType: 'string', + endpoint: EntityTypeEndpoint.Dashboard, + }); + + // Verify the custom property was saved by refreshing the page + await page.reload(); + await page.waitForLoadState('networkidle'); + + const customPropertiesTab = page.getByTestId('custom_properties'); + await customPropertiesTab.click(); + await page.waitForSelector('.ant-skeleton-active', { + state: 'detached', + }); + + const propertyValue = page.getByText(dashboardPropertyValue); + + await expect(propertyValue).toBeVisible(); + + // Wait for the custom property value to be indexed in Elasticsearch + // Dashboards may take longer to index than tables + await page.waitForTimeout(10000); + } + ); + + await test.step( + 'Configure search settings for Dashboard custom property', + async () => { + await settingClick(page, GlobalSettingOptions.SEARCH_SETTINGS); + + const dashboardCard = page.getByTestId( + 'preferences.search-settings.dashboards' + ); + await dashboardCard.click(); + + await expect(page).toHaveURL( + /settings\/preferences\/search-settings\/dashboards$/ + ); + + await page.waitForLoadState('networkidle'); + await page.waitForSelector('[data-testid="loader"]', { + state: 'detached', + }); + + await page.getByTestId('add-field-btn').click(); + + // Wait for the dropdown menu to appear + await page.waitForTimeout(500); + + // Click on the custom property field in the dropdown menu + const customPropertyOption = page.getByText( + `extension.${dashboardPropertyName}`, + { exact: true } + ); + await customPropertyOption.click(); + + const fieldPanel = page.getByTestId( + `field-configuration-panel-extension.${dashboardPropertyName}` + ); + + await expect(fieldPanel).toBeVisible(); + + const customPropertyBadge = fieldPanel.getByTestId( + 'custom-property-badge' + ); + + await expect(customPropertyBadge).toBeVisible(); + + await fieldPanel.click(); + + await setSliderValue(page, 'field-weight-slider', 20); + + const matchTypeSelect = page.getByTestId('match-type-select'); + await matchTypeSelect.click(); + await page + .locator('.ant-select-item-option[title="Standard Match"]') + .click(); + + await page.getByTestId('save-btn').click(); + + await toastNotification(page, /Search Settings updated successfully/); + + // Wait for search settings to be reloaded by the application + // The search repository needs to refresh its configuration + await page.waitForTimeout(15000); + + // Verify the field is still visible after save + const savedFieldPanel = page.getByTestId( + `field-configuration-panel-extension.${dashboardPropertyName}` + ); + + await expect(savedFieldPanel).toBeVisible(); + } + ); + + await test.step( + 'Search for Dashboard using custom property value', + async () => { + await redirectToHomePage(page); + + // First, verify the dashboard exists by searching for it by name + await test.step('Verify dashboard is indexed', async () => { + const searchInput = page.getByTestId('searchBox'); + await searchInput.click(); + await searchInput.clear(); + await searchInput.fill(dashboard.entityResponseData.name); + await searchInput.press('Enter'); + + await page.waitForLoadState('networkidle'); + await page.waitForTimeout(2000); + + const searchResults = page.getByTestId('search-results'); + // Note: All entity search cards use 'table-data-card_' prefix regardless of entity type + const dashboardCard = searchResults.getByTestId( + `table-data-card_${dashboard.entityResponseData.fullyQualifiedName}` + ); + + await expect(dashboardCard).toBeVisible(); + }); + + await test.step( + 'Search for Dashboard using custom property value', + async () => { + await redirectToHomePage(page); + + const searchInput = page.getByTestId('searchBox'); + await searchInput.click(); + await searchInput.fill(dashboardPropertyValue); + await searchInput.press('Enter'); + + await page.waitForLoadState('networkidle'); + await page.waitForTimeout(2000); + + const searchResults = page.getByTestId('search-results'); + + // Note: All entity search cards use 'table-data-card_' prefix regardless of entity type + const dashboardCard = searchResults.getByTestId( + `table-data-card_${dashboard.entityResponseData.fullyQualifiedName}` + ); + + await expect(dashboardCard).toBeVisible(); + } + ); + } + ); + }); + + test('Create custom properties and configure search for Pipeline', async ({ + page, + }) => { + test.slow(true); + + await redirectToHomePage(page); + + await test.step( + 'Create and assign custom property to Pipeline', + async () => { + await settingClick(page, GlobalSettingOptions.PIPELINES, true); + + await addCustomPropertiesForEntity({ + page, + propertyName: pipelinePropertyName, + customPropertyData: CUSTOM_PROPERTIES_ENTITIES['entity_pipeline'], + customType: 'String', + }); + + await pipeline.visitEntityPage(page); + + const customPropertyResponse = page.waitForResponse( + '/api/v1/metadata/types/name/pipeline?fields=customProperties' + ); + await page.getByTestId('custom_properties').click(); + await customPropertyResponse; + + await page.waitForSelector('.ant-skeleton-active', { + state: 'detached', + }); + + await setValueForProperty({ + page, + propertyName: pipelinePropertyName, + value: pipelinePropertyValue, + propertyType: 'string', + endpoint: EntityTypeEndpoint.Pipeline, + }); + } + ); + + await test.step( + 'Configure search settings for Pipeline custom property', + async () => { + await settingClick(page, GlobalSettingOptions.SEARCH_SETTINGS); + + const pipelineCard = page.getByTestId( + 'preferences.search-settings.pipelines' + ); + await pipelineCard.click(); + + await expect(page).toHaveURL( + /settings\/preferences\/search-settings\/pipelines$/ + ); + + await page.waitForLoadState('networkidle'); + await page.waitForSelector('[data-testid="loader"]', { + state: 'detached', + }); + + await page.getByTestId('add-field-btn').click(); + + // Wait for the dropdown menu to appear + await page.waitForTimeout(500); + + // Click on the custom property field in the dropdown menu + const customPropertyOption = page.getByText( + `extension.${pipelinePropertyName}`, + { exact: true } + ); + await customPropertyOption.click(); + + const fieldPanel = page.getByTestId( + `field-configuration-panel-extension.${pipelinePropertyName}` + ); + + await expect(fieldPanel).toBeVisible(); + + const customPropertyBadge = fieldPanel.getByTestId( + 'custom-property-badge' + ); + + await expect(customPropertyBadge).toBeVisible(); + + await fieldPanel.click(); + + await setSliderValue(page, 'field-weight-slider', 12); + + const matchTypeSelect = page.getByTestId('match-type-select'); + await matchTypeSelect.click(); + await page + .locator('.ant-select-item-option[title="Phrase Match"]') + .click(); + + await page.getByTestId('save-btn').click(); + + await toastNotification(page, /Search Settings updated successfully/); + + // Wait for search index to update with new settings + await page.waitForTimeout(10000); + } + ); + + await test.step( + 'Search for Pipeline using custom property value', + async () => { + await redirectToHomePage(page); + + const searchInput = page.getByTestId('searchBox'); + await searchInput.click(); + await searchInput.clear(); + await searchInput.fill(pipelinePropertyValue); + await searchInput.press('Enter'); + + await page.waitForLoadState('networkidle'); + await page.waitForTimeout(2000); + + const searchResults = page.getByTestId('search-results'); + // Note: All entity search cards use 'table-data-card_' prefix regardless of entity type + const pipelineCard = searchResults.getByTestId( + `table-data-card_${pipeline.entityResponseData.fullyQualifiedName}` + ); + + await expect(pipelineCard).toBeVisible(); + } + ); + }); + + test('Verify custom property fields are persisted in search settings', async ({ + page, + }) => { + await redirectToHomePage(page); + + await test.step('Verify Dashboard custom property persists', async () => { + await settingClick(page, GlobalSettingOptions.SEARCH_SETTINGS); + + const dashboardCard = page.getByTestId( + 'preferences.search-settings.dashboards' + ); + await dashboardCard.click(); + + await page.waitForLoadState('networkidle'); + await page.waitForSelector('[data-testid="loader"]', { + state: 'detached', + }); + + const customPropertyField = page.getByTestId( + `field-configuration-panel-extension.${dashboardPropertyName}` + ); + + await expect(customPropertyField).toBeVisible(); + }); + + await test.step('Verify Pipeline custom property persists', async () => { + await settingClick(page, GlobalSettingOptions.SEARCH_SETTINGS); + + const pipelineCard = page.getByTestId( + 'preferences.search-settings.pipelines' + ); + await pipelineCard.click(); + + await page.waitForLoadState('networkidle'); + await page.waitForSelector('[data-testid="loader"]', { + state: 'detached', + }); + + const customPropertyField = page.getByTestId( + `field-configuration-panel-extension.${pipelinePropertyName}` + ); + + await expect(customPropertyField).toBeVisible(); + }); + }); +}); diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/CustomizeDetailPage.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/CustomizeDetailPage.spec.ts index d63c3effaab..d558711c993 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/CustomizeDetailPage.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/CustomizeDetailPage.spec.ts @@ -10,7 +10,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { test as base, expect, Page } from '@playwright/test'; +import { expect, Page, test as base } from '@playwright/test'; import { ECustomizedDataAssets, ECustomizedGovernance, diff --git a/openmetadata-ui/src/main/resources/ui/playwright/utils/customProperty.ts b/openmetadata-ui/src/main/resources/ui/playwright/utils/customProperty.ts index 9c25313c1c2..badbc0c3a42 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/utils/customProperty.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/utils/customProperty.ts @@ -291,6 +291,7 @@ export const validateValueForProperty = async (data: { page.getByRole('row', { name: `${values[0]} ${values[1]}` }) ).toBeVisible(); } else if (propertyType === 'markdown') { + // For markdown, remove * and _ as they are formatting characters await expect( container.locator(descriptionBoxReadOnly).last() ).toContainText(value.replace(/\*|_/gi, '')); @@ -302,7 +303,8 @@ export const validateValueForProperty = async (data: { 'dateTime-cp', ].includes(propertyType) ) { - await expect(container).toContainText(value.replace(/\*|_/gi, '')); + // For other types (string, integer, number, duration), match exact value without transformation + await expect(container.getByTestId('value')).toContainText(value); } }; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Database/Profiler/DataQualityTab/DataQualityTab.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Database/Profiler/DataQualityTab/DataQualityTab.tsx index 1fe81deb193..9e6c5e572bb 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Database/Profiler/DataQualityTab/DataQualityTab.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Database/Profiler/DataQualityTab/DataQualityTab.tsx @@ -16,8 +16,8 @@ import { IconButton, Menu, MenuItem, - Typography as MuiTypography, Skeleton, + Typography as MuiTypography, } from '@mui/material'; import { Typography } from 'antd'; import { ColumnsType, TablePaginationConfig } from 'antd/lib/table'; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/SearchSettings/EntitySeachSettings/EntitySearchSettings.tsx b/openmetadata-ui/src/main/resources/ui/src/components/SearchSettings/EntitySeachSettings/EntitySearchSettings.tsx index 40fd98f5b5e..52dff5b6feb 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/SearchSettings/EntitySeachSettings/EntitySearchSettings.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/SearchSettings/EntitySeachSettings/EntitySearchSettings.tsx @@ -39,6 +39,7 @@ import { import { useAuth } from '../../../hooks/authHooks'; import { useApplicationStore } from '../../../hooks/useApplicationStore'; import { EntitySearchSettingsState } from '../../../pages/SearchSettingsPage/searchSettings.interface'; +import { getCustomPropertiesByEntityType } from '../../../rest/metadataTypeAPI'; import { getSettingsByType, restoreSettingsConfig, @@ -97,6 +98,9 @@ const EntitySearchSettings = () => { FieldValueBoost | undefined >(); const [allowedFields, setAllowedFields] = useState([]); + const [customProperties, setCustomProperties] = useState( + [] + ); const [activeKey, setActiveKey] = useState('1'); const [lastAddedSearchField, setLastAddedSearchField] = useState< string | null @@ -144,11 +148,13 @@ const EntitySearchSettings = () => { allowedFields.find((field) => field.entityType === entityType)?.fields ?? []; - return currentEntityFields.map((field) => ({ + const regularFields = currentEntityFields.map((field) => ({ name: field.name, description: field.description, })); - }, [allowedFields, entityType]); + + return [...regularFields, ...customProperties]; + }, [allowedFields, entityType, customProperties]); const fieldValueBoostOptions = useMemo(() => { if (!isEmpty(searchConfig?.allowedFieldValueBoosts)) { @@ -494,6 +500,25 @@ const EntitySearchSettings = () => { setActiveKey(Array.isArray(key) ? key[0] : key); }; + const fetchCustomProperties = async () => { + try { + const properties = await getCustomPropertiesByEntityType(entityType); + + const formattedProperties: AllowedFieldField[] = properties.map( + (prop) => ({ + name: `extension.${prop.name}`, + description: `${ + prop.description ?? prop.displayName + } (Custom Property)`, + }) + ); + + setCustomProperties(formattedProperties); + } catch (error) { + setCustomProperties([]); + } + }; + useEffect(() => { fetchSearchConfig(); }, []); @@ -504,6 +529,12 @@ const EntitySearchSettings = () => { } }, [searchConfig]); + useEffect(() => { + if (entityType) { + fetchCustomProperties(); + } + }, [entityType]); + useEffect(() => { if (getEntityConfiguration) { setSearchSettings({ diff --git a/openmetadata-ui/src/main/resources/ui/src/components/SearchSettings/FieldConfiguration/FieldConfiguration.tsx b/openmetadata-ui/src/main/resources/ui/src/components/SearchSettings/FieldConfiguration/FieldConfiguration.tsx index 5dd9d1fbaf3..40239713e26 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/SearchSettings/FieldConfiguration/FieldConfiguration.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/SearchSettings/FieldConfiguration/FieldConfiguration.tsx @@ -12,6 +12,7 @@ */ import Icon from '@ant-design/icons'; import { + Badge, Button, Collapse, Divider, @@ -106,9 +107,19 @@ const FieldConfiguration: React.FC = ({ : 'white', }}>
- - {field.fieldName} - +
+ + {field.fieldName} + + {field.fieldName.startsWith('extension.') && ( + + )} +