From 648a8f86deff9a2fd59cc89f3578bd8fc362eefe Mon Sep 17 00:00:00 2001 From: sonika-shah <58761340+sonika-shah@users.noreply.github.com> Date: Thu, 25 Sep 2025 17:04:49 +0530 Subject: [PATCH] add entityType.keyword aggregation in searchSettings.json (#23559) * add searchSettings aggregation in searchSettings.json * refactor migrations to only update the agg in searchSettings * fix migrations --- .../native/1.9.11/mysql/schemaChanges.sql | 2 + .../native/1.9.11/postgres/schemaChanges.sql | 3 + .../migration/mysql/v1911/Migration.java | 19 ++++ .../migration/postgres/v1911/Migration.java | 19 ++++ .../migration/utils/v1911/MigrationUtil.java | 87 +++++++++++++++++++ .../json/data/settings/searchSettings.json | 5 ++ 6 files changed, 135 insertions(+) create mode 100644 bootstrap/sql/migrations/native/1.9.11/mysql/schemaChanges.sql create mode 100644 bootstrap/sql/migrations/native/1.9.11/postgres/schemaChanges.sql create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v1911/Migration.java create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v1911/Migration.java create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v1911/MigrationUtil.java diff --git a/bootstrap/sql/migrations/native/1.9.11/mysql/schemaChanges.sql b/bootstrap/sql/migrations/native/1.9.11/mysql/schemaChanges.sql new file mode 100644 index 00000000000..f3a376a9212 --- /dev/null +++ b/bootstrap/sql/migrations/native/1.9.11/mysql/schemaChanges.sql @@ -0,0 +1,2 @@ +-- Add migrations to fetch updated searchSettings +-- Java migration handles adding entityType.keyword aggregation \ No newline at end of file diff --git a/bootstrap/sql/migrations/native/1.9.11/postgres/schemaChanges.sql b/bootstrap/sql/migrations/native/1.9.11/postgres/schemaChanges.sql new file mode 100644 index 00000000000..c5c39e77b4c --- /dev/null +++ b/bootstrap/sql/migrations/native/1.9.11/postgres/schemaChanges.sql @@ -0,0 +1,3 @@ +-- Add migrations to fetch updated searchSettings +-- Java migration handles adding entityType.keyword aggregation + diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v1911/Migration.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v1911/Migration.java new file mode 100644 index 00000000000..0196e29d28c --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v1911/Migration.java @@ -0,0 +1,19 @@ +package org.openmetadata.service.migration.mysql.v1911; + +import lombok.SneakyThrows; +import org.openmetadata.service.migration.api.MigrationProcessImpl; +import org.openmetadata.service.migration.utils.MigrationFile; +import org.openmetadata.service.migration.utils.v1911.MigrationUtil; + +public class Migration extends MigrationProcessImpl { + + public Migration(MigrationFile migrationFile) { + super(migrationFile); + } + + @Override + @SneakyThrows + public void runDataMigration() { + MigrationUtil.updateSearchSettingsEntityTypeKeyword(); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v1911/Migration.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v1911/Migration.java new file mode 100644 index 00000000000..225c5c37249 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v1911/Migration.java @@ -0,0 +1,19 @@ +package org.openmetadata.service.migration.postgres.v1911; + +import lombok.SneakyThrows; +import org.openmetadata.service.migration.api.MigrationProcessImpl; +import org.openmetadata.service.migration.utils.MigrationFile; +import org.openmetadata.service.migration.utils.v1911.MigrationUtil; + +public class Migration extends MigrationProcessImpl { + + public Migration(MigrationFile migrationFile) { + super(migrationFile); + } + + @Override + @SneakyThrows + public void runDataMigration() { + MigrationUtil.updateSearchSettingsEntityTypeKeyword(); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v1911/MigrationUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v1911/MigrationUtil.java new file mode 100644 index 00000000000..ec11ee0968b --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v1911/MigrationUtil.java @@ -0,0 +1,87 @@ +package org.openmetadata.service.migration.utils.v1911; + +import com.fasterxml.jackson.databind.JsonNode; +import jakarta.json.Json; +import jakarta.json.JsonPatch; +import jakarta.json.JsonPatchBuilder; +import jakarta.json.JsonValue; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.StreamSupport; +import lombok.extern.slf4j.Slf4j; +import org.openmetadata.schema.api.search.SearchSettings; +import org.openmetadata.schema.settings.Settings; +import org.openmetadata.schema.utils.JsonUtils; +import org.openmetadata.service.Entity; +import org.openmetadata.service.jdbi3.SystemRepository; + +@Slf4j +public class MigrationUtil { + + private static final SystemRepository systemRepository = Entity.getSystemRepository(); + private static final String SEARCH_SETTINGS_KEY = "searchSettings"; + private static final String AGGREGATIONS = "aggregations"; + private static final String GLOBAL_SETTINGS = "globalSettings"; + private static final String NAME = "name"; + private static final String TYPE = "terms"; + private static final String FIELD = "field"; + private static final String ENTITY_TYPE_KEYWORD = "entityType.keyword"; + + public static void updateSearchSettingsEntityTypeKeyword() { + try { + LOG.info("Updating search settings to ensure entityType.keyword aggregation exists"); + + Settings searchSettings = systemRepository.getConfigWithKey(SEARCH_SETTINGS_KEY); + if (searchSettings == null) { + LOG.warn("Search settings not found, skipping migration"); + return; + } + + String rawJson = JsonUtils.pojoToJson(searchSettings.getConfigValue()); + LOG.debug("Current search settings JSON: {}", rawJson); + JsonNode settingsNode = JsonUtils.readTree(rawJson); + JsonPatchBuilder patchBuilder = Json.createPatchBuilder(); + AtomicBoolean needsUpdate = new AtomicBoolean(false); + + JsonNode globalSettings = settingsNode.get(GLOBAL_SETTINGS); + if (globalSettings != null) { + JsonNode aggregations = globalSettings.get(AGGREGATIONS); + if (aggregations != null && aggregations.isArray()) { + boolean entityTypeKeywordExists = + StreamSupport.stream(aggregations.spliterator(), false) + .anyMatch( + aggregation -> ENTITY_TYPE_KEYWORD.equals(aggregation.get(NAME).asText())); + + if (!entityTypeKeywordExists) { + LOG.info("Adding missing entityType.keyword aggregation"); + patchBuilder.add( + "/" + GLOBAL_SETTINGS + "/" + AGGREGATIONS + "/-", + Json.createObjectBuilder() + .add(NAME, ENTITY_TYPE_KEYWORD) + .add("type", TYPE) + .add(FIELD, ENTITY_TYPE_KEYWORD) + .build()); + needsUpdate.set(true); + } else { + LOG.info("entityType.keyword aggregation already exists, no update needed"); + } + } + } + + if (needsUpdate.get()) { + JsonPatch patch = patchBuilder.build(); + LOG.debug("Applying patch: {}", patch.toString()); + JsonValue updated = JsonUtils.applyPatch(searchSettings.getConfigValue(), patch); + SearchSettings updatedSettings = + JsonUtils.readValue(updated.toString(), SearchSettings.class); + searchSettings.withConfigValue(updatedSettings); + systemRepository.updateSetting(searchSettings); + LOG.info("Search settings updated successfully with entityType.keyword aggregation"); + } else { + LOG.info("No updates needed for search settings"); + } + + } catch (Exception e) { + LOG.error("Error updating search settings for entityType.keyword aggregation", e); + } + } +} diff --git a/openmetadata-service/src/main/resources/json/data/settings/searchSettings.json b/openmetadata-service/src/main/resources/json/data/settings/searchSettings.json index 05ac64bcccd..80917dd9c50 100644 --- a/openmetadata-service/src/main/resources/json/data/settings/searchSettings.json +++ b/openmetadata-service/src/main/resources/json/data/settings/searchSettings.json @@ -20,6 +20,11 @@ "type": "terms", "field": "entityType" }, + { + "name": "entityType.keyword", + "type": "terms", + "field": "entityType.keyword" + }, { "name": "tier.tagFQN", "type": "terms",