yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-09-29 02:45:25 +00:00
Code Issues Packages Projects Releases Wiki Activity

add entityType.keyword aggregation in searchSettings.json (#23559)

Browse Source 
* add searchSettings aggregation in searchSettings.json

* refactor migrations to only update the agg in searchSettings

* fix migrations
This commit is contained in:
sonika-shah 2025-09-25 17:04:49 +05:30 committed by GitHub
parent e77b4d6faf
commit 648a8f86de
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 135 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
bootstrap/sql/migrations/native/1.9.11/mysql/schemaChanges.sql Normal file
View File

@ -0,0 +1,2 @@
-- Add migrations to fetch updated searchSettings
-- Java migration handles adding entityType.keyword aggregation

3
bootstrap/sql/migrations/native/1.9.11/postgres/schemaChanges.sql Normal file
View File

@ -0,0 +1,3 @@
-- Add migrations to fetch updated searchSettings
-- Java migration handles adding entityType.keyword aggregation

19
openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v1911/Migration.java Normal file
View File

@ -0,0 +1,19 @@
package org.openmetadata.service.migration.mysql.v1911;
import lombok.SneakyThrows;
import org.openmetadata.service.migration.api.MigrationProcessImpl;
import org.openmetadata.service.migration.utils.MigrationFile;
import org.openmetadata.service.migration.utils.v1911.MigrationUtil;
public class Migration extends MigrationProcessImpl {
public Migration(MigrationFile migrationFile) {
super(migrationFile);
}
@Override
@SneakyThrows
public void runDataMigration() {
MigrationUtil.updateSearchSettingsEntityTypeKeyword();
}
}

19
openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v1911/Migration.java Normal file
View File

@ -0,0 +1,19 @@
package org.openmetadata.service.migration.postgres.v1911;
import lombok.SneakyThrows;
import org.openmetadata.service.migration.api.MigrationProcessImpl;
import org.openmetadata.service.migration.utils.MigrationFile;
import org.openmetadata.service.migration.utils.v1911.MigrationUtil;
public class Migration extends MigrationProcessImpl {
public Migration(MigrationFile migrationFile) {
super(migrationFile);
}
@Override
@SneakyThrows
public void runDataMigration() {
MigrationUtil.updateSearchSettingsEntityTypeKeyword();
}
}

87
openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v1911/MigrationUtil.java Normal file
View File

@ -0,0 +1,87 @@
package org.openmetadata.service.migration.utils.v1911;
import com.fasterxml.jackson.databind.JsonNode;
import jakarta.json.Json;
import jakarta.json.JsonPatch;
import jakarta.json.JsonPatchBuilder;
import jakarta.json.JsonValue;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.stream.StreamSupport;
import lombok.extern.slf4j.Slf4j;
import org.openmetadata.schema.api.search.SearchSettings;
import org.openmetadata.schema.settings.Settings;
import org.openmetadata.schema.utils.JsonUtils;
import org.openmetadata.service.Entity;
import org.openmetadata.service.jdbi3.SystemRepository;
@Slf4j
public class MigrationUtil {
private static final SystemRepository systemRepository = Entity.getSystemRepository();
private static final String SEARCH_SETTINGS_KEY = "searchSettings";
private static final String AGGREGATIONS = "aggregations";
private static final String GLOBAL_SETTINGS = "globalSettings";
private static final String NAME = "name";
private static final String TYPE = "terms";
private static final String FIELD = "field";
private static final String ENTITY_TYPE_KEYWORD = "entityType.keyword";
public static void updateSearchSettingsEntityTypeKeyword() {
try {
LOG.info("Updating search settings to ensure entityType.keyword aggregation exists");
Settings searchSettings = systemRepository.getConfigWithKey(SEARCH_SETTINGS_KEY);
if (searchSettings == null) {
LOG.warn("Search settings not found, skipping migration");
return;
}
String rawJson = JsonUtils.pojoToJson(searchSettings.getConfigValue());
LOG.debug("Current search settings JSON: {}", rawJson);
JsonNode settingsNode = JsonUtils.readTree(rawJson);
JsonPatchBuilder patchBuilder = Json.createPatchBuilder();
AtomicBoolean needsUpdate = new AtomicBoolean(false);
JsonNode globalSettings = settingsNode.get(GLOBAL_SETTINGS);
if (globalSettings != null) {
JsonNode aggregations = globalSettings.get(AGGREGATIONS);
if (aggregations != null && aggregations.isArray()) {
boolean entityTypeKeywordExists =
StreamSupport.stream(aggregations.spliterator(), false)
.anyMatch(
aggregation -> ENTITY_TYPE_KEYWORD.equals(aggregation.get(NAME).asText()));
if (!entityTypeKeywordExists) {
LOG.info("Adding missing entityType.keyword aggregation");
patchBuilder.add(
"/" + GLOBAL_SETTINGS + "/" + AGGREGATIONS + "/-",
Json.createObjectBuilder()
.add(NAME, ENTITY_TYPE_KEYWORD)
.add("type", TYPE)
.add(FIELD, ENTITY_TYPE_KEYWORD)
.build());
needsUpdate.set(true);
} else {
LOG.info("entityType.keyword aggregation already exists, no update needed");
}
}
}
if (needsUpdate.get()) {
JsonPatch patch = patchBuilder.build();
LOG.debug("Applying patch: {}", patch.toString());
JsonValue updated = JsonUtils.applyPatch(searchSettings.getConfigValue(), patch);
SearchSettings updatedSettings =
JsonUtils.readValue(updated.toString(), SearchSettings.class);
searchSettings.withConfigValue(updatedSettings);
systemRepository.updateSetting(searchSettings);
LOG.info("Search settings updated successfully with entityType.keyword aggregation");
} else {
LOG.info("No updates needed for search settings");
}
} catch (Exception e) {
LOG.error("Error updating search settings for entityType.keyword aggregation", e);
}
}
}

5
openmetadata-service/src/main/resources/json/data/settings/searchSettings.json
View File

@ -20,6 +20,11 @@
"type": "terms",
"field": "entityType"
},
{
"name": "entityType.keyword",
"type": "terms",
"field": "entityType.keyword"
},
{
"name": "tier.tagFQN",
"type": "terms",