diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/ESIndexBuilder.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/ESIndexBuilder.java index 2d04e99774..6de79b6c4b 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/ESIndexBuilder.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/ESIndexBuilder.java @@ -87,6 +87,8 @@ public class ESIndexBuilder { @Getter private final GitVersion gitVersion; + @Getter private final int maxReindexHours; + private static final RequestOptions REQUEST_OPTIONS = RequestOptions.DEFAULT.toBuilder() .setRequestConfig(RequestConfig.custom().setSocketTimeout(180 * 1000).build()) @@ -106,6 +108,34 @@ public class ESIndexBuilder { boolean enableStructuredPropertiesReindex, ElasticSearchConfiguration elasticSearchConfiguration, GitVersion gitVersion) { + this( + searchClient, + numShards, + numReplicas, + numRetries, + refreshIntervalSeconds, + indexSettingOverrides, + enableIndexSettingsReindex, + enableIndexMappingsReindex, + enableStructuredPropertiesReindex, + elasticSearchConfiguration, + gitVersion, + 0); + } + + public ESIndexBuilder( + RestHighLevelClient searchClient, + int numShards, + int numReplicas, + int numRetries, + int refreshIntervalSeconds, + Map> indexSettingOverrides, + boolean enableIndexSettingsReindex, + boolean enableIndexMappingsReindex, + boolean enableStructuredPropertiesReindex, + ElasticSearchConfiguration elasticSearchConfiguration, + GitVersion gitVersion, + int maxReindexHours) { this._searchClient = searchClient; this.numShards = numShards; this.numReplicas = numReplicas; @@ -117,6 +147,7 @@ public class ESIndexBuilder { this.elasticSearchConfiguration = elasticSearchConfiguration; this.enableStructuredPropertiesReindex = enableStructuredPropertiesReindex; this.gitVersion = gitVersion; + this.maxReindexHours = maxReindexHours; RetryConfig config = RetryConfig.custom() @@ -348,10 +379,10 @@ public class ESIndexBuilder { private void reindex(ReindexConfig indexState) throws Throwable { final long startTime = System.currentTimeMillis(); - final int maxReindexHours = 8; final long initialCheckIntervalMilli = 1000; final long finalCheckIntervalMilli = 60000; - final long timeoutAt = startTime + (1000 * 60 * 60 * maxReindexHours); + final long timeoutAt = + maxReindexHours > 0 ? startTime + (1000L * 60 * 60 * maxReindexHours) : Long.MAX_VALUE; String tempIndexName = getNextIndexName(indexState.name(), startTime); diff --git a/metadata-service/configuration/src/main/resources/application.yaml b/metadata-service/configuration/src/main/resources/application.yaml index 01800e615b..e8d5be9bfb 100644 --- a/metadata-service/configuration/src/main/resources/application.yaml +++ b/metadata-service/configuration/src/main/resources/application.yaml @@ -216,6 +216,7 @@ elasticsearch: mainTokenizer: ${ELASTICSEARCH_MAIN_TOKENIZER:#{null}} enableMappingsReindex: ${ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX:false} enableSettingsReindex: ${ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX:false} + maxReindexHours: ${ELASTICSEARCH_INDEX_BUILDER_MAX_REINDEX_HOURS:0} # <= 0 - no timeout settingsOverrides: ${ELASTICSEARCH_INDEX_BUILDER_SETTINGS_OVERRIDES:#{null}} entitySettingsOverrides: ${ELASTICSEARCH_INDEX_BUILDER_ENTITY_SETTINGS_OVERRIDES:#{null}} docIds: diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/ElasticSearchIndexBuilderFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/ElasticSearchIndexBuilderFactory.java index 4513d9f065..76be500d10 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/ElasticSearchIndexBuilderFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/ElasticSearchIndexBuilderFactory.java @@ -60,6 +60,9 @@ public class ElasticSearchIndexBuilderFactory { @Value("#{new Boolean('${structuredProperties.systemUpdateEnabled}')}") private boolean enableStructuredPropertiesReindex; + @Value("${elasticsearch.index.maxReindexHours}") + private Integer maxReindexHours; + @Bean(name = "elasticSearchIndexSettingsOverrides") @Nonnull protected Map> getIndexSettingsOverrides( @@ -90,7 +93,8 @@ public class ElasticSearchIndexBuilderFactory { enableMappingsReindex, enableStructuredPropertiesReindex, configurationProvider.getElasticSearch(), - gitVersion); + gitVersion, + maxReindexHours); } @Nonnull