yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-11-08 07:23:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

config(reindex): create reindex timeout configuration (#11456)

Browse Source
This commit is contained in:
david-leifker 2024-09-23 11:13:39 -05:00 committed by GitHub
parent a481ea4ffb
commit 090c51423e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 39 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/ESIndexBuilder.java
View File

@ -87,6 +87,8 @@ public class ESIndexBuilder {
@Getter private final GitVersion gitVersion;
@Getter private final int maxReindexHours;
private static final RequestOptions REQUEST_OPTIONS =
RequestOptions.DEFAULT.toBuilder()
.setRequestConfig(RequestConfig.custom().setSocketTimeout(180 * 1000).build())
@ -106,6 +108,34 @@ public class ESIndexBuilder {
boolean enableStructuredPropertiesReindex,
ElasticSearchConfiguration elasticSearchConfiguration,
GitVersion gitVersion) {
this(
searchClient,
numShards,
numReplicas,
numRetries,
refreshIntervalSeconds,
indexSettingOverrides,
enableIndexSettingsReindex,
enableIndexMappingsReindex,
enableStructuredPropertiesReindex,
elasticSearchConfiguration,
gitVersion,
0);
}
public ESIndexBuilder(
RestHighLevelClient searchClient,
int numShards,
int numReplicas,
int numRetries,
int refreshIntervalSeconds,
Map<String, Map<String, String>> indexSettingOverrides,
boolean enableIndexSettingsReindex,
boolean enableIndexMappingsReindex,
boolean enableStructuredPropertiesReindex,
ElasticSearchConfiguration elasticSearchConfiguration,
GitVersion gitVersion,
int maxReindexHours) {
this._searchClient = searchClient;
this.numShards = numShards;
this.numReplicas = numReplicas;
@ -117,6 +147,7 @@ public class ESIndexBuilder {
this.elasticSearchConfiguration = elasticSearchConfiguration;
this.enableStructuredPropertiesReindex = enableStructuredPropertiesReindex;
this.gitVersion = gitVersion;
this.maxReindexHours = maxReindexHours;
RetryConfig config =
RetryConfig.custom()
@ -348,10 +379,10 @@ public class ESIndexBuilder {
private void reindex(ReindexConfig indexState) throws Throwable {
final long startTime = System.currentTimeMillis();
final int maxReindexHours = 8;
final long initialCheckIntervalMilli = 1000;
final long finalCheckIntervalMilli = 60000;
final long timeoutAt = startTime + (1000 * 60 * 60 * maxReindexHours);
final long timeoutAt =
maxReindexHours > 0 ? startTime + (1000L * 60 * 60 * maxReindexHours) : Long.MAX_VALUE;
String tempIndexName = getNextIndexName(indexState.name(), startTime);

1
metadata-service/configuration/src/main/resources/application.yaml
View File

@ -216,6 +216,7 @@ elasticsearch:
mainTokenizer: ${ELASTICSEARCH_MAIN_TOKENIZER:#{null}}
enableMappingsReindex: ${ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX:false}
enableSettingsReindex: ${ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX:false}
maxReindexHours: ${ELASTICSEARCH_INDEX_BUILDER_MAX_REINDEX_HOURS:0} # <= 0 - no timeout
settingsOverrides: ${ELASTICSEARCH_INDEX_BUILDER_SETTINGS_OVERRIDES:#{null}}
entitySettingsOverrides: ${ELASTICSEARCH_INDEX_BUILDER_ENTITY_SETTINGS_OVERRIDES:#{null}}
docIds:

6
metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/ElasticSearchIndexBuilderFactory.java
View File

@ -60,6 +60,9 @@ public class ElasticSearchIndexBuilderFactory {
@Value("#{new Boolean('${structuredProperties.systemUpdateEnabled}')}")
private boolean enableStructuredPropertiesReindex;
@Value("${elasticsearch.index.maxReindexHours}")
private Integer maxReindexHours;
@Bean(name = "elasticSearchIndexSettingsOverrides")
@Nonnull
protected Map<String, Map<String, String>> getIndexSettingsOverrides(
@ -90,7 +93,8 @@ public class ElasticSearchIndexBuilderFactory {
enableMappingsReindex,
enableStructuredPropertiesReindex,
configurationProvider.getElasticSearch(),
gitVersion);
gitVersion,
maxReindexHours);
}
@Nonnull