yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-11-10 00:11:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

config(reindex): create reindex timeout configuration (#11456)

Browse Source
This commit is contained in:
david-leifker 2024-09-23 11:13:39 -05:00 committed by GitHub
parent a481ea4ffb
commit 090c51423e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 39 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/ESIndexBuilder.java
View File

@ -87,6 +87,8 @@ public class ESIndexBuilder {
@Getter private final GitVersion gitVersion; @Getter private final GitVersion gitVersion;
@Getter private final int maxReindexHours;
private static final RequestOptions REQUEST_OPTIONS = private static final RequestOptions REQUEST_OPTIONS =
RequestOptions.DEFAULT.toBuilder() RequestOptions.DEFAULT.toBuilder()
.setRequestConfig(RequestConfig.custom().setSocketTimeout(180 * 1000).build()) .setRequestConfig(RequestConfig.custom().setSocketTimeout(180 * 1000).build())
@ -106,6 +108,34 @@ public class ESIndexBuilder {
boolean enableStructuredPropertiesReindex, boolean enableStructuredPropertiesReindex,
ElasticSearchConfiguration elasticSearchConfiguration, ElasticSearchConfiguration elasticSearchConfiguration,
GitVersion gitVersion) { GitVersion gitVersion) {
this(
searchClient,
numShards,
numReplicas,
numRetries,
refreshIntervalSeconds,
indexSettingOverrides,
enableIndexSettingsReindex,
enableIndexMappingsReindex,
enableStructuredPropertiesReindex,
elasticSearchConfiguration,
gitVersion,
0);
}
public ESIndexBuilder(
RestHighLevelClient searchClient,
int numShards,
int numReplicas,
int numRetries,
int refreshIntervalSeconds,
Map<String, Map<String, String>> indexSettingOverrides,
boolean enableIndexSettingsReindex,
boolean enableIndexMappingsReindex,
boolean enableStructuredPropertiesReindex,
ElasticSearchConfiguration elasticSearchConfiguration,
GitVersion gitVersion,
int maxReindexHours) {
this._searchClient = searchClient; this._searchClient = searchClient;
this.numShards = numShards; this.numShards = numShards;
this.numReplicas = numReplicas; this.numReplicas = numReplicas;
@ -117,6 +147,7 @@ public class ESIndexBuilder {
this.elasticSearchConfiguration = elasticSearchConfiguration; this.elasticSearchConfiguration = elasticSearchConfiguration;
this.enableStructuredPropertiesReindex = enableStructuredPropertiesReindex; this.enableStructuredPropertiesReindex = enableStructuredPropertiesReindex;
this.gitVersion = gitVersion; this.gitVersion = gitVersion;
this.maxReindexHours = maxReindexHours;
RetryConfig config = RetryConfig config =
RetryConfig.custom() RetryConfig.custom()
@ -348,10 +379,10 @@ public class ESIndexBuilder {
private void reindex(ReindexConfig indexState) throws Throwable { private void reindex(ReindexConfig indexState) throws Throwable {
final long startTime = System.currentTimeMillis(); final long startTime = System.currentTimeMillis();
final int maxReindexHours = 8;
final long initialCheckIntervalMilli = 1000; final long initialCheckIntervalMilli = 1000;
final long finalCheckIntervalMilli = 60000; final long finalCheckIntervalMilli = 60000;
final long timeoutAt = startTime + (1000 * 60 * 60 * maxReindexHours); final long timeoutAt =
maxReindexHours > 0 ? startTime + (1000L * 60 * 60 * maxReindexHours) : Long.MAX_VALUE;
String tempIndexName = getNextIndexName(indexState.name(), startTime); String tempIndexName = getNextIndexName(indexState.name(), startTime);

1
metadata-service/configuration/src/main/resources/application.yaml
View File

@ -216,6 +216,7 @@ elasticsearch:
mainTokenizer: ${ELASTICSEARCH_MAIN_TOKENIZER:#{null}} mainTokenizer: ${ELASTICSEARCH_MAIN_TOKENIZER:#{null}}
enableMappingsReindex: ${ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX:false} enableMappingsReindex: ${ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX:false}
enableSettingsReindex: ${ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX:false} enableSettingsReindex: ${ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX:false}
maxReindexHours: ${ELASTICSEARCH_INDEX_BUILDER_MAX_REINDEX_HOURS:0} # <= 0 - no timeout
settingsOverrides: ${ELASTICSEARCH_INDEX_BUILDER_SETTINGS_OVERRIDES:#{null}} settingsOverrides: ${ELASTICSEARCH_INDEX_BUILDER_SETTINGS_OVERRIDES:#{null}}
entitySettingsOverrides: ${ELASTICSEARCH_INDEX_BUILDER_ENTITY_SETTINGS_OVERRIDES:#{null}} entitySettingsOverrides: ${ELASTICSEARCH_INDEX_BUILDER_ENTITY_SETTINGS_OVERRIDES:#{null}}
docIds: docIds:

6
metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/ElasticSearchIndexBuilderFactory.java
View File

@ -60,6 +60,9 @@ public class ElasticSearchIndexBuilderFactory {
@Value("#{new Boolean('${structuredProperties.systemUpdateEnabled}')}") @Value("#{new Boolean('${structuredProperties.systemUpdateEnabled}')}")
private boolean enableStructuredPropertiesReindex; private boolean enableStructuredPropertiesReindex;
@Value("${elasticsearch.index.maxReindexHours}")
private Integer maxReindexHours;
@Bean(name = "elasticSearchIndexSettingsOverrides") @Bean(name = "elasticSearchIndexSettingsOverrides")
@Nonnull @Nonnull
protected Map<String, Map<String, String>> getIndexSettingsOverrides( protected Map<String, Map<String, String>> getIndexSettingsOverrides(
@ -90,7 +93,8 @@ public class ElasticSearchIndexBuilderFactory {
enableMappingsReindex, enableMappingsReindex,
enableStructuredPropertiesReindex, enableStructuredPropertiesReindex,
configurationProvider.getElasticSearch(), configurationProvider.getElasticSearch(),
gitVersion); gitVersion,
maxReindexHours);
} }
@Nonnull @Nonnull