yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-08-23 08:38:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(search): refactor NUM_RETRIES in esindexbuilder to be configurable (#3870)

Browse Source
This commit is contained in:
senni0418 2022-01-17 18:35:59 -05:00 committed by GitHub
parent 9e30b42638
commit 80c46f24ac
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 8 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/ESIndexBuilder.java
View File

@ -39,8 +39,8 @@ public class ESIndexBuilder {
private final RestHighLevelClient searchClient; private final RestHighLevelClient searchClient;
private final int numShards; private final int numShards;
private final int numReplicas; private final int numReplicas;
private final int numRetries;
private static final int NUM_RETRIES = 3;
private static final List<String> SETTINGS_TO_COMPARE = ImmutableList.of("number_of_shards", "number_of_replicas"); private static final List<String> SETTINGS_TO_COMPARE = ImmutableList.of("number_of_shards", "number_of_replicas");
public void buildIndex(String indexName, Map<String, Object> mappings, Map<String, Object> settings) public void buildIndex(String indexName, Map<String, Object> mappings, Map<String, Object> settings)
@ -128,7 +128,7 @@ public class ESIndexBuilder {
// There can be some delay between the reindex finishing and count being fully up to date, so try multiple times // There can be some delay between the reindex finishing and count being fully up to date, so try multiple times
long originalCount = 0; long originalCount = 0;
long reindexedCount = 0; long reindexedCount = 0;
for (int i = 0; i < NUM_RETRIES; i++) { for (int i = 0; i < this.numRetries; i++) {
// Check if reindex succeeded by comparing document counts // Check if reindex succeeded by comparing document counts
originalCount = getCount(indexName); originalCount = getCount(indexName);
reindexedCount = getCount(tempIndexName); reindexedCount = getCount(tempIndexName);

2
metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchServiceTest.java
View File

@ -97,7 +97,7 @@ public class ElasticSearchServiceTest {
} }
public static ESIndexBuilder getIndexBuilder(RestHighLevelClient searchClient) { public static ESIndexBuilder getIndexBuilder(RestHighLevelClient searchClient) {
return new ESIndexBuilder(searchClient, 1, 1); return new ESIndexBuilder(searchClient, 1, 1, 3);
} }
@Nonnull @Nonnull

5
metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/ElasticSearchIndexBuilderFactory.java
View File

@ -28,9 +28,12 @@ public class ElasticSearchIndexBuilderFactory {
@Value("${elasticsearch.index.numReplicas}") @Value("${elasticsearch.index.numReplicas}")
private Integer numReplicas; private Integer numReplicas;
@Value("${elasticsearch.index.numRetries}")
private Integer numRetries;
@Bean(name = "elasticSearchIndexBuilder") @Bean(name = "elasticSearchIndexBuilder")
@Nonnull @Nonnull
protected ESIndexBuilder getInstance() { protected ESIndexBuilder getInstance() {
return new ESIndexBuilder(searchClient, numShards, numReplicas); return new ESIndexBuilder(searchClient, numShards, numReplicas, numRetries);
} }
} }

1
metadata-service/factories/src/main/resources/application.yml
View File

@ -100,6 +100,7 @@ elasticsearch:
prefix: ${INDEX_PREFIX:} prefix: ${INDEX_PREFIX:}
numShards: ${ELASTICSEARCH_NUM_SHARDS_PER_INDEX:1} numShards: ${ELASTICSEARCH_NUM_SHARDS_PER_INDEX:1}
numReplicas: ${ELASTICSEARCH_NUM_REPLICAS_PER_INDEX:1} numReplicas: ${ELASTICSEARCH_NUM_REPLICAS_PER_INDEX:1}
numRetries: ${ELASTICSEARCH_INDEX_BUILDER_NUM_RETRIES :3}
maxArrayLength: ${SEARCH_DOCUMENT_MAX_ARRAY_LENGTH:1000} maxArrayLength: ${SEARCH_DOCUMENT_MAX_ARRAY_LENGTH:1000}
# TODO: Kafka topic convention # TODO: Kafka topic convention