yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-08-26 09:55:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add Queue Size in Configuration (#18672)

Browse Source
This commit is contained in:
Mohit Yadav 2024-11-18 17:15:08 +05:30 committed by GitHub
parent 6f1df37ba1
commit 66c253ef10
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 21 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java
View File

@ -279,14 +279,19 @@ public class OpenMetadataOperations implements Callable<Integer> {
boolean recreateIndexes,
@Option(
names = {"--producer-threads"},
defaultValue = "10",
defaultValue = "5",
description = "Number of threads to use for processing.")
int producerThreads,
@Option(
names = {"--consumer-threads"},
defaultValue = "10",
defaultValue = "5",
description = "Number of threads to use for processing.")
int consumerThreads,
@Option(
names = {"--queue-size"},
defaultValue = "300",
description = "Queue Size to use internally for reindexing.")
int queueSize,
@Option(
names = {"--back-off"},
defaultValue = "1000",
@ -309,10 +314,17 @@ public class OpenMetadataOperations implements Callable<Integer> {
int retries) {
try {
LOG.info(
"Running Reindexing with Batch Size: {}, Payload Size: {}, Recreate-Index: {}",
"Running Reindexing with Batch Size: {}, Payload Size: {}, Recreate-Index: {}, Producer threads: {}, Consumer threads: {}, Queue Size: {}, Back-off: {}, Max Back-off: {}, Max Requests: {}, Retries: {}",
batchSize,
payloadSize,
recreateIndexes);
recreateIndexes,
producerThreads,
consumerThreads,
queueSize,
backOff,
maxBackOff,
maxRequests,
retries);
parseConfig();
CollectionRegistry.initialize();
ApplicationHandler.initialize(config);
@ -327,6 +339,7 @@ public class OpenMetadataOperations implements Callable<Integer> {
recreateIndexes,
producerThreads,
consumerThreads,
queueSize,
backOff,
maxBackOff,
maxRequests,
@ -344,6 +357,7 @@ public class OpenMetadataOperations implements Callable<Integer> {
boolean recreateIndexes,
int producerThreads,
int consumerThreads,
int queueSize,
int backOff,
int maxBackOff,
int maxRequests,
@ -363,6 +377,7 @@ public class OpenMetadataOperations implements Callable<Integer> {
.withRecreateIndex(recreateIndexes)
.withProducerThreads(producerThreads)
.withConsumerThreads(consumerThreads)
.withQueueSize(queueSize)
.withInitialBackoff(backOff)
.withMaxBackoff(maxBackOff)
.withMaxConcurrentRequests(maxRequests)

2
openmetadata-spec/src/main/resources/json/schema/system/eventPublisherJob.json
View File

@ -122,7 +122,7 @@
},
"queueSize": {
"title": "Queue Size to use.",
"description": "Queue Size to user internally for reindexing.",
"description": "Queue Size to use internally for reindexing.",
"type": "integer",
"default": 100
},

2
openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/SearchIndexingApplication.json
View File

@ -30,7 +30,7 @@
},
"queueSize": {
"title": "Queue Size to use.",
"description": "Queue Size to user internally for reindexing.",
"description": "Queue Size to use internally for reindexing.",
"type": "integer",
"default": 100
},