yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-08-26 18:06:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add Queue Size in Configuration (#18672)

Browse Source
This commit is contained in:
Mohit Yadav 2024-11-18 17:15:08 +05:30 committed by GitHub
parent 6f1df37ba1
commit 66c253ef10
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 21 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java
View File

@ -279,14 +279,19 @@ public class OpenMetadataOperations implements Callable<Integer> {
boolean recreateIndexes, boolean recreateIndexes,
@Option( @Option(
names = {"--producer-threads"}, names = {"--producer-threads"},
defaultValue = "10", defaultValue = "5",
description = "Number of threads to use for processing.") description = "Number of threads to use for processing.")
int producerThreads, int producerThreads,
@Option( @Option(
names = {"--consumer-threads"}, names = {"--consumer-threads"},
defaultValue = "10", defaultValue = "5",
description = "Number of threads to use for processing.") description = "Number of threads to use for processing.")
int consumerThreads, int consumerThreads,
@Option(
names = {"--queue-size"},
defaultValue = "300",
description = "Queue Size to use internally for reindexing.")
int queueSize,
@Option( @Option(
names = {"--back-off"}, names = {"--back-off"},
defaultValue = "1000", defaultValue = "1000",
@ -309,10 +314,17 @@ public class OpenMetadataOperations implements Callable<Integer> {
int retries) { int retries) {
try { try {
LOG.info( LOG.info(
"Running Reindexing with Batch Size: {}, Payload Size: {}, Recreate-Index: {}", "Running Reindexing with Batch Size: {}, Payload Size: {}, Recreate-Index: {}, Producer threads: {}, Consumer threads: {}, Queue Size: {}, Back-off: {}, Max Back-off: {}, Max Requests: {}, Retries: {}",
batchSize, batchSize,
payloadSize, payloadSize,
recreateIndexes); recreateIndexes,
producerThreads,
consumerThreads,
queueSize,
backOff,
maxBackOff,
maxRequests,
retries);
parseConfig(); parseConfig();
CollectionRegistry.initialize(); CollectionRegistry.initialize();
ApplicationHandler.initialize(config); ApplicationHandler.initialize(config);
@ -327,6 +339,7 @@ public class OpenMetadataOperations implements Callable<Integer> {
recreateIndexes, recreateIndexes,
producerThreads, producerThreads,
consumerThreads, consumerThreads,
queueSize,
backOff, backOff,
maxBackOff, maxBackOff,
maxRequests, maxRequests,
@ -344,6 +357,7 @@ public class OpenMetadataOperations implements Callable<Integer> {
boolean recreateIndexes, boolean recreateIndexes,
int producerThreads, int producerThreads,
int consumerThreads, int consumerThreads,
int queueSize,
int backOff, int backOff,
int maxBackOff, int maxBackOff,
int maxRequests, int maxRequests,
@ -363,6 +377,7 @@ public class OpenMetadataOperations implements Callable<Integer> {
.withRecreateIndex(recreateIndexes) .withRecreateIndex(recreateIndexes)
.withProducerThreads(producerThreads) .withProducerThreads(producerThreads)
.withConsumerThreads(consumerThreads) .withConsumerThreads(consumerThreads)
.withQueueSize(queueSize)
.withInitialBackoff(backOff) .withInitialBackoff(backOff)
.withMaxBackoff(maxBackOff) .withMaxBackoff(maxBackOff)
.withMaxConcurrentRequests(maxRequests) .withMaxConcurrentRequests(maxRequests)

2
openmetadata-spec/src/main/resources/json/schema/system/eventPublisherJob.json
View File

@ -122,7 +122,7 @@
}, },
"queueSize": { "queueSize": {
"title": "Queue Size to use.", "title": "Queue Size to use.",
"description": "Queue Size to user internally for reindexing.", "description": "Queue Size to use internally for reindexing.",
"type": "integer", "type": "integer",
"default": 100 "default": 100
}, },

2
openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/SearchIndexingApplication.json
View File

@ -30,7 +30,7 @@
}, },
"queueSize": { "queueSize": {
"title": "Queue Size to use.", "title": "Queue Size to use.",
"description": "Queue Size to user internally for reindexing.", "description": "Queue Size to use internally for reindexing.",
"type": "integer", "type": "integer",
"default": 100 "default": 100
}, },