From aa5b9b2985d70001e0ffd25fabf235c646b6578c Mon Sep 17 00:00:00 2001 From: Mohit Yadav <105265192+mohityadav766@users.noreply.github.com> Date: Sun, 11 Aug 2024 21:58:55 +0530 Subject: [PATCH] Increase Payload Size to 100MIB for ELastic Search (#17366) * Increase Payload Size to 100MIB * Increase batchSize Size to 500 * Add Payload Size as option in openmetadataoperations , and searchIndexJob * //Comment * Add Logs --- conf/openmetadata.yaml | 4 ++-- docker/development/docker-compose-gcp.yml | 8 ++++---- docker/development/docker-compose-postgres.yml | 8 ++++---- docker/development/docker-compose.yml | 8 ++++---- .../docker-compose-openmetadata.yml | 8 ++++---- .../docker-compose-postgres.yml | 8 ++++---- .../docker-compose-quickstart/docker-compose.yml | 8 ++++---- .../apps/bundles/searchIndex/SearchIndexApp.java | 3 ++- .../elasticsearch/ElasticSearchIndexSink.java | 7 +++++++ .../service/util/OpenMetadataOperations.java | 15 +++++++++++---- .../json/schema/system/eventPublisherJob.json | 5 +++++ 11 files changed, 51 insertions(+), 31 deletions(-) diff --git a/conf/openmetadata.yaml b/conf/openmetadata.yaml index 2f71591a9ae..08c3fb96281 100644 --- a/conf/openmetadata.yaml +++ b/conf/openmetadata.yaml @@ -284,8 +284,8 @@ elasticsearch: connectionTimeoutSecs: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-5} socketTimeoutSecs: ${ELASTICSEARCH_SOCKET_TIMEOUT_SECS:-60} keepAliveTimeoutSecs: ${ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS:-600} - batchSize: ${ELASTICSEARCH_BATCH_SIZE:-10} - payLoadSize: ${ELASTICSEARCH_PAYLOAD_BYTES_SIZE:-10485760} #max payLoadSize in Bytes + batchSize: ${ELASTICSEARCH_BATCH_SIZE:-500} + payLoadSize: ${ELASTICSEARCH_PAYLOAD_BYTES_SIZE:-104857600} #max payLoadSize in Bytes searchIndexMappingLanguage : ${ELASTICSEARCH_INDEX_MAPPING_LANG:-EN} searchIndexFactoryClassName : org.openmetadata.service.search.SearchIndexFactory diff --git a/docker/development/docker-compose-gcp.yml b/docker/development/docker-compose-gcp.yml index 26094ac07d8..b9b8d581230 100644 --- a/docker/development/docker-compose-gcp.yml +++ b/docker/development/docker-compose-gcp.yml @@ -193,8 +193,8 @@ services: ELASTICSEARCH_CONNECTION_TIMEOUT_SECS: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-5} ELASTICSEARCH_SOCKET_TIMEOUT_SECS: ${ELASTICSEARCH_SOCKET_TIMEOUT_SECS:-60} ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS: ${ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS:-600} - ELASTICSEARCH_BATCH_SIZE: ${ELASTICSEARCH_BATCH_SIZE:-10} - ELASTICSEARCH_PAYLOAD_BYTES_SIZE: ${ELASTICSEARCH_PAYLOAD_BYTES_SIZE:-10485760} #max payLoadSize in Bytes + ELASTICSEARCH_BATCH_SIZE: ${ELASTICSEARCH_BATCH_SIZE:-500} + ELASTICSEARCH_PAYLOAD_BYTES_SIZE: ${ELASTICSEARCH_PAYLOAD_BYTES_SIZE:-104857600} #max payLoadSize in Bytes ELASTICSEARCH_INDEX_MAPPING_LANG: ${ELASTICSEARCH_INDEX_MAPPING_LANG:-EN} #eventMonitoringConfiguration @@ -403,8 +403,8 @@ services: ELASTICSEARCH_CONNECTION_TIMEOUT_SECS: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-5} ELASTICSEARCH_SOCKET_TIMEOUT_SECS: ${ELASTICSEARCH_SOCKET_TIMEOUT_SECS:-60} ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS: ${ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS:-600} - ELASTICSEARCH_BATCH_SIZE: ${ELASTICSEARCH_BATCH_SIZE:-10} - ELASTICSEARCH_PAYLOAD_BYTES_SIZE: ${ELASTICSEARCH_PAYLOAD_BYTES_SIZE:-10485760} #max payLoadSize in Bytes + ELASTICSEARCH_BATCH_SIZE: ${ELASTICSEARCH_BATCH_SIZE:-500} + ELASTICSEARCH_PAYLOAD_BYTES_SIZE: ${ELASTICSEARCH_PAYLOAD_BYTES_SIZE:-104857600} #max payLoadSize in Bytes ELASTICSEARCH_INDEX_MAPPING_LANG: ${ELASTICSEARCH_INDEX_MAPPING_LANG:-EN} #eventMonitoringConfiguration diff --git a/docker/development/docker-compose-postgres.yml b/docker/development/docker-compose-postgres.yml index fc3c6150ca8..f3c5ecce165 100644 --- a/docker/development/docker-compose-postgres.yml +++ b/docker/development/docker-compose-postgres.yml @@ -192,8 +192,8 @@ services: ELASTICSEARCH_CONNECTION_TIMEOUT_SECS: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-5} ELASTICSEARCH_SOCKET_TIMEOUT_SECS: ${ELASTICSEARCH_SOCKET_TIMEOUT_SECS:-60} ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS: ${ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS:-600} - ELASTICSEARCH_BATCH_SIZE: ${ELASTICSEARCH_BATCH_SIZE:-10} - ELASTICSEARCH_PAYLOAD_BYTES_SIZE: ${ELASTICSEARCH_PAYLOAD_BYTES_SIZE:-10485760} #max payLoadSize in Bytes + ELASTICSEARCH_BATCH_SIZE: ${ELASTICSEARCH_BATCH_SIZE:-500} + ELASTICSEARCH_PAYLOAD_BYTES_SIZE: ${ELASTICSEARCH_PAYLOAD_BYTES_SIZE:-104857600} #max payLoadSize in Bytes ELASTICSEARCH_INDEX_MAPPING_LANG: ${ELASTICSEARCH_INDEX_MAPPING_LANG:-EN} #eventMonitoringConfiguration @@ -407,8 +407,8 @@ services: ELASTICSEARCH_CONNECTION_TIMEOUT_SECS: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-5} ELASTICSEARCH_SOCKET_TIMEOUT_SECS: ${ELASTICSEARCH_SOCKET_TIMEOUT_SECS:-60} ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS: ${ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS:-600} - ELASTICSEARCH_BATCH_SIZE: ${ELASTICSEARCH_BATCH_SIZE:-10} - ELASTICSEARCH_PAYLOAD_BYTES_SIZE: ${ELASTICSEARCH_PAYLOAD_BYTES_SIZE:-10485760} #max payLoadSize in Bytes + ELASTICSEARCH_BATCH_SIZE: ${ELASTICSEARCH_BATCH_SIZE:-500} + ELASTICSEARCH_PAYLOAD_BYTES_SIZE: ${ELASTICSEARCH_PAYLOAD_BYTES_SIZE:-104857600} #max payLoadSize in Bytes ELASTICSEARCH_INDEX_MAPPING_LANG: ${ELASTICSEARCH_INDEX_MAPPING_LANG:-EN} #eventMonitoringConfiguration diff --git a/docker/development/docker-compose.yml b/docker/development/docker-compose.yml index 6d07bf21eb6..2d3fd9a3ab2 100644 --- a/docker/development/docker-compose.yml +++ b/docker/development/docker-compose.yml @@ -191,8 +191,8 @@ services: ELASTICSEARCH_CONNECTION_TIMEOUT_SECS: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-5} ELASTICSEARCH_SOCKET_TIMEOUT_SECS: ${ELASTICSEARCH_SOCKET_TIMEOUT_SECS:-60} ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS: ${ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS:-600} - ELASTICSEARCH_BATCH_SIZE: ${ELASTICSEARCH_BATCH_SIZE:-10} - ELASTICSEARCH_PAYLOAD_BYTES_SIZE: ${ELASTICSEARCH_PAYLOAD_BYTES_SIZE:-10485760} #max payLoadSize in Bytes + ELASTICSEARCH_BATCH_SIZE: ${ELASTICSEARCH_BATCH_SIZE:-500} + ELASTICSEARCH_PAYLOAD_BYTES_SIZE: ${ELASTICSEARCH_PAYLOAD_BYTES_SIZE:-104857600} #max payLoadSize in Bytes ELASTICSEARCH_INDEX_MAPPING_LANG: ${ELASTICSEARCH_INDEX_MAPPING_LANG:-EN} #eventMonitoringConfiguration @@ -403,8 +403,8 @@ services: ELASTICSEARCH_CONNECTION_TIMEOUT_SECS: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-5} ELASTICSEARCH_SOCKET_TIMEOUT_SECS: ${ELASTICSEARCH_SOCKET_TIMEOUT_SECS:-60} ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS: ${ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS:-600} - ELASTICSEARCH_BATCH_SIZE: ${ELASTICSEARCH_BATCH_SIZE:-10} - ELASTICSEARCH_PAYLOAD_BYTES_SIZE: ${ELASTICSEARCH_PAYLOAD_BYTES_SIZE:-10485760} #max payLoadSize in Bytes + ELASTICSEARCH_BATCH_SIZE: ${ELASTICSEARCH_BATCH_SIZE:-500} + ELASTICSEARCH_PAYLOAD_BYTES_SIZE: ${ELASTICSEARCH_PAYLOAD_BYTES_SIZE:-104857600} #max payLoadSize in Bytes ELASTICSEARCH_INDEX_MAPPING_LANG: ${ELASTICSEARCH_INDEX_MAPPING_LANG:-EN} #eventMonitoringConfiguration diff --git a/docker/docker-compose-openmetadata/docker-compose-openmetadata.yml b/docker/docker-compose-openmetadata/docker-compose-openmetadata.yml index 6eb130c275f..3716badbd58 100644 --- a/docker/docker-compose-openmetadata/docker-compose-openmetadata.yml +++ b/docker/docker-compose-openmetadata/docker-compose-openmetadata.yml @@ -134,8 +134,8 @@ services: ELASTICSEARCH_CONNECTION_TIMEOUT_SECS: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-5} ELASTICSEARCH_SOCKET_TIMEOUT_SECS: ${ELASTICSEARCH_SOCKET_TIMEOUT_SECS:-60} ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS: ${ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS:-600} - ELASTICSEARCH_BATCH_SIZE: ${ELASTICSEARCH_BATCH_SIZE:-10} - ELASTICSEARCH_PAYLOAD_BYTES_SIZE: ${ELASTICSEARCH_PAYLOAD_BYTES_SIZE:-10485760} #max payLoadSize in Bytes + ELASTICSEARCH_BATCH_SIZE: ${ELASTICSEARCH_BATCH_SIZE:-500} + ELASTICSEARCH_PAYLOAD_BYTES_SIZE: ${ELASTICSEARCH_PAYLOAD_BYTES_SIZE:-104857600} #max payLoadSize in Bytes ELASTICSEARCH_INDEX_MAPPING_LANG: ${ELASTICSEARCH_INDEX_MAPPING_LANG:-EN} #eventMonitoringConfiguration @@ -344,8 +344,8 @@ services: ELASTICSEARCH_CONNECTION_TIMEOUT_SECS: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-5} ELASTICSEARCH_SOCKET_TIMEOUT_SECS: ${ELASTICSEARCH_SOCKET_TIMEOUT_SECS:-60} ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS: ${ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS:-600} - ELASTICSEARCH_BATCH_SIZE: ${ELASTICSEARCH_BATCH_SIZE:-10} - ELASTICSEARCH_PAYLOAD_BYTES_SIZE: ${ELASTICSEARCH_PAYLOAD_BYTES_SIZE:-10485760} #max payLoadSize in Bytes + ELASTICSEARCH_BATCH_SIZE: ${ELASTICSEARCH_BATCH_SIZE:-500} + ELASTICSEARCH_PAYLOAD_BYTES_SIZE: ${ELASTICSEARCH_PAYLOAD_BYTES_SIZE:-104857600} #max payLoadSize in Bytes ELASTICSEARCH_INDEX_MAPPING_LANG: ${ELASTICSEARCH_INDEX_MAPPING_LANG:-EN} #eventMonitoringConfiguration diff --git a/docker/docker-compose-quickstart/docker-compose-postgres.yml b/docker/docker-compose-quickstart/docker-compose-postgres.yml index e8755196ab8..088d2d338b4 100644 --- a/docker/docker-compose-quickstart/docker-compose-postgres.yml +++ b/docker/docker-compose-quickstart/docker-compose-postgres.yml @@ -181,8 +181,8 @@ services: ELASTICSEARCH_CONNECTION_TIMEOUT_SECS: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-5} ELASTICSEARCH_SOCKET_TIMEOUT_SECS: ${ELASTICSEARCH_SOCKET_TIMEOUT_SECS:-60} ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS: ${ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS:-600} - ELASTICSEARCH_BATCH_SIZE: ${ELASTICSEARCH_BATCH_SIZE:-10} - ELASTICSEARCH_PAYLOAD_BYTES_SIZE: ${ELASTICSEARCH_PAYLOAD_BYTES_SIZE:-10485760} #max payLoadSize in Bytes + ELASTICSEARCH_BATCH_SIZE: ${ELASTICSEARCH_BATCH_SIZE:-500} + ELASTICSEARCH_PAYLOAD_BYTES_SIZE: ${ELASTICSEARCH_PAYLOAD_BYTES_SIZE:-104857600} #max payLoadSize in Bytes ELASTICSEARCH_INDEX_MAPPING_LANG: ${ELASTICSEARCH_INDEX_MAPPING_LANG:-EN} #eventMonitoringConfiguration @@ -391,8 +391,8 @@ services: ELASTICSEARCH_CONNECTION_TIMEOUT_SECS: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-5} ELASTICSEARCH_SOCKET_TIMEOUT_SECS: ${ELASTICSEARCH_SOCKET_TIMEOUT_SECS:-60} ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS: ${ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS:-600} - ELASTICSEARCH_BATCH_SIZE: ${ELASTICSEARCH_BATCH_SIZE:-10} - ELASTICSEARCH_PAYLOAD_BYTES_SIZE: ${ELASTICSEARCH_PAYLOAD_BYTES_SIZE:-10485760} #max payLoadSize in Bytes + ELASTICSEARCH_BATCH_SIZE: ${ELASTICSEARCH_BATCH_SIZE:-500} + ELASTICSEARCH_PAYLOAD_BYTES_SIZE: ${ELASTICSEARCH_PAYLOAD_BYTES_SIZE:-104857600} #max payLoadSize in Bytes ELASTICSEARCH_INDEX_MAPPING_LANG: ${ELASTICSEARCH_INDEX_MAPPING_LANG:-EN} #eventMonitoringConfiguration diff --git a/docker/docker-compose-quickstart/docker-compose.yml b/docker/docker-compose-quickstart/docker-compose.yml index 00aa61a0d6f..921fdc9719e 100644 --- a/docker/docker-compose-quickstart/docker-compose.yml +++ b/docker/docker-compose-quickstart/docker-compose.yml @@ -179,8 +179,8 @@ services: ELASTICSEARCH_CONNECTION_TIMEOUT_SECS: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-5} ELASTICSEARCH_SOCKET_TIMEOUT_SECS: ${ELASTICSEARCH_SOCKET_TIMEOUT_SECS:-60} ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS: ${ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS:-600} - ELASTICSEARCH_BATCH_SIZE: ${ELASTICSEARCH_BATCH_SIZE:-10} - ELASTICSEARCH_PAYLOAD_BYTES_SIZE: ${ELASTICSEARCH_PAYLOAD_BYTES_SIZE:-10485760} #max payLoadSize in Bytes + ELASTICSEARCH_BATCH_SIZE: ${ELASTICSEARCH_BATCH_SIZE:-500} + ELASTICSEARCH_PAYLOAD_BYTES_SIZE: ${ELASTICSEARCH_PAYLOAD_BYTES_SIZE:-104857600} #max payLoadSize in Bytes ELASTICSEARCH_INDEX_MAPPING_LANG: ${ELASTICSEARCH_INDEX_MAPPING_LANG:-EN} #eventMonitoringConfiguration @@ -389,8 +389,8 @@ services: ELASTICSEARCH_CONNECTION_TIMEOUT_SECS: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-5} ELASTICSEARCH_SOCKET_TIMEOUT_SECS: ${ELASTICSEARCH_SOCKET_TIMEOUT_SECS:-60} ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS: ${ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS:-600} - ELASTICSEARCH_BATCH_SIZE: ${ELASTICSEARCH_BATCH_SIZE:-10} - ELASTICSEARCH_PAYLOAD_BYTES_SIZE: ${ELASTICSEARCH_PAYLOAD_BYTES_SIZE:-10485760} #max payLoadSize in Bytes + ELASTICSEARCH_BATCH_SIZE: ${ELASTICSEARCH_BATCH_SIZE:-500} + ELASTICSEARCH_PAYLOAD_BYTES_SIZE: ${ELASTICSEARCH_PAYLOAD_BYTES_SIZE:-104857600} #max payLoadSize in Bytes ELASTICSEARCH_INDEX_MAPPING_LANG: ${ELASTICSEARCH_INDEX_MAPPING_LANG:-EN} #eventMonitoringConfiguration diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/SearchIndexApp.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/SearchIndexApp.java index fba0decf9e1..7cf35bb4a15 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/SearchIndexApp.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/SearchIndexApp.java @@ -199,7 +199,8 @@ public class SearchIndexApp extends AbstractNativeApplication { } else { this.entityProcessor = new ElasticSearchEntitiesProcessor(totalRecords); this.entityTimeSeriesProcessor = new ElasticSearchEntityTimeSeriesProcessor(totalRecords); - this.searchIndexSink = new ElasticSearchIndexSink(searchRepository, totalRecords); + this.searchIndexSink = + new ElasticSearchIndexSink(searchRepository, totalRecords, jobData.getPayLoadSize()); } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchIndexSink.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchIndexSink.java index 1ce3162efc5..115b4d6d9cc 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchIndexSink.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchIndexSink.java @@ -36,6 +36,13 @@ public class ElasticSearchIndexSink implements Sink { this.stats.withTotalRecords(total).withSuccessRecords(0).withFailedRecords(0); } + public ElasticSearchIndexSink( + SearchRepository searchRepository, int total, int maxPayLoadSizeInBytes) { + this.searchRepository = searchRepository; + this.maxPayLoadSizeInBytes = maxPayLoadSizeInBytes; + this.stats.withTotalRecords(total).withSuccessRecords(0).withFailedRecords(0); + } + @Override public BulkResponse write(BulkRequest data, Map contextData) throws SearchIndexException { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java index b7fa137d24b..4bfa33e2ee0 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java @@ -268,6 +268,10 @@ public class OpenMetadataOperations implements Callable { names = {"-b", "--batch-size"}, defaultValue = "100") int batchSize, + @Option( + names = {"-p", "--payload-size"}, + defaultValue = "100") + int payloadSize, @Option( names = {"--recreate-indexes"}, defaultValue = "true") @@ -283,14 +287,15 @@ public class OpenMetadataOperations implements Callable { AppScheduler.initialize(config, collectionDAO, searchRepository); String appName = "SearchIndexingApplication"; - return executeSearchReindexApp(appName, batchSize, recreateIndexes); + return executeSearchReindexApp(appName, batchSize, payloadSize, recreateIndexes); } catch (Exception e) { LOG.error("Failed to reindex due to ", e); return 1; } } - private int executeSearchReindexApp(String appName, int batchSize, boolean recreateIndexes) { + private int executeSearchReindexApp( + String appName, int batchSize, int payloadSize, boolean recreateIndexes) { AppRepository appRepository = (AppRepository) Entity.getEntityRepository(Entity.APPLICATION); App originalSearchIndexApp = appRepository.getByName(null, appName, appRepository.getFields("id")); @@ -302,10 +307,11 @@ public class OpenMetadataOperations implements Callable { EventPublisherJob updatedJob = JsonUtils.deepCopy(storedJob, EventPublisherJob.class); updatedJob .withBatchSize(batchSize) + .withPayLoadSize(payloadSize) .withRecreateIndex(recreateIndexes) .withEntities(Set.of("all")); - // Update the search index app with the new batch size and recreate index flag + // Update the search index app with the new batch size, payload size and recreate index flag App updatedSearchIndexApp = JsonUtils.deepCopy(originalSearchIndexApp, App.class); updatedSearchIndexApp.withAppConfiguration(updatedJob); JsonPatch patch = JsonUtils.getJsonPatch(originalSearchIndexApp, updatedSearchIndexApp); @@ -371,7 +377,8 @@ public class OpenMetadataOperations implements Callable { } catch (Exception ignored) { } LOG.info( - "Reindexing Status not available yet, waiting for 10 seconds to fetch the status again."); + "[Reindexing] Current Available Status : {}. Reindexing is still, waiting for 10 seconds to fetch the latest status again.", + JsonUtils.pojoToJson(appRunRecord)); Thread.sleep(10000); } while (!isRunCompleted(appRunRecord)); diff --git a/openmetadata-spec/src/main/resources/json/schema/system/eventPublisherJob.json b/openmetadata-spec/src/main/resources/json/schema/system/eventPublisherJob.json index 5a486859372..c513b0fb2dd 100644 --- a/openmetadata-spec/src/main/resources/json/schema/system/eventPublisherJob.json +++ b/openmetadata-spec/src/main/resources/json/schema/system/eventPublisherJob.json @@ -101,6 +101,11 @@ "description": "Maximum number of events sent in a batch (Default 10).", "type": "integer" }, + "payLoadSize": { + "description": "Payload size in bytes depending on config", + "type": "integer", + "default": 104857600 + }, "searchIndexMappingLanguage": { "description": "Recreate Indexes with updated Language", "$ref": "../configuration/elasticSearchConfiguration.json#/definitions/searchIndexMappingLanguage"