diff --git a/docker/kafka-setup/Dockerfile b/docker/kafka-setup/Dockerfile index 34531413e1..9bb2f57733 100644 --- a/docker/kafka-setup/Dockerfile +++ b/docker/kafka-setup/Dockerfile @@ -56,7 +56,7 @@ ENV METADATA_CHANGE_LOG_TIMESERIES_TOPIC_NAME="MetadataChangeLog_Timeseries_v1" ENV METADATA_CHANGE_PROPOSAL_TOPIC_NAME="MetadataChangeProposal_v1" ENV FAILED_METADATA_CHANGE_PROPOSAL_TOPIC_NAME="FailedMetadataChangeProposal_v1" ENV PLATFORM_EVENT_TOPIC_NAME="PlatformEvent_v1" -ENV DATAHUB_UPGRADE_HISTORY_TOPIC="DataHubUpgradeHistory_v1" +ENV DATAHUB_UPGRADE_HISTORY_TOPIC_NAME="DataHubUpgradeHistory_v1" COPY docker/kafka-setup/kafka-setup.sh ./kafka-setup.sh COPY docker/kafka-setup/kafka-config.sh ./kafka-config.sh diff --git a/docker/kafka-setup/env/docker.env b/docker/kafka-setup/env/docker.env index 0295be1776..22a7e3907b 100644 --- a/docker/kafka-setup/env/docker.env +++ b/docker/kafka-setup/env/docker.env @@ -15,7 +15,7 @@ KAFKA_BOOTSTRAP_SERVER=broker:29092 # FAILED_METADATA_CHANGE_EVENT_NAME=FailedMetadataChangeEvent_v4 # PLATFORM_EVENT_TOPIC_NAME=PlatformEvent_v1 # DATAHUB_USAGE_EVENT_NAME=DataHubUsageEvent_v1 -# DATAHUB_UPGRADE_HISTORY_TOPIC=DataHubUpgradeHistory_v1 +# DATAHUB_UPGRADE_HISTORY_TOPIC_NAME=DataHubUpgradeHistory_v1 # PARTITIONS=1 # REPLICATION_FACTOR=1 diff --git a/docker/kafka-setup/kafka-setup.sh b/docker/kafka-setup/kafka-setup.sh index 92c870b066..e7446b8545 100644 --- a/docker/kafka-setup/kafka-setup.sh +++ b/docker/kafka-setup/kafka-setup.sh @@ -116,7 +116,7 @@ send "$FAILED_METADATA_CHANGE_PROPOSAL_TOPIC_NAME" "--topic $FAILED_METADATA_CHA send "$PLATFORM_EVENT_TOPIC_NAME" "--topic $PLATFORM_EVENT_TOPIC_NAME" # Infinite retention upgrade topic -send "$DATAHUB_UPGRADE_HISTORY_TOPIC" "config retention.ms=-1 --topic $DATAHUB_UPGRADE_HISTORY_TOPIC" +send "$DATAHUB_UPGRADE_HISTORY_TOPIC_NAME" "config retention.ms=-1 --topic $DATAHUB_UPGRADE_HISTORY_TOPIC_NAME" # Create topic for datahub usage event if [[ $DATAHUB_ANALYTICS_ENABLED == true ]]; then send "$DATAHUB_USAGE_EVENT_NAME" "--topic $DATAHUB_USAGE_EVENT_NAME" diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/ESIndexBuilder.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/ESIndexBuilder.java index b59481bf15..8444fadf38 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/ESIndexBuilder.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/ESIndexBuilder.java @@ -439,7 +439,7 @@ public class ESIndexBuilder { .minus(Duration.of(esConfig.getBuildIndices().getRetentionValue(), ChronoUnit.valueOf(esConfig.getBuildIndices().getRetentionUnit())))); - GetIndexResponse response = searchClient.indices().get(new GetIndexRequest(indexState.indexPattern()), RequestOptions.DEFAULT); + GetIndexResponse response = searchClient.indices().get(new GetIndexRequest(indexState.indexCleanPattern()), RequestOptions.DEFAULT); for (String index : response.getIndices()) { var creationDateStr = response.getSetting(index, "index.creation_date"); @@ -456,7 +456,7 @@ public class ESIndexBuilder { } } } catch (Exception e) { - log.info("Failed to get orphaned indices with pattern {}: Exception {}", indexState.indexPattern(), e.toString()); + log.info("Failed to get orphaned indices with pattern {}: Exception {}", indexState.indexCleanPattern(), e.toString()); } return orphanedIndices; } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/ReindexConfig.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/ReindexConfig.java index 63228645e5..e064d61eea 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/ReindexConfig.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/ReindexConfig.java @@ -103,6 +103,10 @@ public class ReindexConfig { return name + "*"; } + public String indexCleanPattern() { + return name + "_*"; + } + private static class CalculatedBuilder extends ReindexConfigBuilder { @Override public ReindexConfig build() {