yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-08-17 13:45:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(elasticsearch): fix orphan index clean up pattern, consistent top… (#7242)

Browse Source
This commit is contained in:
david-leifker 2023-02-03 07:09:19 -06:00 committed by GitHub
parent d2bcdc301a
commit d557fd65ff
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 9 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docker/kafka-setup/Dockerfile
View File

@ -56,7 +56,7 @@ ENV METADATA_CHANGE_LOG_TIMESERIES_TOPIC_NAME="MetadataChangeLog_Timeseries_v1"
ENV METADATA_CHANGE_PROPOSAL_TOPIC_NAME="MetadataChangeProposal_v1" ENV METADATA_CHANGE_PROPOSAL_TOPIC_NAME="MetadataChangeProposal_v1"
ENV FAILED_METADATA_CHANGE_PROPOSAL_TOPIC_NAME="FailedMetadataChangeProposal_v1" ENV FAILED_METADATA_CHANGE_PROPOSAL_TOPIC_NAME="FailedMetadataChangeProposal_v1"
ENV PLATFORM_EVENT_TOPIC_NAME="PlatformEvent_v1" ENV PLATFORM_EVENT_TOPIC_NAME="PlatformEvent_v1"
ENV DATAHUB_UPGRADE_HISTORY_TOPIC="DataHubUpgradeHistory_v1" ENV DATAHUB_UPGRADE_HISTORY_TOPIC_NAME="DataHubUpgradeHistory_v1"
COPY docker/kafka-setup/kafka-setup.sh ./kafka-setup.sh COPY docker/kafka-setup/kafka-setup.sh ./kafka-setup.sh
COPY docker/kafka-setup/kafka-config.sh ./kafka-config.sh COPY docker/kafka-setup/kafka-config.sh ./kafka-config.sh

2
docker/kafka-setup/env/docker.env vendored
View File

@ -15,7 +15,7 @@ KAFKA_BOOTSTRAP_SERVER=broker:29092
# FAILED_METADATA_CHANGE_EVENT_NAME=FailedMetadataChangeEvent_v4 # FAILED_METADATA_CHANGE_EVENT_NAME=FailedMetadataChangeEvent_v4
# PLATFORM_EVENT_TOPIC_NAME=PlatformEvent_v1 # PLATFORM_EVENT_TOPIC_NAME=PlatformEvent_v1
# DATAHUB_USAGE_EVENT_NAME=DataHubUsageEvent_v1 # DATAHUB_USAGE_EVENT_NAME=DataHubUsageEvent_v1
# DATAHUB_UPGRADE_HISTORY_TOPIC=DataHubUpgradeHistory_v1 # DATAHUB_UPGRADE_HISTORY_TOPIC_NAME=DataHubUpgradeHistory_v1
# PARTITIONS=1 # PARTITIONS=1
# REPLICATION_FACTOR=1 # REPLICATION_FACTOR=1

2
docker/kafka-setup/kafka-setup.sh
View File

@ -116,7 +116,7 @@ send "$FAILED_METADATA_CHANGE_PROPOSAL_TOPIC_NAME" "--topic $FAILED_METADATA_CHA
send "$PLATFORM_EVENT_TOPIC_NAME" "--topic $PLATFORM_EVENT_TOPIC_NAME" send "$PLATFORM_EVENT_TOPIC_NAME" "--topic $PLATFORM_EVENT_TOPIC_NAME"
# Infinite retention upgrade topic # Infinite retention upgrade topic
send "$DATAHUB_UPGRADE_HISTORY_TOPIC" "config retention.ms=-1 --topic $DATAHUB_UPGRADE_HISTORY_TOPIC" send "$DATAHUB_UPGRADE_HISTORY_TOPIC_NAME" "config retention.ms=-1 --topic $DATAHUB_UPGRADE_HISTORY_TOPIC_NAME"
# Create topic for datahub usage event # Create topic for datahub usage event
if [[ $DATAHUB_ANALYTICS_ENABLED == true ]]; then if [[ $DATAHUB_ANALYTICS_ENABLED == true ]]; then
send "$DATAHUB_USAGE_EVENT_NAME" "--topic $DATAHUB_USAGE_EVENT_NAME" send "$DATAHUB_USAGE_EVENT_NAME" "--topic $DATAHUB_USAGE_EVENT_NAME"

4
metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/ESIndexBuilder.java
View File

@ -439,7 +439,7 @@ public class ESIndexBuilder {
.minus(Duration.of(esConfig.getBuildIndices().getRetentionValue(), .minus(Duration.of(esConfig.getBuildIndices().getRetentionValue(),
ChronoUnit.valueOf(esConfig.getBuildIndices().getRetentionUnit())))); ChronoUnit.valueOf(esConfig.getBuildIndices().getRetentionUnit()))));
GetIndexResponse response = searchClient.indices().get(new GetIndexRequest(indexState.indexPattern()), RequestOptions.DEFAULT); GetIndexResponse response = searchClient.indices().get(new GetIndexRequest(indexState.indexCleanPattern()), RequestOptions.DEFAULT);
for (String index : response.getIndices()) { for (String index : response.getIndices()) {
var creationDateStr = response.getSetting(index, "index.creation_date"); var creationDateStr = response.getSetting(index, "index.creation_date");
@ -456,7 +456,7 @@ public class ESIndexBuilder {
} }
} }
} catch (Exception e) { } catch (Exception e) {
log.info("Failed to get orphaned indices with pattern {}: Exception {}", indexState.indexPattern(), e.toString()); log.info("Failed to get orphaned indices with pattern {}: Exception {}", indexState.indexCleanPattern(), e.toString());
} }
return orphanedIndices; return orphanedIndices;
} }

4
metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/ReindexConfig.java
View File

@ -103,6 +103,10 @@ public class ReindexConfig {
return name + "*"; return name + "*";
} }
public String indexCleanPattern() {
return name + "_*";
}
private static class CalculatedBuilder extends ReindexConfigBuilder { private static class CalculatedBuilder extends ReindexConfigBuilder {
@Override @Override
public ReindexConfig build() { public ReindexConfig build() {