yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-11-01 03:09:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(kafka-setup): add option for SSL and topic partition config via environment (#2398)

Browse Source
This commit is contained in:
G-nther 2021-04-16 18:40:39 +02:00 committed by GitHub
parent 379c2e1c0c
commit 177161a850
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 41 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
docker/kafka-setup/Dockerfile
View File

@ -5,8 +5,6 @@ ENV METADATA_AUDIT_EVENT_NAME="MetadataAuditEvent_v4"
ENV METADATA_CHANGE_EVENT_NAME="MetadataChangeEvent_v4"
ENV FAILED_METADATA_CHANGE_EVENT_NAME="FailedMetadataChangeEvent_v4"
CMD echo Waiting for Kafka to be ready... && \
cub kafka-ready -b $KAFKA_BOOTSTRAP_SERVER 1 60 && \
kafka-topics --create --if-not-exists --zookeeper $KAFKA_ZOOKEEPER_CONNECT --partitions 1 --replication-factor 1 --topic $METADATA_AUDIT_EVENT_NAME && \
kafka-topics --create --if-not-exists --zookeeper $KAFKA_ZOOKEEPER_CONNECT --partitions 1 --replication-factor 1 --topic $METADATA_CHANGE_EVENT_NAME && \
kafka-topics --create --if-not-exists --zookeeper $KAFKA_ZOOKEEPER_CONNECT --partitions 1 --replication-factor 1 --topic $FAILED_METADATA_CHANGE_EVENT_NAME
COPY ./kafka-setup.sh ./kafka-setup.sh
CMD ./kafka-setup.sh

16
docker/kafka-setup/env/docker.env vendored
View File

@ -1,2 +1,18 @@
KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
KAFKA_BOOTSTRAP_SERVER=broker:29092
# Configure the topics that are created by kafka-setup
# METADATA_AUDIT_EVENT_NAME=MetadataAuditEvent_v4
# METADATA_CHANGE_EVENT_NAME=MetadataChangeEvent_v4
# FAILED_METADATA_CHANGE_EVENT_NAME=FailedMetadataChangeEvent_v4
# PARTITIONS=1
# REPLICATION_FACTOR=1
# Configure for an SSL-Connection to Kafka
# KAFKA_PROPERTIES_SECURITY_PROTOCOL=SSL
# KAFKA_PROPERTIES_SSL_KEYSTORE_LOCATION=
# KAFKA_PROPERTIES_SSL_KEYSTORE_PASSWORD=
# KAFKA_PROPERTIES_SSL_KEY_PASSWORD=
# KAFKA_PROPERTIES_SSL_TRUSTSTORE_LOCATION=
# KAFKA_PROPERTIES_SSL_TRUSTSTORE_PASSWORD=
# KAFKA_PROPERTIES_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM=

22
docker/kafka-setup/kafka-setup.sh Executable file
View File

@ -0,0 +1,22 @@
#!/bin/bash
: ${PARTITIONS:=1}
: ${REPLICATION_FACTOR:=1}
: ${KAFKA_PROPERTIES_SECURITY_PROTOCOL:=PLAINTEXT}
echo "bootstrap.servers=$KAFKA_BOOTSTRAP_SERVER" > connection.properties
echo "security.protocol=$KAFKA_PROPERTIES_SECURITY_PROTOCOL" >> connection.properties
if [[ $KAFKA_PROPERTIES_SECURITY_PROTOCOL == "SSL" ]]; then
echo "ssl.keystore.location=$KAFKA_PROPERTIES_SSL_KEYSTORE_LOCATION" >> connection.properties
echo "ssl.keystore.password=$KAFKA_PROPERTIES_SSL_KEYSTORE_PASSWORD" >> connection.properties
echo "ssl.key.password=$KAFKA_PROPERTIES_SSL_KEY_PASSWORD" >> connection.properties
echo "ssl.truststore.location=$KAFKA_PROPERTIES_SSL_TRUSTSTORE_LOCATION" >> connection.properties
echo "ssl.truststore.password=$KAFKA_PROPERTIES_SSL_TRUSTSTORE_PASSWORD" >> connection.properties
echo "ssl.endpoint.identification.algorithm=$KAFKA_PROPERTIES_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM" >> connection.properties
fi
cub kafka-ready -c connection.properties -b $KAFKA_BOOTSTRAP_SERVER 1 60 && \
kafka-topics --create --if-not-exists --command-config connection.properties --zookeeper $KAFKA_ZOOKEEPER_CONNECT --partitions $PARTITIONS --replication-factor $REPLICATION_FACTOR --topic $METADATA_AUDIT_EVENT_NAME && \
kafka-topics --create --if-not-exists --command-config connection.properties --zookeeper $KAFKA_ZOOKEEPER_CONNECT --partitions $PARTITIONS --replication-factor $REPLICATION_FACTOR --topic $METADATA_CHANGE_EVENT_NAME && \
kafka-topics --create --if-not-exists --command-config connection.properties --zookeeper $KAFKA_ZOOKEEPER_CONNECT --partitions $PARTITIONS --replication-factor $REPLICATION_FACTOR --topic $FAILED_METADATA_CHANGE_EVENT_NAME