fix(elasticsearch): Use auth tokens to authorize curl requests in dockerize (#3596)

2025-12-24 08:28:12 +00:00 · 2021-12-08 10:50:08 +09:00 · 2021-12-08 10:50:08 +09:00 · 0576b76eb3
commit 0576b76eb3
parent 44544dfa37
9 changed files with 98 additions and 64 deletions
--- a/docker/datahub-gms/Dockerfile
+++ b/docker/datahub-gms/Dockerfile
@ -14,7 +14,7 @@ RUN apk --no-cache --update-cache --available upgrade \
    else \
      echo >&2 "Unsupported architecture $(arch)" ; exit 1; \
    fi \
-    && apk --no-cache add tar curl openjdk8-jre \
+    && apk --no-cache add tar curl openjdk8-jre bash \
    && curl https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-runner/9.4.20.v20190813/jetty-runner-9.4.20.v20190813.jar --output jetty-runner.jar \
    && curl https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-jmx/9.4.20.v20190813/jetty-jmx-9.4.20.v20190813.jar --output jetty-jmx.jar \
    && curl https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-util/9.4.20.v20190813/jetty-util-9.4.20.v20190813.jar --output jetty-util.jar \
--- a/docker/datahub-gms/start.sh
+++ b/docker/datahub-gms/start.sh
@ -1,18 +1,13 @@
-#!/bin/sh
-
+#!/bin/bash
+set -x
 # Add default URI (http) scheme if needed
 if ! echo $NEO4J_HOST | grep -q "://" ; then
    NEO4J_HOST="http://$NEO4J_HOST"
 fi

-if [[ -z $ELASTICSEARCH_USERNAME ]]; then
-    ELASTICSEARCH_HOST_URL=$ELASTICSEARCH_HOST
-else
-  if [[ -z $ELASTICSEARCH_AUTH_HEADER ]]; then
-    ELASTICSEARCH_HOST_URL=$ELASTICSEARCH_USERNAME:$ELASTICSEARCH_PASSWORD@$ELASTICSEARCH_HOST
-  else
-    ELASTICSEARCH_HOST_URL=$ELASTICSEARCH_HOST
-  fi
+if [[ ! -z $ELASTICSEARCH_USERNAME ]] && [[ -z $ELASTICSEARCH_AUTH_HEADER ]]; then
+  AUTH_TOKEN=$(echo -ne "$ELASTICSEARCH_USERNAME:$ELASTICSEARCH_PASSWORD" | base64 --wrap 0)
+  ELASTICSEARCH_AUTH_HEADER="Authorization:Basic $AUTH_TOKEN"
 fi

 # Add default header if needed
@ -26,9 +21,18 @@ else
  ELASTICSEARCH_PROTOCOL=http
 fi

-WAIT_FOR_NEO4J=""
+WAIT_FOR_EBEAN=""
+if [[ $SKIP_EBEAN_CHECK != true ]]; then
+  WAIT_FOR_EBEAN=" -wait tcp://$EBEAN_DATASOURCE_HOST "
+fi

-if [[ $GRAPH_SERVICE_IMPL != elasticsearch ]]; then
+WAIT_FOR_KAFKA=""
+if [[ $SKIP_KAFKA_CHECK != true ]]; then
+  WAIT_FOR_KAFKA=" -wait tcp://$(echo $KAFKA_BOOTSTRAP_SERVER | sed 's/,/ -wait tcp:\/\//g') "
+fi
+
+WAIT_FOR_NEO4J=""
+if [[ $GRAPH_SERVICE_IMPL != elasticsearch ]] && [[ $SKIP_NEO4J_CHECK != true ]]; then
  WAIT_FOR_NEO4J=" -wait $NEO4J_HOST "
 fi

@ -42,16 +46,23 @@ if [[ $ENABLE_PROMETHEUS == true ]]; then
  PROMETHEUS_AGENT="-javaagent:jmx_prometheus_javaagent.jar=4318:/datahub/datahub-gms/scripts/prometheus-config.yaml "
 fi

-dockerize \
-  -wait tcp://$EBEAN_DATASOURCE_HOST \
-  -wait tcp://$(echo $KAFKA_BOOTSTRAP_SERVER | sed 's/,/ -wait tcp:\/\//g') \
-  -wait $ELASTICSEARCH_PROTOCOL://$ELASTICSEARCH_HOST_URL:$ELASTICSEARCH_PORT -wait-http-header "$ELASTICSEARCH_AUTH_HEADER" \
-  $WAIT_FOR_NEO4J \
-  -timeout 240s \
-  java $JAVA_OPTS $JMX_OPTS \
-  $OTEL_AGENT \
-  $PROMETHEUS_AGENT \
-  -jar /jetty-runner.jar \
-  --jar jetty-util.jar \
-  --jar jetty-jmx.jar \
-  /datahub/datahub-gms/bin/war.war
+COMMON="
+    $WAIT_FOR_EBEAN \
+    $WAIT_FOR_KAFKA \
+    $WAIT_FOR_NEO4J \
+    -timeout 240s \
+    java $JAVA_OPTS $JMX_OPTS \
+    $OTEL_AGENT \
+    $PROMETHEUS_AGENT \
+    -jar /jetty-runner.jar \
+    --jar jetty-util.jar \
+    --jar jetty-jmx.jar \
+    /datahub/datahub-gms/bin/war.war"
+
+if [[ $SKIP_ELASTICSEARCH_CHECK != true ]]; then
+  dockerize \
+    -wait $ELASTICSEARCH_PROTOCOL://$ELASTICSEARCH_HOST:$ELASTICSEARCH_PORT -wait-http-header "$ELASTICSEARCH_AUTH_HEADER" \
+    $COMMON
+else
+  dockerize $COMMON
+fi
--- a/docker/datahub-mae-consumer/Dockerfile
+++ b/docker/datahub-mae-consumer/Dockerfile
@ -3,7 +3,7 @@ ARG APP_ENV=prod

 FROM adoptopenjdk/openjdk8:alpine-jre as base
 ENV DOCKERIZE_VERSION v0.6.1
-RUN apk --no-cache add curl tar wget \
+RUN apk --no-cache add curl tar wget bash \
    && wget https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/download/v1.4.1/opentelemetry-javaagent-all.jar \
    && wget https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.16.1/jmx_prometheus_javaagent-0.16.1.jar -O jmx_prometheus_javaagent.jar \
    && curl -L https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz | tar -C /usr/local/bin -xzv
--- a/docker/datahub-mae-consumer/start.sh
+++ b/docker/datahub-mae-consumer/start.sh
@ -1,18 +1,13 @@
-#!/bin/sh
+#!/bin/bash

 # Add default URI (http) scheme if needed
 if ! echo $NEO4J_HOST | grep -q "://" ; then
    NEO4J_HOST="http://$NEO4J_HOST"
 fi

-if [[ -z $ELASTICSEARCH_USERNAME ]]; then
-    ELASTICSEARCH_HOST_URL=$ELASTICSEARCH_HOST
-else
-  if [[ -z $ELASTICSEARCH_AUTH_HEADER ]]; then
-    ELASTICSEARCH_HOST_URL=$ELASTICSEARCH_USERNAME:$ELASTICSEARCH_PASSWORD@$ELASTICSEARCH_HOST
-  else
-    ELASTICSEARCH_HOST_URL=$ELASTICSEARCH_HOST
-  fi
+if [[ ! -z $ELASTICSEARCH_USERNAME ]] && [[ -z $ELASTICSEARCH_AUTH_HEADER ]]; then
+  AUTH_TOKEN=$(echo -ne "$ELASTICSEARCH_USERNAME:$ELASTICSEARCH_PASSWORD" | base64 --wrap 0)
+  ELASTICSEARCH_AUTH_HEADER="Authorization:Basic $AUTH_TOKEN"
 fi

 # Add default header if needed
@ -26,9 +21,18 @@ else
    ELASTICSEARCH_PROTOCOL=http
 fi

-WAIT_FOR_NEO4J=""
+WAIT_FOR_KAFKA=""
+if [[ $SKIP_KAFKA_CHECK != true ]]; then
+  WAIT_FOR_KAFKA=" -wait tcp://$(echo $KAFKA_BOOTSTRAP_SERVER | sed 's/,/ -wait tcp:\/\//g') "
+fi

-if [[ $GRAPH_SERVICE_IMPL != elasticsearch ]]; then
+WAIT_FOR_ELASTICSEARCH=""
+if [[ $SKIP_ELASTICSEARCH_CHECK != true ]]; then
+  WAIT_FOR_ELASTICSEARCH=" -wait $ELASTICSEARCH_PROTOCOL://$ELASTICSEARCH_HOST:$ELASTICSEARCH_PORT -wait-http-header \"$ELASTICSEARCH_AUTH_HEADER\""
+fi
+
+WAIT_FOR_NEO4J=""
+if [[ $GRAPH_SERVICE_IMPL != elasticsearch ]] && [[ $SKIP_NEO4J_CHECK != true ]]; then
  WAIT_FOR_NEO4J=" -wait $NEO4J_HOST "
 fi

@ -42,9 +46,16 @@ if [[ $ENABLE_PROMETHEUS == true ]]; then
  PROMETHEUS_AGENT="-javaagent:jmx_prometheus_javaagent.jar=4318:/datahub/datahub-mae-consumer/scripts/prometheus-config.yaml "
 fi

-dockerize \
-  -wait tcp://$(echo $KAFKA_BOOTSTRAP_SERVER | sed 's/,/ -wait tcp:\/\//g') \
-  -wait $ELASTICSEARCH_PROTOCOL://$ELASTICSEARCH_HOST_URL:$ELASTICSEARCH_PORT -wait-http-header "$ELASTICSEARCH_AUTH_HEADER" \
-  $WAIT_FOR_NEO4J \
-  -timeout 240s \
-  java $JAVA_OPTS $JMX_OPTS $OTEL_AGENT $PROMETHEUS_AGENT -jar /datahub/datahub-mae-consumer/bin/mae-consumer-job.jar
+COMMON="
+    $WAIT_FOR_KAFKA \
+    $WAIT_FOR_NEO4J \
+    -timeout 240s \
+    java $JAVA_OPTS $JMX_OPTS $OTEL_AGENT $PROMETHEUS_AGENT -jar /datahub/datahub-mae-consumer/bin/mae-consumer-job.jar
+"
+if [[ $SKIP_ELASTICSEARCH_CHECK != true ]]; then
+  dockerize $COMMON
+else
+  dockerize \
+    -wait $ELASTICSEARCH_PROTOCOL://$ELASTICSEARCH_HOST:$ELASTICSEARCH_PORT -wait-http-header "$ELASTICSEARCH_AUTH_HEADER" \
+    $COMMON
+fi
--- a/docker/datahub-mce-consumer/Dockerfile
+++ b/docker/datahub-mce-consumer/Dockerfile
@ -3,7 +3,7 @@ ARG APP_ENV=prod

 FROM adoptopenjdk/openjdk8:alpine-jre as base
 ENV DOCKERIZE_VERSION v0.6.1
-RUN apk --no-cache add curl tar wget \
+RUN apk --no-cache add curl tar wget bash \
    && wget https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/download/v1.4.1/opentelemetry-javaagent-all.jar \
    && wget https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.16.1/jmx_prometheus_javaagent-0.16.1.jar -O jmx_prometheus_javaagent.jar \
    && curl -L https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz | tar -C /usr/local/bin -xzv
--- a/docker/datahub-mce-consumer/start.sh
+++ b/docker/datahub-mce-consumer/start.sh
@ -1,4 +1,9 @@
-#!/bin/sh
+#!/bin/bash
+
+WAIT_FOR_KAFKA=""
+if [[ $SKIP_KAFKA_CHECK != true ]]; then
+  WAIT_FOR_KAFKA=" -wait tcp://$(echo $KAFKA_BOOTSTRAP_SERVER | sed 's/,/ -wait tcp:\/\//g') "
+fi

 OTEL_AGENT=""
 if [[ $ENABLE_OTEL == true ]]; then
@ -11,6 +16,6 @@ if [[ $ENABLE_PROMETHEUS == true ]]; then
 fi

 dockerize \
-  -wait tcp://$(echo $KAFKA_BOOTSTRAP_SERVER | sed 's/,/ -wait tcp:\/\//g') \
+  $WAIT_FOR_KAFKA \
  -timeout 240s \
  java $JAVA_OPTS $JMX_OPTS $OTEL_AGENT $PROMETHEUS_AGENT -jar /datahub/datahub-mce-consumer/bin/mce-consumer-job.jar
--- a/docker/docker-compose.dev.yml
+++ b/docker/docker-compose.dev.yml
@ -37,6 +37,8 @@ services:
      dockerfile: Dockerfile
      args:
        APP_ENV: dev
+    environment:
+      - SKIP_ELASTICSEARCH_CHECK=false
    volumes:
      - ./datahub-gms/start.sh:/datahub/datahub-gms/scripts/start.sh
      - ./monitoring/client-prometheus-config.yaml:/datahub/datahub-gms/scripts/prometheus-config.yaml
--- a/docker/elasticsearch-setup/Dockerfile
+++ b/docker/elasticsearch-setup/Dockerfile
@ -5,7 +5,7 @@ ARG APP_ENV=prod

 FROM alpine:3 as base
 ENV DOCKERIZE_VERSION v0.6.1
-RUN apk add --no-cache curl jq tar \
+RUN apk add --no-cache curl jq tar bash \
    && curl -L https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz | tar -C /usr/local/bin -xzv

 FROM base AS prod-install
@ -21,4 +21,6 @@ FROM base AS dev-install
 FROM ${APP_ENV}-install AS final
 CMD if [ "$ELASTICSEARCH_USE_SSL" == "true" ]; then ELASTICSEARCH_PROTOCOL=https; else ELASTICSEARCH_PROTOCOL=http; fi \
    && if [[ -n "$ELASTICSEARCH_USERNAME" ]]; then ELASTICSEARCH_HTTP_HEADERS="Authorization: Basic $(echo -ne "$ELASTICSEARCH_USERNAME:$ELASTICSEARCH_PASSWORD" | base64)"; else ELASTICSEARCH_HTTP_HEADERS="Accept: */*"; fi \
-    && dockerize -wait $ELASTICSEARCH_PROTOCOL://$ELASTICSEARCH_HOST:$ELASTICSEARCH_PORT -wait-http-header "${ELASTICSEARCH_HTTP_HEADERS}" -timeout 120s /create-indices.sh
+    && if [[ "$SKIP_ELASTICSEARCH_CHECK" != "true" ]]; then \
+        dockerize -wait $ELASTICSEARCH_PROTOCOL://$ELASTICSEARCH_HOST:$ELASTICSEARCH_PORT -wait-http-header "${ELASTICSEARCH_HTTP_HEADERS}" -timeout 120s /create-indices.sh; \
+    else /create-indices.sh; fi
--- a/docker/elasticsearch-setup/create-indices.sh
+++ b/docker/elasticsearch-setup/create-indices.sh
@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash

 set -e

@ -11,10 +11,14 @@ else
    ELASTICSEARCH_PROTOCOL=http
 fi

-if [[ -z $ELASTICSEARCH_USERNAME ]]; then
-    ELASTICSEARCH_HOST_URL=$ELASTICSEARCH_HOST
-else
-    ELASTICSEARCH_HOST_URL=$ELASTICSEARCH_USERNAME:$ELASTICSEARCH_PASSWORD@$ELASTICSEARCH_HOST
+if [[ ! -z $ELASTICSEARCH_USERNAME ]] && [[ -z $ELASTICSEARCH_AUTH_HEADER ]]; then
+  AUTH_TOKEN=$(echo -ne "$ELASTICSEARCH_USERNAME:$ELASTICSEARCH_PASSWORD" | base64 --wrap 0)
+  ELASTICSEARCH_AUTH_HEADER="Authorization:Basic $AUTH_TOKEN"
+fi
+
+# Add default header if needed
+if [[ -z $ELASTICSEARCH_AUTH_HEADER ]]; then
+  ELASTICSEARCH_AUTH_HEADER="Accept: */*"
 fi

 function create_datahub_usage_event_datastream() {
@ -24,19 +28,19 @@ function create_datahub_usage_event_datastream() {
    PREFIX="${INDEX_PREFIX}_"
  fi

-  if [ $(curl -o /dev/null -s -w "%{http_code}" "$ELASTICSEARCH_PROTOCOL://$ELASTICSEARCH_HOST_URL:$ELASTICSEARCH_PORT/_ilm/policy/${PREFIX}datahub_usage_event_policy") -eq 404 ]
+  if [ $(curl -o /dev/null -s -w "%{http_code}" --header "$ELASTICSEARCH_AUTH_HEADER" "$ELASTICSEARCH_PROTOCOL://$ELASTICSEARCH_HOST:$ELASTICSEARCH_PORT/_ilm/policy/${PREFIX}datahub_usage_event_policy") -eq 404 ]
  then
    echo -e "\ncreating datahub_usage_event_policy"
    sed -e "s/PREFIX/${PREFIX}/g" /index/usage-event/policy.json | tee -a /tmp/policy.json
-    curl -XPUT "$ELASTICSEARCH_PROTOCOL://$ELASTICSEARCH_HOST_URL:$ELASTICSEARCH_PORT/_ilm/policy/${PREFIX}datahub_usage_event_policy" -H 'Content-Type: application/json' --data @/tmp/policy.json
+    curl -XPUT --header "$ELASTICSEARCH_AUTH_HEADER" "$ELASTICSEARCH_PROTOCOL://$ELASTICSEARCH_HOST:$ELASTICSEARCH_PORT/_ilm/policy/${PREFIX}datahub_usage_event_policy" -H 'Content-Type: application/json' --data @/tmp/policy.json
  else
    echo -e "\ndatahub_usage_event_policy exists"
  fi
-  if [ $(curl -o /dev/null -s -w "%{http_code}" "$ELASTICSEARCH_PROTOCOL://$ELASTICSEARCH_HOST_URL:$ELASTICSEARCH_PORT/_index_template/${PREFIX}datahub_usage_event_index_template") -eq 404 ]
+  if [ $(curl -o /dev/null -s -w "%{http_code}" --header "$ELASTICSEARCH_AUTH_HEADER" "$ELASTICSEARCH_PROTOCOL://$ELASTICSEARCH_HOST:$ELASTICSEARCH_PORT/_index_template/${PREFIX}datahub_usage_event_index_template") -eq 404 ]
  then
    echo -e "\ncreating datahub_usage_event_index_template"
    sed -e "s/PREFIX/${PREFIX}/g" /index/usage-event/index_template.json | tee -a /tmp/index_template.json
-    curl -XPUT "$ELASTICSEARCH_PROTOCOL://$ELASTICSEARCH_HOST_URL:$ELASTICSEARCH_PORT/_index_template/${PREFIX}datahub_usage_event_index_template" -H 'Content-Type: application/json' --data @/tmp/index_template.json
+    curl -XPUT --header "$ELASTICSEARCH_AUTH_HEADER" "$ELASTICSEARCH_PROTOCOL://$ELASTICSEARCH_HOST:$ELASTICSEARCH_PORT/_index_template/${PREFIX}datahub_usage_event_index_template" -H 'Content-Type: application/json' --data @/tmp/index_template.json
  else
    echo -e "\ndatahub_usage_event_index_template exists"
  fi
@ -49,20 +53,20 @@ function create_datahub_usage_event_aws_elasticsearch() {
    PREFIX="${INDEX_PREFIX}_"
  fi

-  if [ $(curl -o /dev/null -s -w "%{http_code}" "$ELASTICSEARCH_PROTOCOL://$ELASTICSEARCH_HOST_URL:$ELASTICSEARCH_PORT/_opendistro/_ism/policies/${PREFIX}datahub_usage_event_policy") -eq 404 ]
+  if [ $(curl -o /dev/null -s -w "%{http_code}" --header "$ELASTICSEARCH_AUTH_HEADER" "$ELASTICSEARCH_PROTOCOL://$ELASTICSEARCH_HOST:$ELASTICSEARCH_PORT/_opendistro/_ism/policies/${PREFIX}datahub_usage_event_policy") -eq 404 ]
  then
    echo -e "\ncreating datahub_usage_event_policy"
    sed -e "s/PREFIX/${PREFIX}/g" /index/usage-event/aws_es_ism_policy.json | tee -a /tmp/aws_es_ism_policy.json
-    curl -XPUT "$ELASTICSEARCH_PROTOCOL://$ELASTICSEARCH_HOST_URL:$ELASTICSEARCH_PORT/_opendistro/_ism/policies/${PREFIX}datahub_usage_event_policy" -H 'Content-Type: application/json' --data @/tmp/aws_es_ism_policy.json
+    curl -XPUT --header "$ELASTICSEARCH_AUTH_HEADER" "$ELASTICSEARCH_PROTOCOL://$ELASTICSEARCH_HOST:$ELASTICSEARCH_PORT/_opendistro/_ism/policies/${PREFIX}datahub_usage_event_policy" -H 'Content-Type: application/json' --data @/tmp/aws_es_ism_policy.json
  else
    echo -e "\ndatahub_usage_event_policy exists"
  fi
-  if [ $(curl -o /dev/null -s -w "%{http_code}" "$ELASTICSEARCH_PROTOCOL://$ELASTICSEARCH_HOST_URL:$ELASTICSEARCH_PORT/_template/${PREFIX}datahub_usage_event_index_template") -eq 404 ]
+  if [ $(curl -o /dev/null -s -w "%{http_code}" --header "$ELASTICSEARCH_AUTH_HEADER" "$ELASTICSEARCH_PROTOCOL://$ELASTICSEARCH_HOST:$ELASTICSEARCH_PORT/_template/${PREFIX}datahub_usage_event_index_template") -eq 404 ]
  then
    echo -e "\ncreating datahub_usagAe_event_index_template"
    sed -e "s/PREFIX/${PREFIX}/g" /index/usage-event/aws_es_index_template.json | tee -a /tmp/aws_es_index_template.json
-    curl -XPUT "$ELASTICSEARCH_PROTOCOL://$ELASTICSEARCH_HOST_URL:$ELASTICSEARCH_PORT/_template/${PREFIX}datahub_usage_event_index_template" -H 'Content-Type: application/json' --data @/tmp/aws_es_index_template.json
-    curl -XPUT "$ELASTICSEARCH_PROTOCOL://$ELASTICSEARCH_HOST_URL:$ELASTICSEARCH_PORT/${PREFIX}datahub_usage_event-000001"  -H 'Content-Type: application/json' --data "{\"aliases\":{\"${PREFIX}datahub_usage_event\":{\"is_write_index\":true}}}"
+    curl -XPUT --header "$ELASTICSEARCH_AUTH_HEADER" "$ELASTICSEARCH_PROTOCOL://$ELASTICSEARCH_HOST:$ELASTICSEARCH_PORT/_template/${PREFIX}datahub_usage_event_index_template" -H 'Content-Type: application/json' --data @/tmp/aws_es_index_template.json
+    curl -XPUT --header "$ELASTICSEARCH_AUTH_HEADER" "$ELASTICSEARCH_PROTOCOL://$ELASTICSEARCH_HOST:$ELASTICSEARCH_PORT/${PREFIX}datahub_usage_event-000001"  -H 'Content-Type: application/json' --data "{\"aliases\":{\"${PREFIX}datahub_usage_event\":{\"is_write_index\":true}}}"
  else
    echo -e "\ndatahub_usage_event_index_template exists"
  fi
@ -75,4 +79,3 @@ if [[ $DATAHUB_ANALYTICS_ENABLED == true ]]; then
    create_datahub_usage_event_aws_elasticsearch || exit 1
  fi
 fi
-