mirror of
https://github.com/datahub-project/datahub.git
synced 2025-06-27 05:03:31 +00:00
feat(dev): Make repositories configurable for enterprise developers (#9230)
Co-authored-by: Hendrik Richert <hendrik.richert@swisscom.com> Co-authored-by: david-leifker <114954101+david-leifker@users.noreply.github.com>
This commit is contained in:
parent
08fb730676
commit
966cb175f7
@ -89,6 +89,22 @@ docker {
|
|||||||
buildx(true)
|
buildx(true)
|
||||||
load(true)
|
load(true)
|
||||||
push(false)
|
push(false)
|
||||||
|
|
||||||
|
// Add build args if they are defined (needed for some CI or enterprise environments)
|
||||||
|
def dockerBuildArgs = [:]
|
||||||
|
if (project.hasProperty('alpineApkRepositoryUrl')) {
|
||||||
|
dockerBuildArgs.ALPINE_REPO_URL = project.getProperty('alpineApkRepositoryUrl')
|
||||||
|
}
|
||||||
|
if (project.hasProperty('githubMirrorUrl')) {
|
||||||
|
dockerBuildArgs.GITHUB_REPO_URL = project.getProperty('githubMirrorUrl')
|
||||||
|
}
|
||||||
|
if (project.hasProperty('mavenCentralRepositoryUrl')) {
|
||||||
|
dockerBuildArgs.MAVEN_CENTRAL_REPO_URL = project.getProperty('mavenCentralRepositoryUrl')
|
||||||
|
}
|
||||||
|
|
||||||
|
if (dockerBuildArgs.size() > 0) {
|
||||||
|
buildArgs(dockerBuildArgs)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
task unversionZip(type: Copy, dependsOn: [':datahub-web-react:build', dist]) {
|
task unversionZip(type: Copy, dependsOn: [':datahub-web-react:build', dist]) {
|
||||||
|
@ -108,6 +108,22 @@ docker {
|
|||||||
buildx(true)
|
buildx(true)
|
||||||
load(true)
|
load(true)
|
||||||
push(false)
|
push(false)
|
||||||
|
|
||||||
|
// Add build args if they are defined (needed for some CI or enterprise environments)
|
||||||
|
def dockerBuildArgs = [:]
|
||||||
|
if (project.hasProperty('alpineApkRepositoryUrl')) {
|
||||||
|
dockerBuildArgs.ALPINE_REPO_URL = project.getProperty('alpineApkRepositoryUrl')
|
||||||
|
}
|
||||||
|
if (project.hasProperty('githubMirrorUrl')) {
|
||||||
|
dockerBuildArgs.GITHUB_REPO_URL = project.getProperty('githubMirrorUrl')
|
||||||
|
}
|
||||||
|
if (project.hasProperty('mavenCentralRepositoryUrl')) {
|
||||||
|
dockerBuildArgs.MAVEN_CENTRAL_REPO_URL = project.getProperty('mavenCentralRepositoryUrl')
|
||||||
|
}
|
||||||
|
|
||||||
|
if (dockerBuildArgs.size() > 0) {
|
||||||
|
buildArgs(dockerBuildArgs)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
tasks.getByPath(":datahub-upgrade:docker").dependsOn([bootJar])
|
tasks.getByPath(":datahub-upgrade:docker").dependsOn([bootJar])
|
||||||
|
|
||||||
|
@ -3,14 +3,22 @@ ARG APP_ENV=prod
|
|||||||
|
|
||||||
FROM alpine:3 AS base
|
FROM alpine:3 AS base
|
||||||
|
|
||||||
|
# Configurable repositories
|
||||||
|
ARG ALPINE_REPO_URL=http://dl-cdn.alpinelinux.org/alpine
|
||||||
|
ARG GITHUB_REPO_URL=https://github.com
|
||||||
|
ARG MAVEN_CENTRAL_REPO_URL=https://repo1.maven.org/maven2
|
||||||
|
|
||||||
RUN addgroup -S datahub && adduser -S datahub -G datahub
|
RUN addgroup -S datahub && adduser -S datahub -G datahub
|
||||||
|
|
||||||
|
# Optionally set corporate mirror for apk
|
||||||
|
RUN if [ "${ALPINE_REPO_URL}" != "http://dl-cdn.alpinelinux.org/alpine" ] ; then sed -i "s#http.*://dl-cdn.alpinelinux.org/alpine#${ALPINE_REPO_URL}#g" /etc/apk/repositories ; fi
|
||||||
|
|
||||||
# Upgrade Alpine and base packages
|
# Upgrade Alpine and base packages
|
||||||
# PFP-260: Upgrade Sqlite to >=3.28.0-r0 to fix https://security.snyk.io/vuln/SNYK-ALPINE39-SQLITE-449762
|
# PFP-260: Upgrade Sqlite to >=3.28.0-r0 to fix https://security.snyk.io/vuln/SNYK-ALPINE39-SQLITE-449762
|
||||||
RUN apk --no-cache --update-cache --available upgrade \
|
RUN apk --no-cache --update-cache --available upgrade \
|
||||||
&& apk --no-cache add curl sqlite libc6-compat java-snappy \
|
&& apk --no-cache add curl sqlite libc6-compat java-snappy \
|
||||||
&& apk --no-cache add openjdk11-jre-headless --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community \
|
&& apk --no-cache add openjdk11-jre-headless --repository=${ALPINE_REPO_URL}/edge/community \
|
||||||
&& apk --no-cache add jattach --repository http://dl-cdn.alpinelinux.org/alpine/edge/community/
|
&& apk --no-cache add jattach --repository ${ALPINE_REPO_URL}/edge/community/
|
||||||
|
|
||||||
ENV LD_LIBRARY_PATH="/lib:/lib64"
|
ENV LD_LIBRARY_PATH="/lib:/lib64"
|
||||||
|
|
||||||
@ -22,8 +30,8 @@ COPY ./docker/monitoring/client-prometheus-config.yaml /datahub-frontend/
|
|||||||
RUN chown -R datahub:datahub /datahub-frontend && chmod 755 /datahub-frontend
|
RUN chown -R datahub:datahub /datahub-frontend && chmod 755 /datahub-frontend
|
||||||
|
|
||||||
ENV JMX_VERSION=0.18.0
|
ENV JMX_VERSION=0.18.0
|
||||||
RUN wget https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/download/v1.24.0/opentelemetry-javaagent.jar -O opentelemetry-javaagent.jar \
|
RUN wget ${GITHUB_REPO_URL}/open-telemetry/opentelemetry-java-instrumentation/releases/download/v1.24.0/opentelemetry-javaagent.jar -O opentelemetry-javaagent.jar \
|
||||||
&& wget https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/${JMX_VERSION}/jmx_prometheus_javaagent-${JMX_VERSION}.jar -O jmx_prometheus_javaagent.jar
|
&& wget ${MAVEN_CENTRAL_REPO_URL}/io/prometheus/jmx/jmx_prometheus_javaagent/${JMX_VERSION}/jmx_prometheus_javaagent-${JMX_VERSION}.jar -O jmx_prometheus_javaagent.jar
|
||||||
|
|
||||||
FROM base as dev-install
|
FROM base as dev-install
|
||||||
# Dummy stage for development. Assumes code is built on your machine and mounted to this image.
|
# Dummy stage for development. Assumes code is built on your machine and mounted to this image.
|
||||||
|
@ -1,11 +1,23 @@
|
|||||||
# Defining environment
|
# Defining environment
|
||||||
ARG APP_ENV=prod
|
ARG APP_ENV=prod
|
||||||
|
|
||||||
|
# Defining custom repo urls for use in enterprise environments. Re-used between stages below.
|
||||||
|
ARG ALPINE_REPO_URL=http://dl-cdn.alpinelinux.org/alpine
|
||||||
|
ARG GITHUB_REPO_URL=https://github.com
|
||||||
|
ARG MAVEN_CENTRAL_REPO_URL=https://repo1.maven.org/maven2
|
||||||
|
|
||||||
FROM golang:1-alpine3.18 AS binary
|
FROM golang:1-alpine3.18 AS binary
|
||||||
|
FROM golang:1-alpine3.18 AS binary
|
||||||
|
|
||||||
|
# Re-declaring arg from above to make it available in this stage (will inherit default value)
|
||||||
|
ARG ALPINE_REPO_URL
|
||||||
|
|
||||||
ENV DOCKERIZE_VERSION v0.6.1
|
ENV DOCKERIZE_VERSION v0.6.1
|
||||||
WORKDIR /go/src/github.com/jwilder
|
WORKDIR /go/src/github.com/jwilder
|
||||||
|
|
||||||
|
# Optionally set corporate mirror for apk
|
||||||
|
RUN if [ "${ALPINE_REPO_URL}" != "http://dl-cdn.alpinelinux.org/alpine" ] ; then sed -i "s#http.*://dl-cdn.alpinelinux.org/alpine#${ALPINE_REPO_URL}#g" /etc/apk/repositories ; fi
|
||||||
|
|
||||||
RUN apk --no-cache --update add openssl git tar curl
|
RUN apk --no-cache --update add openssl git tar curl
|
||||||
|
|
||||||
WORKDIR /go/src/github.com/jwilder/dockerize
|
WORKDIR /go/src/github.com/jwilder/dockerize
|
||||||
@ -16,16 +28,25 @@ FROM alpine:3 AS base
|
|||||||
|
|
||||||
# Upgrade Alpine and base packages
|
# Upgrade Alpine and base packages
|
||||||
ENV JMX_VERSION=0.18.0
|
ENV JMX_VERSION=0.18.0
|
||||||
|
|
||||||
|
# Re-declaring args from above to make them available in this stage (will inherit default values)
|
||||||
|
ARG ALPINE_REPO_URL
|
||||||
|
ARG GITHUB_REPO_URL
|
||||||
|
ARG MAVEN_CENTRAL_REPO_URL
|
||||||
|
|
||||||
|
# Optionally set corporate mirror for apk
|
||||||
|
RUN if [ "${ALPINE_REPO_URL}" != "http://dl-cdn.alpinelinux.org/alpine" ] ; then sed -i "s#http.*://dl-cdn.alpinelinux.org/alpine#${ALPINE_REPO_URL}#g" /etc/apk/repositories ; fi
|
||||||
|
|
||||||
# PFP-260: Upgrade Sqlite to >=3.28.0-r0 to fix https://security.snyk.io/vuln/SNYK-ALPINE39-SQLITE-449762
|
# PFP-260: Upgrade Sqlite to >=3.28.0-r0 to fix https://security.snyk.io/vuln/SNYK-ALPINE39-SQLITE-449762
|
||||||
RUN apk --no-cache --update-cache --available upgrade \
|
RUN apk --no-cache --update-cache --available upgrade \
|
||||||
&& apk --no-cache add curl bash coreutils gcompat sqlite libc6-compat java-snappy \
|
&& apk --no-cache add curl bash coreutils gcompat sqlite libc6-compat java-snappy \
|
||||||
&& apk --no-cache add openjdk11-jre-headless --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community \
|
&& apk --no-cache add openjdk11-jre-headless --repository=${ALPINE_REPO_URL}/edge/community \
|
||||||
&& apk --no-cache add jattach --repository http://dl-cdn.alpinelinux.org/alpine/edge/community/ \
|
&& apk --no-cache add jattach --repository ${ALPINE_REPO_URL}/edge/community/ \
|
||||||
&& curl -sS https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-runner/9.4.46.v20220331/jetty-runner-9.4.46.v20220331.jar --output jetty-runner.jar \
|
&& curl -sS ${MAVEN_CENTRAL_REPO_URL}/org/eclipse/jetty/jetty-runner/9.4.46.v20220331/jetty-runner-9.4.46.v20220331.jar --output jetty-runner.jar \
|
||||||
&& curl -sS https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-jmx/9.4.46.v20220331/jetty-jmx-9.4.46.v20220331.jar --output jetty-jmx.jar \
|
&& curl -sS ${MAVEN_CENTRAL_REPO_URL}/org/eclipse/jetty/jetty-jmx/9.4.46.v20220331/jetty-jmx-9.4.46.v20220331.jar --output jetty-jmx.jar \
|
||||||
&& curl -sS https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-util/9.4.46.v20220331/jetty-util-9.4.46.v20220331.jar --output jetty-util.jar \
|
&& curl -sS ${MAVEN_CENTRAL_REPO_URL}/org/eclipse/jetty/jetty-util/9.4.46.v20220331/jetty-util-9.4.46.v20220331.jar --output jetty-util.jar \
|
||||||
&& wget --no-verbose https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/download/v1.24.0/opentelemetry-javaagent.jar \
|
&& wget --no-verbose ${GITHUB_REPO_URL}/open-telemetry/opentelemetry-java-instrumentation/releases/download/v1.24.0/opentelemetry-javaagent.jar \
|
||||||
&& wget --no-verbose https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/${JMX_VERSION}/jmx_prometheus_javaagent-${JMX_VERSION}.jar -O jmx_prometheus_javaagent.jar \
|
&& wget --no-verbose ${MAVEN_CENTRAL_REPO_URL}/io/prometheus/jmx/jmx_prometheus_javaagent/${JMX_VERSION}/jmx_prometheus_javaagent-${JMX_VERSION}.jar -O jmx_prometheus_javaagent.jar \
|
||||||
&& cp /usr/lib/jvm/java-11-openjdk/jre/lib/security/cacerts /tmp/kafka.client.truststore.jks
|
&& cp /usr/lib/jvm/java-11-openjdk/jre/lib/security/cacerts /tmp/kafka.client.truststore.jks
|
||||||
COPY --from=binary /go/bin/dockerize /usr/local/bin
|
COPY --from=binary /go/bin/dockerize /usr/local/bin
|
||||||
|
|
||||||
|
@ -1,11 +1,23 @@
|
|||||||
ARG APP_ENV=full
|
ARG APP_ENV=full
|
||||||
ARG BASE_IMAGE=base
|
ARG BASE_IMAGE=base
|
||||||
|
|
||||||
|
# Defining custom repo urls for use in enterprise environments. Re-used between stages below.
|
||||||
|
ARG ALPINE_REPO_URL=http://dl-cdn.alpinelinux.org/alpine
|
||||||
|
ARG GITHUB_REPO_URL=https://github.com
|
||||||
|
ARG DEBIAN_REPO_URL=http://deb.debian.org/debian
|
||||||
|
ARG PIP_MIRROR_URL=null
|
||||||
|
|
||||||
FROM golang:1-alpine3.18 AS dockerize-binary
|
FROM golang:1-alpine3.18 AS dockerize-binary
|
||||||
|
|
||||||
|
# Re-declaring arg from above to make it available in this stage (will inherit default value)
|
||||||
|
ARG ALPINE_REPO_URL
|
||||||
|
|
||||||
ENV DOCKERIZE_VERSION v0.6.1
|
ENV DOCKERIZE_VERSION v0.6.1
|
||||||
WORKDIR /go/src/github.com/jwilder
|
WORKDIR /go/src/github.com/jwilder
|
||||||
|
|
||||||
|
# Optionally set corporate mirror for apk
|
||||||
|
RUN if [ "${ALPINE_REPO_URL}" != "http://dl-cdn.alpinelinux.org/alpine" ] ; then sed -i "s#http.*://dl-cdn.alpinelinux.org/alpine#${ALPINE_REPO_URL}#g" /etc/apk/repositories ; fi
|
||||||
|
|
||||||
RUN apk --no-cache --update add openssl git tar curl
|
RUN apk --no-cache --update add openssl git tar curl
|
||||||
|
|
||||||
WORKDIR /go/src/github.com/jwilder/dockerize
|
WORKDIR /go/src/github.com/jwilder/dockerize
|
||||||
@ -14,11 +26,19 @@ RUN go install github.com/jwilder/dockerize@$DOCKERIZE_VERSION
|
|||||||
|
|
||||||
FROM python:3.10 as base
|
FROM python:3.10 as base
|
||||||
|
|
||||||
|
ARG DEBIAN_REPO_URL
|
||||||
|
ARG PIP_MIRROR_URL
|
||||||
|
ARG GITHUB_REPO_URL
|
||||||
|
|
||||||
ENV LIBRDKAFKA_VERSION=1.6.2
|
ENV LIBRDKAFKA_VERSION=1.6.2
|
||||||
ENV CONFLUENT_KAFKA_VERSION=1.6.1
|
ENV CONFLUENT_KAFKA_VERSION=1.6.1
|
||||||
|
|
||||||
ENV DEBIAN_FRONTEND noninteractive
|
ENV DEBIAN_FRONTEND noninteractive
|
||||||
|
|
||||||
|
# Optionally set corporate mirror for apk and pip
|
||||||
|
RUN if [ "${DEBIAN_REPO_URL}" != "http://deb.debian.org/debian" ] ; then sed -i "s#http.*://deb.debian.org/debian#${DEBIAN_REPO_URL}#g" /etc/apt/sources.list.d/debian.sources ; fi
|
||||||
|
RUN if [ "${PIP_MIRROR_URL}" != "null" ] ; then pip config set global.index-url ${PIP_MIRROR_URL} ; fi
|
||||||
|
|
||||||
RUN apt-get update && apt-get install -y -qq \
|
RUN apt-get update && apt-get install -y -qq \
|
||||||
make \
|
make \
|
||||||
python3-ldap \
|
python3-ldap \
|
||||||
@ -33,7 +53,7 @@ RUN apt-get update && apt-get install -y -qq \
|
|||||||
unzip \
|
unzip \
|
||||||
ldap-utils \
|
ldap-utils \
|
||||||
&& python -m pip install --no-cache --upgrade pip wheel setuptools \
|
&& python -m pip install --no-cache --upgrade pip wheel setuptools \
|
||||||
&& wget -q https://github.com/edenhill/librdkafka/archive/v${LIBRDKAFKA_VERSION}.tar.gz -O - | \
|
&& wget -q ${GITHUB_REPO_URL}/edenhill/librdkafka/archive/v${LIBRDKAFKA_VERSION}.tar.gz -O - | \
|
||||||
tar -xz -C /root \
|
tar -xz -C /root \
|
||||||
&& cd /root/librdkafka-${LIBRDKAFKA_VERSION} \
|
&& cd /root/librdkafka-${LIBRDKAFKA_VERSION} \
|
||||||
&& ./configure --prefix /usr && make && make install && cd .. && rm -rf /root/librdkafka-${LIBRDKAFKA_VERSION} \
|
&& ./configure --prefix /usr && make && make install && cd .. && rm -rf /root/librdkafka-${LIBRDKAFKA_VERSION} \
|
||||||
|
@ -25,7 +25,24 @@ docker {
|
|||||||
}.exclude {
|
}.exclude {
|
||||||
i -> (!i.file.name.endsWith(".dockerignore") && i.file.isHidden())
|
i -> (!i.file.name.endsWith(".dockerignore") && i.file.isHidden())
|
||||||
}
|
}
|
||||||
buildArgs([APP_ENV: docker_target])
|
|
||||||
|
def dockerBuildArgs = [APP_ENV: docker_target]
|
||||||
|
|
||||||
|
// Add build args if they are defined (needed for some CI or enterprise environments)
|
||||||
|
if (project.hasProperty('alpineApkRepositoryUrl')) {
|
||||||
|
dockerBuildArgs.ALPINE_REPO_URL = project.getProperty('alpineApkRepositoryUrl')
|
||||||
|
}
|
||||||
|
if (project.hasProperty('githubMirrorUrl')) {
|
||||||
|
dockerBuildArgs.GITHUB_REPO_URL = project.getProperty('githubMirrorUrl')
|
||||||
|
}
|
||||||
|
if (project.hasProperty('debianAptRepositoryUrl')) {
|
||||||
|
dockerBuildArgs.DEBIAN_REPO_URL = project.getProperty('debianAptRepositoryUrl')
|
||||||
|
}
|
||||||
|
if (project.hasProperty('pipMirrorUrl')) {
|
||||||
|
dockerBuildArgs.PIP_MIRROR_URL = project.getProperty('pipMirrorUrl')
|
||||||
|
}
|
||||||
|
|
||||||
|
buildArgs(dockerBuildArgs)
|
||||||
}
|
}
|
||||||
tasks.getByName('docker').dependsOn('build')
|
tasks.getByName('docker').dependsOn('build')
|
||||||
|
|
||||||
|
@ -2,6 +2,8 @@
|
|||||||
ARG APP_ENV=full
|
ARG APP_ENV=full
|
||||||
ARG BASE_IMAGE=acryldata/datahub-ingestion-base
|
ARG BASE_IMAGE=acryldata/datahub-ingestion-base
|
||||||
ARG DOCKER_VERSION=head
|
ARG DOCKER_VERSION=head
|
||||||
|
ARG PIP_MIRROR_URL=null
|
||||||
|
ARG DEBIAN_REPO_URL=http://deb.debian.org/debian
|
||||||
|
|
||||||
FROM $BASE_IMAGE:$DOCKER_VERSION as base
|
FROM $BASE_IMAGE:$DOCKER_VERSION as base
|
||||||
USER 0
|
USER 0
|
||||||
@ -20,16 +22,23 @@ USER datahub
|
|||||||
ENV PATH="/datahub-ingestion/.local/bin:$PATH"
|
ENV PATH="/datahub-ingestion/.local/bin:$PATH"
|
||||||
|
|
||||||
FROM base as slim-install
|
FROM base as slim-install
|
||||||
|
ARG PIP_MIRROR_URL
|
||||||
|
|
||||||
|
RUN if [ "${PIP_MIRROR_URL}" != "null" ] ; then pip config set global.index-url ${PIP_MIRROR_URL} ; fi
|
||||||
RUN pip install --no-cache --user ".[base,datahub-rest,datahub-kafka,snowflake,bigquery,redshift,mysql,postgres,hive,clickhouse,glue,dbt,looker,lookml,tableau,powerbi,superset,datahub-business-glossary]"
|
RUN pip install --no-cache --user ".[base,datahub-rest,datahub-kafka,snowflake,bigquery,redshift,mysql,postgres,hive,clickhouse,glue,dbt,looker,lookml,tableau,powerbi,superset,datahub-business-glossary]"
|
||||||
|
|
||||||
FROM base as full-install-build
|
FROM base as full-install-build
|
||||||
|
ARG PIP_MIRROR_URL
|
||||||
|
ARG DEBIAN_REPO_URL
|
||||||
|
|
||||||
USER 0
|
USER 0
|
||||||
|
RUN if [ "${DEBIAN_REPO_URL}" != "http://deb.debian.org/debian" ] ; then sed -i "s#http.*://deb.debian.org/debian#${DEBIAN_REPO_URL}#g" /etc/apt/sources.list.d/debian.sources ; fi
|
||||||
RUN apt-get update && apt-get install -y -qq maven
|
RUN apt-get update && apt-get install -y -qq maven
|
||||||
|
|
||||||
USER datahub
|
USER datahub
|
||||||
COPY ./docker/datahub-ingestion/pyspark_jars.sh .
|
COPY ./docker/datahub-ingestion/pyspark_jars.sh .
|
||||||
|
|
||||||
|
RUN if [ "${PIP_MIRROR_URL}" != "null" ] ; then pip config set global.index-url ${PIP_MIRROR_URL} ; fi
|
||||||
RUN pip install --no-cache --user ".[base]" && \
|
RUN pip install --no-cache --user ".[base]" && \
|
||||||
pip install --no-cache --user "./airflow-plugin[acryl-datahub-airflow-plugin]" && \
|
pip install --no-cache --user "./airflow-plugin[acryl-datahub-airflow-plugin]" && \
|
||||||
pip install --no-cache --user ".[all]"
|
pip install --no-cache --user ".[all]"
|
||||||
|
@ -1,6 +1,7 @@
|
|||||||
# Defining environment
|
# Defining environment
|
||||||
ARG BASE_IMAGE=acryldata/datahub-ingestion-base
|
ARG BASE_IMAGE=acryldata/datahub-ingestion-base
|
||||||
ARG DOCKER_VERSION=head-slim
|
ARG DOCKER_VERSION=head-slim
|
||||||
|
ARG PIP_MIRROR_URL=null
|
||||||
|
|
||||||
FROM $BASE_IMAGE:$DOCKER_VERSION as base
|
FROM $BASE_IMAGE:$DOCKER_VERSION as base
|
||||||
USER 0
|
USER 0
|
||||||
@ -17,6 +18,10 @@ USER datahub
|
|||||||
ENV PATH="/datahub-ingestion/.local/bin:$PATH"
|
ENV PATH="/datahub-ingestion/.local/bin:$PATH"
|
||||||
|
|
||||||
FROM base as slim-install
|
FROM base as slim-install
|
||||||
|
|
||||||
|
ARG PIP_MIRROR_URL
|
||||||
|
|
||||||
|
RUN if [ "${PIP_MIRROR_URL}" != "null" ] ; then pip config set global.index-url ${PIP_MIRROR_URL} ; fi
|
||||||
RUN pip install --no-cache --user ".[base,datahub-rest,datahub-kafka,snowflake,bigquery,redshift,mysql,postgres,hive,clickhouse,glue,dbt,looker,lookml,tableau,powerbi,superset,datahub-business-glossary]"
|
RUN pip install --no-cache --user ".[base,datahub-rest,datahub-kafka,snowflake,bigquery,redshift,mysql,postgres,hive,clickhouse,glue,dbt,looker,lookml,tableau,powerbi,superset,datahub-business-glossary]"
|
||||||
|
|
||||||
FROM slim-install as final
|
FROM slim-install as final
|
||||||
|
@ -32,8 +32,18 @@ docker {
|
|||||||
}.exclude {
|
}.exclude {
|
||||||
i -> (!i.file.name.endsWith(".dockerignore") && i.file.isHidden())
|
i -> (!i.file.name.endsWith(".dockerignore") && i.file.isHidden())
|
||||||
}
|
}
|
||||||
buildArgs([DOCKER_VERSION: version,
|
|
||||||
RELEASE_VERSION: version.replace('-SNAPSHOT', '').replace('v', '').replace("-slim", '')])
|
def dockerBuildArgs = [DOCKER_VERSION: version, RELEASE_VERSION: version.replace('-SNAPSHOT', '').replace('v', '').replace("-slim", '')]
|
||||||
|
|
||||||
|
// Add build args if they are defined (needed for some CI or enterprise environments)
|
||||||
|
if (project.hasProperty('pipMirrorUrl')) {
|
||||||
|
dockerBuildArgs.PIP_MIRROR_URL = project.getProperty('pipMirrorUrl')
|
||||||
|
}
|
||||||
|
if (project.hasProperty('debianAptRepositoryUrl')) {
|
||||||
|
dockerBuildArgs.DEBIAN_REPO_URL = project.getProperty('debianAptRepositoryUrl')
|
||||||
|
}
|
||||||
|
|
||||||
|
buildArgs(dockerBuildArgs)
|
||||||
}
|
}
|
||||||
tasks.getByName('docker').dependsOn(['build',
|
tasks.getByName('docker').dependsOn(['build',
|
||||||
':docker:datahub-ingestion-base:docker',
|
':docker:datahub-ingestion-base:docker',
|
||||||
|
@ -1,11 +1,22 @@
|
|||||||
# Defining environment
|
# Defining environment
|
||||||
ARG APP_ENV=prod
|
ARG APP_ENV=prod
|
||||||
|
|
||||||
|
# Defining custom repo urls for use in enterprise environments. Re-used between stages below.
|
||||||
|
ARG ALPINE_REPO_URL=http://dl-cdn.alpinelinux.org/alpine
|
||||||
|
ARG GITHUB_REPO_URL=https://github.com
|
||||||
|
ARG MAVEN_CENTRAL_REPO_URL=https://repo1.maven.org/maven2
|
||||||
|
|
||||||
FROM golang:1-alpine3.18 AS binary
|
FROM golang:1-alpine3.18 AS binary
|
||||||
|
|
||||||
|
# Re-declaring arg from above to make it available in this stage (will inherit default value)
|
||||||
|
ARG ALPINE_REPO_URL
|
||||||
|
|
||||||
ENV DOCKERIZE_VERSION v0.6.1
|
ENV DOCKERIZE_VERSION v0.6.1
|
||||||
WORKDIR /go/src/github.com/jwilder
|
WORKDIR /go/src/github.com/jwilder
|
||||||
|
|
||||||
|
# Optionally set corporate mirror for apk
|
||||||
|
RUN if [ "${ALPINE_REPO_URL}" != "http://dl-cdn.alpinelinux.org/alpine" ] ; then sed -i "s#http.*://dl-cdn.alpinelinux.org/alpine#${ALPINE_REPO_URL}#g" /etc/apk/repositories ; fi
|
||||||
|
|
||||||
RUN apk --no-cache --update add openssl git tar curl
|
RUN apk --no-cache --update add openssl git tar curl
|
||||||
|
|
||||||
WORKDIR /go/src/github.com/jwilder/dockerize
|
WORKDIR /go/src/github.com/jwilder/dockerize
|
||||||
@ -14,15 +25,23 @@ RUN go install github.com/jwilder/dockerize@$DOCKERIZE_VERSION
|
|||||||
|
|
||||||
FROM alpine:3 AS base
|
FROM alpine:3 AS base
|
||||||
|
|
||||||
|
# Re-declaring args from above to make them available in this stage (will inherit default values)
|
||||||
|
ARG ALPINE_REPO_URL
|
||||||
|
ARG GITHUB_REPO_URL
|
||||||
|
ARG MAVEN_CENTRAL_REPO_URL
|
||||||
|
|
||||||
|
# Optionally set corporate mirror for apk
|
||||||
|
RUN if [ "${ALPINE_REPO_URL}" != "http://dl-cdn.alpinelinux.org/alpine" ] ; then sed -i "s#http.*://dl-cdn.alpinelinux.org/alpine#${ALPINE_REPO_URL}#g" /etc/apk/repositories ; fi
|
||||||
|
|
||||||
# Upgrade Alpine and base packages
|
# Upgrade Alpine and base packages
|
||||||
ENV JMX_VERSION=0.18.0
|
ENV JMX_VERSION=0.18.0
|
||||||
# PFP-260: Upgrade Sqlite to >=3.28.0-r0 to fix https://security.snyk.io/vuln/SNYK-ALPINE39-SQLITE-449762
|
# PFP-260: Upgrade Sqlite to >=3.28.0-r0 to fix https://security.snyk.io/vuln/SNYK-ALPINE39-SQLITE-449762
|
||||||
RUN apk --no-cache --update-cache --available upgrade \
|
RUN apk --no-cache --update-cache --available upgrade \
|
||||||
&& apk --no-cache add curl bash coreutils sqlite libc6-compat java-snappy \
|
&& apk --no-cache add curl bash coreutils sqlite libc6-compat java-snappy \
|
||||||
&& apk --no-cache add openjdk11-jre-headless --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community \
|
&& apk --no-cache add openjdk11-jre-headless --repository=${ALPINE_REPO_URL}/edge/community \
|
||||||
&& apk --no-cache add jattach --repository http://dl-cdn.alpinelinux.org/alpine/edge/community/ \
|
&& apk --no-cache add jattach --repository ${ALPINE_REPO_URL}/edge/community/ \
|
||||||
&& wget --no-verbose https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/download/v1.24.0/opentelemetry-javaagent.jar \
|
&& wget --no-verbose ${GITHUB_REPO_URL}/open-telemetry/opentelemetry-java-instrumentation/releases/download/v1.24.0/opentelemetry-javaagent.jar \
|
||||||
&& wget --no-verbose https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/${JMX_VERSION}/jmx_prometheus_javaagent-${JMX_VERSION}.jar -O jmx_prometheus_javaagent.jar \
|
&& wget --no-verbose ${MAVEN_CENTRAL_REPO_URL}/io/prometheus/jmx/jmx_prometheus_javaagent/${JMX_VERSION}/jmx_prometheus_javaagent-${JMX_VERSION}.jar -O jmx_prometheus_javaagent.jar \
|
||||||
&& cp /usr/lib/jvm/java-11-openjdk/jre/lib/security/cacerts /tmp/kafka.client.truststore.jks
|
&& cp /usr/lib/jvm/java-11-openjdk/jre/lib/security/cacerts /tmp/kafka.client.truststore.jks
|
||||||
COPY --from=binary /go/bin/dockerize /usr/local/bin
|
COPY --from=binary /go/bin/dockerize /usr/local/bin
|
||||||
|
|
||||||
|
@ -1,11 +1,22 @@
|
|||||||
# Defining environment
|
# Defining environment
|
||||||
ARG APP_ENV=prod
|
ARG APP_ENV=prod
|
||||||
|
|
||||||
|
# Defining custom repo urls for use in enterprise environments. Re-used between stages below.
|
||||||
|
ARG ALPINE_REPO_URL=http://dl-cdn.alpinelinux.org/alpine
|
||||||
|
ARG GITHUB_REPO_URL=https://github.com
|
||||||
|
ARG MAVEN_CENTRAL_REPO_URL=https://repo1.maven.org/maven2
|
||||||
|
|
||||||
FROM golang:1-alpine3.18 AS binary
|
FROM golang:1-alpine3.18 AS binary
|
||||||
|
|
||||||
|
# Re-declaring arg from above to make it available in this stage (will inherit default value)
|
||||||
|
ARG ALPINE_REPO_URL
|
||||||
|
|
||||||
ENV DOCKERIZE_VERSION v0.6.1
|
ENV DOCKERIZE_VERSION v0.6.1
|
||||||
WORKDIR /go/src/github.com/jwilder
|
WORKDIR /go/src/github.com/jwilder
|
||||||
|
|
||||||
|
# Optionally set corporate mirror for apk
|
||||||
|
RUN if [ "${ALPINE_REPO_URL}" != "http://dl-cdn.alpinelinux.org/alpine" ] ; then sed -i "s#http.*://dl-cdn.alpinelinux.org/alpine#${ALPINE_REPO_URL}#g" /etc/apk/repositories ; fi
|
||||||
|
|
||||||
RUN apk --no-cache --update add openssl git tar curl
|
RUN apk --no-cache --update add openssl git tar curl
|
||||||
|
|
||||||
WORKDIR /go/src/github.com/jwilder/dockerize
|
WORKDIR /go/src/github.com/jwilder/dockerize
|
||||||
@ -14,15 +25,23 @@ RUN go install github.com/jwilder/dockerize@$DOCKERIZE_VERSION
|
|||||||
|
|
||||||
FROM alpine:3 AS base
|
FROM alpine:3 AS base
|
||||||
|
|
||||||
|
# Re-declaring args from above to make them available in this stage (will inherit default values)
|
||||||
|
ARG ALPINE_REPO_URL
|
||||||
|
ARG GITHUB_REPO_URL
|
||||||
|
ARG MAVEN_CENTRAL_REPO_URL
|
||||||
|
|
||||||
|
# Optionally set corporate mirror for apk
|
||||||
|
RUN if [ "${ALPINE_REPO_URL}" != "http://dl-cdn.alpinelinux.org/alpine" ] ; then sed -i "s#http.*://dl-cdn.alpinelinux.org/alpine#${ALPINE_REPO_URL}#g" /etc/apk/repositories ; fi
|
||||||
|
|
||||||
# Upgrade Alpine and base packages
|
# Upgrade Alpine and base packages
|
||||||
ENV JMX_VERSION=0.18.0
|
ENV JMX_VERSION=0.18.0
|
||||||
# PFP-260: Upgrade Sqlite to >=3.28.0-r0 to fix https://security.snyk.io/vuln/SNYK-ALPINE39-SQLITE-449762
|
# PFP-260: Upgrade Sqlite to >=3.28.0-r0 to fix https://security.snyk.io/vuln/SNYK-ALPINE39-SQLITE-449762
|
||||||
RUN apk --no-cache --update-cache --available upgrade \
|
RUN apk --no-cache --update-cache --available upgrade \
|
||||||
&& apk --no-cache add curl bash sqlite libc6-compat java-snappy \
|
&& apk --no-cache add curl bash sqlite libc6-compat java-snappy \
|
||||||
&& apk --no-cache add openjdk11-jre-headless --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community \
|
&& apk --no-cache add openjdk11-jre-headless --repository=${ALPINE_REPO_URL}/edge/community \
|
||||||
&& apk --no-cache add jattach --repository http://dl-cdn.alpinelinux.org/alpine/edge/community/ \
|
&& apk --no-cache add jattach --repository ${ALPINE_REPO_URL}/edge/community/ \
|
||||||
&& wget --no-verbose https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/download/v1.24.0/opentelemetry-javaagent.jar \
|
&& wget --no-verbose ${GITHUB_REPO_URL}/open-telemetry/opentelemetry-java-instrumentation/releases/download/v1.24.0/opentelemetry-javaagent.jar \
|
||||||
&& wget --no-verbose https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/${JMX_VERSION}/jmx_prometheus_javaagent-${JMX_VERSION}.jar -O jmx_prometheus_javaagent.jar \
|
&& wget --no-verbose ${MAVEN_CENTRAL_REPO_URL}/io/prometheus/jmx/jmx_prometheus_javaagent/${JMX_VERSION}/jmx_prometheus_javaagent-${JMX_VERSION}.jar -O jmx_prometheus_javaagent.jar \
|
||||||
&& cp /usr/lib/jvm/java-11-openjdk/jre/lib/security/cacerts /tmp/kafka.client.truststore.jks
|
&& cp /usr/lib/jvm/java-11-openjdk/jre/lib/security/cacerts /tmp/kafka.client.truststore.jks
|
||||||
COPY --from=binary /go/bin/dockerize /usr/local/bin
|
COPY --from=binary /go/bin/dockerize /usr/local/bin
|
||||||
|
|
||||||
|
@ -1,11 +1,22 @@
|
|||||||
# Defining environment
|
# Defining environment
|
||||||
ARG APP_ENV=prod
|
ARG APP_ENV=prod
|
||||||
|
|
||||||
|
# Defining custom repo urls for use in enterprise environments. Re-used between stages below.
|
||||||
|
ARG ALPINE_REPO_URL=http://dl-cdn.alpinelinux.org/alpine
|
||||||
|
ARG GITHUB_REPO_URL=https://github.com
|
||||||
|
ARG MAVEN_CENTRAL_REPO_URL=https://repo1.maven.org/maven2
|
||||||
|
|
||||||
FROM golang:1-alpine3.18 AS binary
|
FROM golang:1-alpine3.18 AS binary
|
||||||
|
|
||||||
|
# Re-declaring arg from above to make it available in this stage (will inherit default value)
|
||||||
|
ARG ALPINE_REPO_URL
|
||||||
|
|
||||||
ENV DOCKERIZE_VERSION v0.6.1
|
ENV DOCKERIZE_VERSION v0.6.1
|
||||||
WORKDIR /go/src/github.com/jwilder
|
WORKDIR /go/src/github.com/jwilder
|
||||||
|
|
||||||
|
# Optionally set corporate mirror for apk
|
||||||
|
RUN if [ "${ALPINE_REPO_URL}" != "http://dl-cdn.alpinelinux.org/alpine" ] ; then sed -i "s#http.*://dl-cdn.alpinelinux.org/alpine#${ALPINE_REPO_URL}#g" /etc/apk/repositories ; fi
|
||||||
|
|
||||||
RUN apk --no-cache --update add openssl git tar curl
|
RUN apk --no-cache --update add openssl git tar curl
|
||||||
|
|
||||||
WORKDIR /go/src/github.com/jwilder/dockerize
|
WORKDIR /go/src/github.com/jwilder/dockerize
|
||||||
@ -14,17 +25,25 @@ RUN go install github.com/jwilder/dockerize@$DOCKERIZE_VERSION
|
|||||||
|
|
||||||
FROM alpine:3 AS base
|
FROM alpine:3 AS base
|
||||||
|
|
||||||
|
# Re-declaring args from above to make them available in this stage (will inherit default values)
|
||||||
|
ARG ALPINE_REPO_URL
|
||||||
|
ARG GITHUB_REPO_URL
|
||||||
|
ARG MAVEN_CENTRAL_REPO_URL
|
||||||
|
|
||||||
|
# Optionally set corporate mirror for apk
|
||||||
|
RUN if [ "${ALPINE_REPO_URL}" != "http://dl-cdn.alpinelinux.org/alpine" ] ; then sed -i "s#http.*://dl-cdn.alpinelinux.org/alpine#${ALPINE_REPO_URL}#g" /etc/apk/repositories ; fi
|
||||||
|
|
||||||
# Upgrade Alpine and base packages
|
# Upgrade Alpine and base packages
|
||||||
ENV JMX_VERSION=0.18.0
|
ENV JMX_VERSION=0.18.0
|
||||||
# PFP-260: Upgrade Sqlite to >=3.28.0-r0 to fix https://security.snyk.io/vuln/SNYK-ALPINE39-SQLITE-449762
|
# PFP-260: Upgrade Sqlite to >=3.28.0-r0 to fix https://security.snyk.io/vuln/SNYK-ALPINE39-SQLITE-449762
|
||||||
RUN apk --no-cache --update-cache --available upgrade \
|
RUN apk --no-cache --update-cache --available upgrade \
|
||||||
&& apk --no-cache add curl bash coreutils gcompat sqlite libc6-compat java-snappy \
|
&& apk --no-cache add curl bash coreutils gcompat sqlite libc6-compat java-snappy \
|
||||||
&& apk --no-cache add openjdk11-jre-headless --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community \
|
&& apk --no-cache add openjdk11-jre-headless --repository=${ALPINE_REPO_URL}/edge/community \
|
||||||
&& curl -sS https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-runner/9.4.46.v20220331/jetty-runner-9.4.46.v20220331.jar --output jetty-runner.jar \
|
&& curl -sS ${MAVEN_CENTRAL_REPO_URL}/org/eclipse/jetty/jetty-runner/9.4.46.v20220331/jetty-runner-9.4.46.v20220331.jar --output jetty-runner.jar \
|
||||||
&& curl -sS https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-jmx/9.4.46.v20220331/jetty-jmx-9.4.46.v20220331.jar --output jetty-jmx.jar \
|
&& curl -sS ${MAVEN_CENTRAL_REPO_URL}/org/eclipse/jetty/jetty-jmx/9.4.46.v20220331/jetty-jmx-9.4.46.v20220331.jar --output jetty-jmx.jar \
|
||||||
&& curl -sS https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-util/9.4.46.v20220331/jetty-util-9.4.46.v20220331.jar --output jetty-util.jar \
|
&& curl -sS ${MAVEN_CENTRAL_REPO_URL}/org/eclipse/jetty/jetty-util/9.4.46.v20220331/jetty-util-9.4.46.v20220331.jar --output jetty-util.jar \
|
||||||
&& wget --no-verbose https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/download/v1.24.0/opentelemetry-javaagent.jar \
|
&& wget --no-verbose ${GITHUB_REPO_URL}/open-telemetry/opentelemetry-java-instrumentation/releases/download/v1.24.0/opentelemetry-javaagent.jar \
|
||||||
&& wget --no-verbose https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/${JMX_VERSION}/jmx_prometheus_javaagent-${JMX_VERSION}.jar -O jmx_prometheus_javaagent.jar \
|
&& wget --no-verbose ${MAVEN_CENTRAL_REPO_URL}/io/prometheus/jmx/jmx_prometheus_javaagent/${JMX_VERSION}/jmx_prometheus_javaagent-${JMX_VERSION}.jar -O jmx_prometheus_javaagent.jar \
|
||||||
&& cp /usr/lib/jvm/java-11-openjdk/jre/lib/security/cacerts /tmp/kafka.client.truststore.jks
|
&& cp /usr/lib/jvm/java-11-openjdk/jre/lib/security/cacerts /tmp/kafka.client.truststore.jks
|
||||||
COPY --from=binary /go/bin/dockerize /usr/local/bin
|
COPY --from=binary /go/bin/dockerize /usr/local/bin
|
||||||
|
|
||||||
|
@ -3,11 +3,19 @@
|
|||||||
# Defining environment
|
# Defining environment
|
||||||
ARG APP_ENV=prod
|
ARG APP_ENV=prod
|
||||||
|
|
||||||
|
# Defining custom repo urls for use in enterprise environments. Re-used between stages below.
|
||||||
|
ARG ALPINE_REPO_URL=http://dl-cdn.alpinelinux.org/alpine
|
||||||
|
|
||||||
FROM golang:1-alpine3.18 AS binary
|
FROM golang:1-alpine3.18 AS binary
|
||||||
|
|
||||||
|
ARG ALPINE_REPO_URL
|
||||||
|
|
||||||
ENV DOCKERIZE_VERSION v0.6.1
|
ENV DOCKERIZE_VERSION v0.6.1
|
||||||
WORKDIR /go/src/github.com/jwilder
|
WORKDIR /go/src/github.com/jwilder
|
||||||
|
|
||||||
|
# Optionally set corporate mirror for apk
|
||||||
|
RUN if [ "${ALPINE_REPO_URL}" != "http://dl-cdn.alpinelinux.org/alpine" ] ; then sed -i "s#http.*://dl-cdn.alpinelinux.org/alpine#${ALPINE_REPO_URL}#g" /etc/apk/repositories ; fi
|
||||||
|
|
||||||
# PFP-260: Upgrade Sqlite to >=3.28.0-r0 to fix https://security.snyk.io/vuln/SNYK-ALPINE39-SQLITE-449762
|
# PFP-260: Upgrade Sqlite to >=3.28.0-r0 to fix https://security.snyk.io/vuln/SNYK-ALPINE39-SQLITE-449762
|
||||||
RUN apk --no-cache --update add openssl git tar curl sqlite
|
RUN apk --no-cache --update add openssl git tar curl sqlite
|
||||||
|
|
||||||
@ -16,6 +24,12 @@ WORKDIR /go/src/github.com/jwilder/dockerize
|
|||||||
RUN go install github.com/jwilder/dockerize@$DOCKERIZE_VERSION
|
RUN go install github.com/jwilder/dockerize@$DOCKERIZE_VERSION
|
||||||
|
|
||||||
FROM alpine:3 AS base
|
FROM alpine:3 AS base
|
||||||
|
|
||||||
|
ARG ALPINE_REPO_URL
|
||||||
|
|
||||||
|
# Optionally set corporate mirror for apk
|
||||||
|
RUN if [ "${ALPINE_REPO_URL}" != "http://dl-cdn.alpinelinux.org/alpine" ] ; then sed -i "s#http.*://dl-cdn.alpinelinux.org/alpine#${ALPINE_REPO_URL}#g" /etc/apk/repositories ; fi
|
||||||
|
|
||||||
RUN apk add --no-cache curl jq bash coreutils
|
RUN apk add --no-cache curl jq bash coreutils
|
||||||
COPY --from=binary /go/bin/dockerize /usr/local/bin
|
COPY --from=binary /go/bin/dockerize /usr/local/bin
|
||||||
|
|
||||||
|
@ -27,6 +27,16 @@ docker {
|
|||||||
buildx(true)
|
buildx(true)
|
||||||
load(true)
|
load(true)
|
||||||
push(false)
|
push(false)
|
||||||
|
|
||||||
|
// Add build args if they are defined (needed for some CI or enterprise environments)
|
||||||
|
def dockerBuildArgs = [:]
|
||||||
|
if (project.hasProperty('alpineApkRepositoryUrl')) {
|
||||||
|
dockerBuildArgs.ALPINE_REPO_URL = project.getProperty('alpineApkRepositoryUrl')
|
||||||
|
}
|
||||||
|
|
||||||
|
if (dockerBuildArgs.size() > 0) {
|
||||||
|
buildArgs(dockerBuildArgs)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
tasks.getByName('docker').dependsOn('build')
|
tasks.getByName('docker').dependsOn('build')
|
||||||
|
|
||||||
|
@ -1,28 +1,41 @@
|
|||||||
ARG KAFKA_DOCKER_VERSION=7.4.1
|
ARG KAFKA_DOCKER_VERSION=7.4.1
|
||||||
|
|
||||||
|
# Defining custom repo urls for use in enterprise environments. Re-used between stages below.
|
||||||
|
ARG ALPINE_REPO_URL=http://dl-cdn.alpinelinux.org/alpine
|
||||||
|
ARG GITHUB_REPO_URL=https://github.com
|
||||||
|
ARG MAVEN_CENTRAL_REPO_URL=https://repo1.maven.org/maven2
|
||||||
|
ARG APACHE_DOWNLOAD_URL=null
|
||||||
|
|
||||||
# Using as a base image because to get the needed jars for confluent utils
|
# Using as a base image because to get the needed jars for confluent utils
|
||||||
FROM confluentinc/cp-base-new:$KAFKA_DOCKER_VERSION as confluent_base
|
FROM confluentinc/cp-base-new:$KAFKA_DOCKER_VERSION as confluent_base
|
||||||
|
|
||||||
ARG MAVEN_REPO="https://repo1.maven.org/maven2"
|
ARG MAVEN_CENTRAL_REPO_URL
|
||||||
ARG SNAKEYAML_VERSION="2.0"
|
ARG SNAKEYAML_VERSION="2.0"
|
||||||
|
|
||||||
RUN rm /usr/share/java/cp-base-new/snakeyaml-*.jar \
|
RUN rm /usr/share/java/cp-base-new/snakeyaml-*.jar \
|
||||||
&& wget -P /usr/share/java/cp-base-new $MAVEN_REPO/org/yaml/snakeyaml/$SNAKEYAML_VERSION/snakeyaml-$SNAKEYAML_VERSION.jar
|
&& wget -P /usr/share/java/cp-base-new $MAVEN_CENTRAL_REPO_URL/org/yaml/snakeyaml/$SNAKEYAML_VERSION/snakeyaml-$SNAKEYAML_VERSION.jar
|
||||||
|
|
||||||
# Based on https://github.com/blacktop's alpine kafka build
|
# Based on https://github.com/blacktop's alpine kafka build
|
||||||
FROM python:3-alpine
|
FROM python:3-alpine
|
||||||
|
|
||||||
|
ARG ALPINE_REPO_URL
|
||||||
|
ARG APACHE_DOWNLOAD_URL
|
||||||
|
ARG GITHUB_REPO_URL
|
||||||
|
|
||||||
ENV KAFKA_VERSION 3.4.1
|
ENV KAFKA_VERSION 3.4.1
|
||||||
ENV SCALA_VERSION 2.13
|
ENV SCALA_VERSION 2.13
|
||||||
|
|
||||||
LABEL name="kafka" version=${KAFKA_VERSION}
|
LABEL name="kafka" version=${KAFKA_VERSION}
|
||||||
|
|
||||||
|
# Optionally set corporate mirror for apk
|
||||||
|
RUN if [ "${ALPINE_REPO_URL}" != "http://dl-cdn.alpinelinux.org/alpine" ] ; then sed -i "s#http.*://dl-cdn.alpinelinux.org/alpine#${ALPINE_REPO_URL}#g" /etc/apk/repositories ; fi
|
||||||
|
|
||||||
RUN apk add --no-cache bash coreutils
|
RUN apk add --no-cache bash coreutils
|
||||||
RUN apk --no-cache add openjdk11-jre-headless --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community
|
RUN apk --no-cache add openjdk11-jre-headless --repository=${ALPINE_REPO_URL}/edge/community
|
||||||
|
|
||||||
RUN apk add --no-cache -t .build-deps git curl ca-certificates jq gcc musl-dev libffi-dev zip
|
RUN apk add --no-cache -t .build-deps git curl ca-certificates jq gcc musl-dev libffi-dev zip
|
||||||
RUN mkdir -p /opt \
|
RUN mkdir -p /opt \
|
||||||
&& mirror=$(curl --stderr /dev/null https://www.apache.org/dyn/closer.cgi\?as_json\=1 | jq -r '.preferred') \
|
&& if [ "${APACHE_DOWNLOAD_URL}" != "null" ] ; then mirror="${APACHE_DOWNLOAD_URL}/" ; else mirror=$(curl --stderr /dev/null https://www.apache.org/dyn/closer.cgi\?as_json\=1 | jq -r '.preferred'); fi \
|
||||||
&& curl -sSL "${mirror}kafka/${KAFKA_VERSION}/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz" \
|
&& curl -sSL "${mirror}kafka/${KAFKA_VERSION}/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz" \
|
||||||
| tar -xzf - -C /opt \
|
| tar -xzf - -C /opt \
|
||||||
&& mv /opt/kafka_${SCALA_VERSION}-${KAFKA_VERSION} /opt/kafka \
|
&& mv /opt/kafka_${SCALA_VERSION}-${KAFKA_VERSION} /opt/kafka \
|
||||||
@ -39,8 +52,8 @@ RUN ls -la
|
|||||||
COPY --from=confluent_base /usr/share/java/cp-base-new/ /usr/share/java/cp-base-new/
|
COPY --from=confluent_base /usr/share/java/cp-base-new/ /usr/share/java/cp-base-new/
|
||||||
COPY --from=confluent_base /etc/cp-base-new/log4j.properties /etc/cp-base-new/log4j.properties
|
COPY --from=confluent_base /etc/cp-base-new/log4j.properties /etc/cp-base-new/log4j.properties
|
||||||
|
|
||||||
ADD --chown=kafka:kafka https://github.com/aws/aws-msk-iam-auth/releases/download/v1.1.6/aws-msk-iam-auth-1.1.6-all.jar /usr/share/java/cp-base-new
|
ADD --chown=kafka:kafka ${GITHUB_REPO_URL}/aws/aws-msk-iam-auth/releases/download/v1.1.6/aws-msk-iam-auth-1.1.6-all.jar /usr/share/java/cp-base-new
|
||||||
ADD --chown=kafka:kafka https://github.com/aws/aws-msk-iam-auth/releases/download/v1.1.6/aws-msk-iam-auth-1.1.6-all.jar /opt/kafka/libs
|
ADD --chown=kafka:kafka ${GITHUB_REPO_URL}/aws/aws-msk-iam-auth/releases/download/v1.1.6/aws-msk-iam-auth-1.1.6-all.jar /opt/kafka/libs
|
||||||
|
|
||||||
ENV METADATA_AUDIT_EVENT_NAME="MetadataAuditEvent_v4"
|
ENV METADATA_AUDIT_EVENT_NAME="MetadataAuditEvent_v4"
|
||||||
ENV METADATA_CHANGE_EVENT_NAME="MetadataChangeEvent_v4"
|
ENV METADATA_CHANGE_EVENT_NAME="MetadataChangeEvent_v4"
|
||||||
|
@ -26,6 +26,25 @@ docker {
|
|||||||
buildx(true)
|
buildx(true)
|
||||||
load(true)
|
load(true)
|
||||||
push(false)
|
push(false)
|
||||||
|
|
||||||
|
// Add build args if they are defined (needed for some CI or enterprise environments)
|
||||||
|
def dockerBuildArgs = [:]
|
||||||
|
if (project.hasProperty('alpineApkRepositoryUrl')) {
|
||||||
|
dockerBuildArgs.ALPINE_REPO_URL = project.getProperty('alpineApkRepositoryUrl')
|
||||||
|
}
|
||||||
|
if (project.hasProperty('githubMirrorUrl')) {
|
||||||
|
dockerBuildArgs.GITHUB_REPO_URL = project.getProperty('githubMirrorUrl')
|
||||||
|
}
|
||||||
|
if (project.hasProperty('mavenCentralRepositoryUrl')) {
|
||||||
|
dockerBuildArgs.MAVEN_CENTRAL_REPO_URL = project.getProperty('mavenCentralRepositoryUrl')
|
||||||
|
}
|
||||||
|
if (project.hasProperty('apacheDownloadUrl')) {
|
||||||
|
dockerBuildArgs.APACHE_DOWNLOAD_URL = project.getProperty('apacheDownloadUrl')
|
||||||
|
}
|
||||||
|
|
||||||
|
if (dockerBuildArgs.size() > 0) {
|
||||||
|
buildArgs(dockerBuildArgs)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
tasks.getByName('docker').dependsOn('build')
|
tasks.getByName('docker').dependsOn('build')
|
||||||
|
|
||||||
|
@ -1,8 +1,16 @@
|
|||||||
|
# Defining custom repo urls for use in enterprise environments. Re-used between stages below.
|
||||||
|
ARG ALPINE_REPO_URL=http://dl-cdn.alpinelinux.org/alpine
|
||||||
|
|
||||||
FROM golang:1-alpine3.18 AS binary
|
FROM golang:1-alpine3.18 AS binary
|
||||||
|
|
||||||
|
ARG ALPINE_REPO_URL
|
||||||
|
|
||||||
ENV DOCKERIZE_VERSION v0.6.1
|
ENV DOCKERIZE_VERSION v0.6.1
|
||||||
WORKDIR /go/src/github.com/jwilder
|
WORKDIR /go/src/github.com/jwilder
|
||||||
|
|
||||||
|
# Optionally set corporate mirror for apk
|
||||||
|
RUN if [ "${ALPINE_REPO_URL}" != "http://dl-cdn.alpinelinux.org/alpine" ] ; then sed -i "s#http.*://dl-cdn.alpinelinux.org/alpine#${ALPINE_REPO_URL}#g" /etc/apk/repositories ; fi
|
||||||
|
|
||||||
RUN apk --no-cache --update add openssl git tar curl
|
RUN apk --no-cache --update add openssl git tar curl
|
||||||
|
|
||||||
WORKDIR /go/src/github.com/jwilder/dockerize
|
WORKDIR /go/src/github.com/jwilder/dockerize
|
||||||
@ -12,6 +20,11 @@ RUN go install github.com/jwilder/dockerize@$DOCKERIZE_VERSION
|
|||||||
FROM alpine:3
|
FROM alpine:3
|
||||||
COPY --from=binary /go/bin/dockerize /usr/local/bin
|
COPY --from=binary /go/bin/dockerize /usr/local/bin
|
||||||
|
|
||||||
|
ARG ALPINE_REPO_URL
|
||||||
|
|
||||||
|
# Optionally set corporate mirror for apk
|
||||||
|
RUN if [ "${ALPINE_REPO_URL}" != "http://dl-cdn.alpinelinux.org/alpine" ] ; then sed -i "s#http.*://dl-cdn.alpinelinux.org/alpine#${ALPINE_REPO_URL}#g" /etc/apk/repositories ; fi
|
||||||
|
|
||||||
# PFP-260: Upgrade Sqlite to >=3.28.0-r0 to fix https://security.snyk.io/vuln/SNYK-ALPINE39-SQLITE-449762
|
# PFP-260: Upgrade Sqlite to >=3.28.0-r0 to fix https://security.snyk.io/vuln/SNYK-ALPINE39-SQLITE-449762
|
||||||
RUN apk add --no-cache mysql-client bash mariadb-connector-c sqlite
|
RUN apk add --no-cache mysql-client bash mariadb-connector-c sqlite
|
||||||
|
|
||||||
|
@ -27,6 +27,16 @@ docker {
|
|||||||
buildx(true)
|
buildx(true)
|
||||||
load(true)
|
load(true)
|
||||||
push(false)
|
push(false)
|
||||||
|
|
||||||
|
// Add build args if they are defined (needed for some CI or enterprise environments)
|
||||||
|
def dockerBuildArgs = [:]
|
||||||
|
if (project.hasProperty('alpineApkRepositoryUrl')) {
|
||||||
|
dockerBuildArgs.ALPINE_REPO_URL = project.getProperty('alpineApkRepositoryUrl')
|
||||||
|
}
|
||||||
|
|
||||||
|
if (dockerBuildArgs.size() > 0) {
|
||||||
|
buildArgs(dockerBuildArgs)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
tasks.getByName('docker').dependsOn('build')
|
tasks.getByName('docker').dependsOn('build')
|
||||||
|
|
||||||
|
@ -1,8 +1,16 @@
|
|||||||
|
# Defining custom repo urls for use in enterprise environments. Re-used between stages below.
|
||||||
|
ARG ALPINE_REPO_URL=http://dl-cdn.alpinelinux.org/alpine
|
||||||
|
|
||||||
FROM golang:1-alpine3.18 AS binary
|
FROM golang:1-alpine3.18 AS binary
|
||||||
|
|
||||||
|
ARG ALPINE_REPO_URL
|
||||||
|
|
||||||
ENV DOCKERIZE_VERSION v0.6.1
|
ENV DOCKERIZE_VERSION v0.6.1
|
||||||
WORKDIR /go/src/github.com/jwilder
|
WORKDIR /go/src/github.com/jwilder
|
||||||
|
|
||||||
|
# Optionally set corporate mirror for apk
|
||||||
|
RUN if [ "${ALPINE_REPO_URL}" != "http://dl-cdn.alpinelinux.org/alpine" ] ; then sed -i "s#http.*://dl-cdn.alpinelinux.org/alpine#${ALPINE_REPO_URL}#g" /etc/apk/repositories ; fi
|
||||||
|
|
||||||
RUN apk --no-cache --update add openssl git tar curl
|
RUN apk --no-cache --update add openssl git tar curl
|
||||||
|
|
||||||
WORKDIR /go/src/github.com/jwilder/dockerize
|
WORKDIR /go/src/github.com/jwilder/dockerize
|
||||||
@ -12,6 +20,11 @@ RUN go install github.com/jwilder/dockerize@$DOCKERIZE_VERSION
|
|||||||
FROM alpine:3
|
FROM alpine:3
|
||||||
COPY --from=binary /go/bin/dockerize /usr/local/bin
|
COPY --from=binary /go/bin/dockerize /usr/local/bin
|
||||||
|
|
||||||
|
ARG ALPINE_REPO_URL
|
||||||
|
|
||||||
|
# Optionally set corporate mirror for apk
|
||||||
|
RUN if [ "${ALPINE_REPO_URL}" != "http://dl-cdn.alpinelinux.org/alpine" ] ; then sed -i "s#http.*://dl-cdn.alpinelinux.org/alpine#${ALPINE_REPO_URL}#g" /etc/apk/repositories ; fi
|
||||||
|
|
||||||
# PFP-260: Upgrade Sqlite to >=3.28.0-r0 to fix https://security.snyk.io/vuln/SNYK-ALPINE39-SQLITE-449762
|
# PFP-260: Upgrade Sqlite to >=3.28.0-r0 to fix https://security.snyk.io/vuln/SNYK-ALPINE39-SQLITE-449762
|
||||||
RUN apk add --no-cache postgresql-client sqlite
|
RUN apk add --no-cache postgresql-client sqlite
|
||||||
|
|
||||||
|
@ -27,6 +27,16 @@ docker {
|
|||||||
buildx(true)
|
buildx(true)
|
||||||
load(true)
|
load(true)
|
||||||
push(false)
|
push(false)
|
||||||
|
|
||||||
|
// Add build args if they are defined (needed for some CI or enterprise environments)
|
||||||
|
def dockerBuildArgs = [:]
|
||||||
|
if (project.hasProperty('alpineApkRepositoryUrl')) {
|
||||||
|
dockerBuildArgs.ALPINE_REPO_URL = project.getProperty('alpineApkRepositoryUrl')
|
||||||
|
}
|
||||||
|
|
||||||
|
if (dockerBuildArgs.size() > 0) {
|
||||||
|
buildArgs(dockerBuildArgs)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
tasks.getByName('docker').dependsOn('build')
|
tasks.getByName('docker').dependsOn('build')
|
||||||
|
|
||||||
|
@ -2,8 +2,12 @@ import org.yaml.snakeyaml.Yaml
|
|||||||
|
|
||||||
buildscript {
|
buildscript {
|
||||||
repositories{
|
repositories{
|
||||||
|
if (project.hasProperty('apacheMavenRepositoryUrl')) {
|
||||||
|
maven { url project.getProperty('apacheMavenRepositoryUrl') }
|
||||||
|
} else {
|
||||||
mavenCentral()
|
mavenCentral()
|
||||||
}
|
}
|
||||||
|
}
|
||||||
dependencies {
|
dependencies {
|
||||||
classpath("org.yaml:snakeyaml:1.33")
|
classpath("org.yaml:snakeyaml:1.33")
|
||||||
}
|
}
|
||||||
|
@ -4,7 +4,11 @@ plugins {
|
|||||||
}
|
}
|
||||||
|
|
||||||
repositories {
|
repositories {
|
||||||
|
if (project.hasProperty('apacheMavenRepositoryUrl')) {
|
||||||
|
maven { url project.getProperty('apacheMavenRepositoryUrl') }
|
||||||
|
} else {
|
||||||
mavenCentral()
|
mavenCentral()
|
||||||
|
}
|
||||||
mavenLocal()
|
mavenLocal()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -58,6 +58,22 @@ docker {
|
|||||||
buildx(true)
|
buildx(true)
|
||||||
load(true)
|
load(true)
|
||||||
push(false)
|
push(false)
|
||||||
|
|
||||||
|
// Add build args if they are defined (needed for some CI or enterprise environments)
|
||||||
|
def dockerBuildArgs = [:]
|
||||||
|
if (project.hasProperty('alpineApkRepositoryUrl')) {
|
||||||
|
dockerBuildArgs.ALPINE_REPO_URL = project.getProperty('alpineApkRepositoryUrl')
|
||||||
|
}
|
||||||
|
if (project.hasProperty('githubMirrorUrl')) {
|
||||||
|
dockerBuildArgs.GITHUB_REPO_URL = project.getProperty('githubMirrorUrl')
|
||||||
|
}
|
||||||
|
if (project.hasProperty('mavenCentralRepositoryUrl')) {
|
||||||
|
dockerBuildArgs.MAVEN_CENTRAL_REPO_URL = project.getProperty('mavenCentralRepositoryUrl')
|
||||||
|
}
|
||||||
|
|
||||||
|
if (dockerBuildArgs.size() > 0) {
|
||||||
|
buildArgs(dockerBuildArgs)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
tasks.getByPath(":metadata-jobs:mae-consumer-job:docker").dependsOn([bootJar])
|
tasks.getByPath(":metadata-jobs:mae-consumer-job:docker").dependsOn([bootJar])
|
||||||
|
|
||||||
|
@ -69,6 +69,22 @@ docker {
|
|||||||
buildx(true)
|
buildx(true)
|
||||||
load(true)
|
load(true)
|
||||||
push(false)
|
push(false)
|
||||||
|
|
||||||
|
// Add build args if they are defined (needed for some CI or enterprise environments)
|
||||||
|
def dockerBuildArgs = [:]
|
||||||
|
if (project.hasProperty('alpineApkRepositoryUrl')) {
|
||||||
|
dockerBuildArgs.ALPINE_REPO_URL = project.getProperty('alpineApkRepositoryUrl')
|
||||||
|
}
|
||||||
|
if (project.hasProperty('githubMirrorUrl')) {
|
||||||
|
dockerBuildArgs.GITHUB_REPO_URL = project.getProperty('githubMirrorUrl')
|
||||||
|
}
|
||||||
|
if (project.hasProperty('mavenCentralRepositoryUrl')) {
|
||||||
|
dockerBuildArgs.MAVEN_CENTRAL_REPO_URL = project.getProperty('mavenCentralRepositoryUrl')
|
||||||
|
}
|
||||||
|
|
||||||
|
if (dockerBuildArgs.size() > 0) {
|
||||||
|
buildArgs(dockerBuildArgs)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
tasks.getByPath(":metadata-jobs:mce-consumer-job:docker").dependsOn([bootJar])
|
tasks.getByPath(":metadata-jobs:mce-consumer-job:docker").dependsOn([bootJar])
|
||||||
|
|
||||||
|
@ -2,8 +2,12 @@ import org.yaml.snakeyaml.Yaml
|
|||||||
|
|
||||||
buildscript {
|
buildscript {
|
||||||
repositories{
|
repositories{
|
||||||
|
if (project.hasProperty('apacheMavenRepositoryUrl')) {
|
||||||
|
maven { url project.getProperty('apacheMavenRepositoryUrl') }
|
||||||
|
} else {
|
||||||
mavenCentral()
|
mavenCentral()
|
||||||
}
|
}
|
||||||
|
}
|
||||||
dependencies {
|
dependencies {
|
||||||
classpath("org.yaml:snakeyaml:1.33")
|
classpath("org.yaml:snakeyaml:1.33")
|
||||||
}
|
}
|
||||||
|
@ -83,6 +83,22 @@ docker {
|
|||||||
buildx(true)
|
buildx(true)
|
||||||
load(true)
|
load(true)
|
||||||
push(false)
|
push(false)
|
||||||
|
|
||||||
|
// Add build args if they are defined (needed for some CI or enterprise environments)
|
||||||
|
def dockerBuildArgs = [:]
|
||||||
|
if (project.hasProperty('alpineApkRepositoryUrl')) {
|
||||||
|
dockerBuildArgs.ALPINE_REPO_URL = project.getProperty('alpineApkRepositoryUrl')
|
||||||
|
}
|
||||||
|
if (project.hasProperty('githubMirrorUrl')) {
|
||||||
|
dockerBuildArgs.GITHUB_REPO_URL = project.getProperty('githubMirrorUrl')
|
||||||
|
}
|
||||||
|
if (project.hasProperty('mavenCentralRepositoryUrl')) {
|
||||||
|
dockerBuildArgs.MAVEN_CENTRAL_REPO_URL = project.getProperty('mavenCentralRepositoryUrl')
|
||||||
|
}
|
||||||
|
|
||||||
|
if (dockerBuildArgs.size() > 0) {
|
||||||
|
buildArgs(dockerBuildArgs)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
tasks.getByPath(":metadata-service:war:docker").dependsOn([build, war])
|
tasks.getByPath(":metadata-service:war:docker").dependsOn([build, war])
|
||||||
|
|
||||||
|
@ -1,16 +1,32 @@
|
|||||||
repositories {
|
repositories {
|
||||||
gradlePluginPortal()
|
gradlePluginPortal()
|
||||||
mavenLocal()
|
mavenLocal()
|
||||||
|
|
||||||
|
if (project.hasProperty('apacheMavenRepositoryUrl')) {
|
||||||
|
maven { url project.getProperty('apacheMavenRepositoryUrl') }
|
||||||
|
} else {
|
||||||
mavenCentral()
|
mavenCentral()
|
||||||
|
}
|
||||||
|
|
||||||
|
if (project.hasProperty('confluentMavenRepositoryUrl')) {
|
||||||
|
maven {
|
||||||
|
url project.getProperty('confluentMavenRepositoryUrl')
|
||||||
|
}
|
||||||
|
} else {
|
||||||
maven {
|
maven {
|
||||||
url "https://packages.confluent.io/maven/"
|
url "https://packages.confluent.io/maven/"
|
||||||
}
|
}
|
||||||
maven {
|
|
||||||
url "https://plugins.gradle.org/m2/"
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (project.hasProperty('linkedinOpenSourceRepositoryUrl')) {
|
||||||
|
maven {
|
||||||
|
url project.getProperty('linkedinOpenSourceRepositoryUrl')
|
||||||
|
}
|
||||||
|
} else {
|
||||||
maven {
|
maven {
|
||||||
url "https://linkedin.jfrog.io/artifactory/open-source/" // GMA, pegasus
|
url "https://linkedin.jfrog.io/artifactory/open-source/" // GMA, pegasus
|
||||||
}
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
Loading…
x
Reference in New Issue
Block a user