diff --git a/build.gradle b/build.gradle index 18ff5f4ea5..9d0f33b61d 100644 --- a/build.gradle +++ b/build.gradle @@ -141,7 +141,8 @@ project.ext.externalDependency = [ 'datastaxOssCore': 'com.datastax.oss:java-driver-core:4.14.1', 'datastaxOssQueryBuilder': 'com.datastax.oss:java-driver-query-builder:4.14.1', 'dgraph4j' : 'io.dgraph:dgraph4j:24.1.1', - 'dgraphNetty': 'io.grpc:grpc-netty-shaded:1.69.0', + 'dgraphNetty': 'io.grpc:grpc-netty:1.71.0', + 'dgraphShadedNetty': 'io.grpc:grpc-netty-shaded:1.71.0', 'dropwizardMetricsCore': 'io.dropwizard.metrics:metrics-core:4.2.3', 'dropwizardMetricsJmx': 'io.dropwizard.metrics:metrics-jmx:4.2.3', 'ebean': 'io.ebean:ebean:' + ebeanVersion, @@ -201,7 +202,7 @@ project.ext.externalDependency = [ 'kafkaAvroSerde': "io.confluent:kafka-streams-avro-serde:$kafkaVersion", 'kafkaAvroSerializer': "io.confluent:kafka-avro-serializer:$kafkaVersion", 'kafkaClients': "org.apache.kafka:kafka-clients:$kafkaVersion-ccs", - 'snappy': 'org.xerial.snappy:snappy-java:1.1.10.5', + 'snappy': 'org.xerial.snappy:snappy-java:1.1.10.7', 'logbackClassic': "ch.qos.logback:logback-classic:$logbackClassic", 'logbackClassicJava8' : "ch.qos.logback:logback-classic:$logbackClassicJava8", 'slf4jApi': "org.slf4j:slf4j-api:$slf4jVersion", diff --git a/datahub-actions/pyproject.toml b/datahub-actions/pyproject.toml index 020db3af0a..ded2df4951 100644 --- a/datahub-actions/pyproject.toml +++ b/datahub-actions/pyproject.toml @@ -1,6 +1,6 @@ [build-system] build-backend = "setuptools.build_meta" -requires = ["setuptools>=54.0.0", "wheel", "pip>=21.0.0"] +requires = ["setuptools>65.5.1", "wheel>0.38.1", "pip>=21.0.0"] [tool.ruff] line-length = 88 diff --git a/docker/datahub-actions/Dockerfile b/docker/datahub-actions/Dockerfile index c59974442c..0d1b7b29f3 100644 --- a/docker/datahub-actions/Dockerfile +++ b/docker/datahub-actions/Dockerfile @@ -37,8 +37,8 @@ RUN existing_group=$(getent group 1000 | cut -d: -f1) && \ fi && \ # Create and set proper permissions for datahub directories mkdir -p $HOME && \ - chown -R datahub:datahub $HOME - + chown -R datahub:datahub $HOME && \ + chmod g-s $HOME # Setup the PPA for alternative Python versions. # TODO: Eventually we should switch to using uv's support for python-build-standalone. diff --git a/docker/datahub-frontend/Dockerfile b/docker/datahub-frontend/Dockerfile index 9fae1bac17..5b341ac597 100644 --- a/docker/datahub-frontend/Dockerfile +++ b/docker/datahub-frontend/Dockerfile @@ -17,7 +17,9 @@ RUN if [ "${ALPINE_REPO_URL}" != "http://dl-cdn.alpinelinux.org/alpine" ] ; then # PFP-260: Upgrade Sqlite to >=3.28.0-r0 to fix https://security.snyk.io/vuln/SNYK-ALPINE39-SQLITE-449762 ENV JMX_VERSION=0.20.0 RUN apk --no-cache --update-cache --available upgrade \ - && apk --no-cache add curl sqlite libc6-compat snappy \ + && apk --no-cache add 'c-ares>1.34.5' --repository=${ALPINE_REPO_URL}/edge/main \ + && apk --no-cache add curl sqlite libc6-compat \ + && apk --no-cache add snappy=~1.2 --repository=${ALPINE_REPO_URL}/edge/main \ && apk --no-cache add openjdk17-jre-headless --repository=${ALPINE_REPO_URL}/edge/community \ && apk --no-cache add jattach --repository ${ALPINE_REPO_URL}/edge/community/ \ && wget ${GITHUB_REPO_URL}/open-telemetry/opentelemetry-java-instrumentation/releases/download/v2.15.0/opentelemetry-javaagent.jar -O opentelemetry-javaagent.jar \ diff --git a/docker/datahub-gms/Dockerfile b/docker/datahub-gms/Dockerfile index 027e27f960..d607c13699 100644 --- a/docker/datahub-gms/Dockerfile +++ b/docker/datahub-gms/Dockerfile @@ -11,7 +11,7 @@ FROM golang:1-alpine3.21 AS binary # Re-declaring arg from above to make it available in this stage (will inherit default value) ARG ALPINE_REPO_URL -ENV DOCKERIZE_VERSION=v0.9.1 +ENV DOCKERIZE_VERSION=v0.9.3 WORKDIR /go/src/github.com/jwilder # Optionally set corporate mirror for apk @@ -38,7 +38,9 @@ RUN if [ "${ALPINE_REPO_URL}" != "http://dl-cdn.alpinelinux.org/alpine" ] ; then # PFP-260: Upgrade Sqlite to >=3.28.0-r0 to fix https://security.snyk.io/vuln/SNYK-ALPINE39-SQLITE-449762 RUN apk --no-cache --update-cache --available upgrade \ - && apk --no-cache add curl bash coreutils gcompat sqlite libc6-compat snappy \ + && apk --no-cache add 'c-ares>1.34.5' --repository=${ALPINE_REPO_URL}/edge/main \ + && apk --no-cache add curl bash coreutils gcompat sqlite libc6-compat \ + && apk --no-cache add snappy=~1.2 --repository=${ALPINE_REPO_URL}/edge/main \ && apk --no-cache add openjdk17-jre-headless --repository=${ALPINE_REPO_URL}/edge/community \ && apk --no-cache add jattach --repository ${ALPINE_REPO_URL}/edge/community/ \ && wget --no-verbose ${GITHUB_REPO_URL}/open-telemetry/opentelemetry-java-instrumentation/releases/download/v2.15.0/opentelemetry-javaagent.jar \ @@ -63,7 +65,7 @@ FROM ${APP_ENV}-install AS final RUN mkdir -p /etc/datahub/plugins/auth/resources -RUN addgroup -S datahub && adduser -S datahub -G datahub +RUN addgroup -S datahub && adduser -S datahub -G datahub && chmod g-s /home/datahub RUN chown -R datahub:datahub /etc/datahub /datahub USER datahub diff --git a/docker/datahub-ingestion-base/Dockerfile b/docker/datahub-ingestion-base/Dockerfile index d5e5102e54..eeee59c509 100644 --- a/docker/datahub-ingestion-base/Dockerfile +++ b/docker/datahub-ingestion-base/Dockerfile @@ -36,8 +36,8 @@ RUN existing_group=$(getent group 1000 | cut -d: -f1) && \ fi && \ # Create and set proper permissions for datahub directories mkdir -p $HOME && \ - chown -R datahub:datahub $HOME - + chown -R datahub:datahub $HOME && \ + chmod g-s $HOME # Setup the PPA for alternative Python versions. # TODO: Eventually we should switch to using uv's support for python-build-standalone. diff --git a/docker/datahub-ingestion/Dockerfile b/docker/datahub-ingestion/Dockerfile index eba2a1ec8b..6893d77398 100644 --- a/docker/datahub-ingestion/Dockerfile +++ b/docker/datahub-ingestion/Dockerfile @@ -37,8 +37,8 @@ RUN existing_group=$(getent group 1000 | cut -d: -f1) && \ fi && \ # Create and set proper permissions for datahub directories mkdir -p $HOME && \ - chown -R datahub:datahub $HOME - + chown -R datahub:datahub $HOME && \ + chmod g-s $HOME # Setup the PPA for alternative Python versions. # TODO: Eventually we should switch to using uv's support for python-build-standalone. diff --git a/docker/datahub-mae-consumer/Dockerfile b/docker/datahub-mae-consumer/Dockerfile index 09fa76732f..eb3228fa44 100644 --- a/docker/datahub-mae-consumer/Dockerfile +++ b/docker/datahub-mae-consumer/Dockerfile @@ -11,7 +11,7 @@ FROM golang:1-alpine3.21 AS binary # Re-declaring arg from above to make it available in this stage (will inherit default value) ARG ALPINE_REPO_URL -ENV DOCKERIZE_VERSION=v0.9.1 +ENV DOCKERIZE_VERSION=v0.9.3 WORKDIR /go/src/github.com/jwilder # Optionally set corporate mirror for apk @@ -37,7 +37,9 @@ RUN if [ "${ALPINE_REPO_URL}" != "http://dl-cdn.alpinelinux.org/alpine" ] ; then ENV JMX_VERSION=0.20.0 # PFP-260: Upgrade Sqlite to >=3.28.0-r0 to fix https://security.snyk.io/vuln/SNYK-ALPINE39-SQLITE-449762 RUN apk --no-cache --update-cache --available upgrade \ - && apk --no-cache add curl bash coreutils sqlite libc6-compat snappy \ + && apk --no-cache add 'c-ares>1.34.5' --repository=${ALPINE_REPO_URL}/edge/main \ + && apk --no-cache add curl bash coreutils sqlite libc6-compat \ + && apk --no-cache add snappy=~1.2 --repository=${ALPINE_REPO_URL}/edge/main \ && apk --no-cache add openjdk17-jre-headless --repository=${ALPINE_REPO_URL}/edge/community \ && apk --no-cache add jattach --repository ${ALPINE_REPO_URL}/edge/community/ \ && wget --no-verbose ${GITHUB_REPO_URL}/open-telemetry/opentelemetry-java-instrumentation/releases/download/v2.15.0/opentelemetry-javaagent.jar \ @@ -60,7 +62,7 @@ FROM base AS dev-install FROM ${APP_ENV}-install AS final -RUN addgroup -S datahub && adduser -S datahub -G datahub +RUN addgroup -S datahub && adduser -S datahub -G datahub && chmod g-s /home/datahub USER datahub ENV JMX_OPTS="" diff --git a/docker/datahub-mce-consumer/Dockerfile b/docker/datahub-mce-consumer/Dockerfile index 0e5599f4a5..2aa39f666d 100644 --- a/docker/datahub-mce-consumer/Dockerfile +++ b/docker/datahub-mce-consumer/Dockerfile @@ -11,7 +11,7 @@ FROM golang:1-alpine3.21 AS binary # Re-declaring arg from above to make it available in this stage (will inherit default value) ARG ALPINE_REPO_URL -ENV DOCKERIZE_VERSION=v0.9.1 +ENV DOCKERIZE_VERSION=v0.9.3 WORKDIR /go/src/github.com/jwilder # Optionally set corporate mirror for apk @@ -37,7 +37,9 @@ RUN if [ "${ALPINE_REPO_URL}" != "http://dl-cdn.alpinelinux.org/alpine" ] ; then ENV JMX_VERSION=0.20.0 # PFP-260: Upgrade Sqlite to >=3.28.0-r0 to fix https://security.snyk.io/vuln/SNYK-ALPINE39-SQLITE-449762 RUN apk --no-cache --update-cache --available upgrade \ - && apk --no-cache add curl bash sqlite libc6-compat snappy \ + && apk --no-cache add 'c-ares>1.34.5' --repository=${ALPINE_REPO_URL}/edge/main \ + && apk --no-cache add curl bash sqlite libc6-compat \ + && apk --no-cache add snappy=~1.2 --repository=${ALPINE_REPO_URL}/edge/main \ && apk --no-cache add openjdk17-jre-headless --repository=${ALPINE_REPO_URL}/edge/community \ && apk --no-cache add jattach --repository ${ALPINE_REPO_URL}/edge/community/ \ && wget --no-verbose ${GITHUB_REPO_URL}/open-telemetry/opentelemetry-java-instrumentation/releases/download/v2.15.0/opentelemetry-javaagent.jar \ @@ -61,7 +63,7 @@ COPY metadata-models/src/main/resources/entity-registry.yml /datahub/datahub-mce FROM ${APP_ENV}-install AS final -RUN addgroup -S datahub && adduser -S datahub -G datahub +RUN addgroup -S datahub && adduser -S datahub -G datahub && chmod g-s /home/datahub USER datahub ENV JMX_OPTS="" diff --git a/docker/datahub-upgrade/Dockerfile b/docker/datahub-upgrade/Dockerfile index 2ae2f8dda7..c88da156fc 100644 --- a/docker/datahub-upgrade/Dockerfile +++ b/docker/datahub-upgrade/Dockerfile @@ -11,7 +11,7 @@ FROM golang:1-alpine3.21 AS binary # Re-declaring arg from above to make it available in this stage (will inherit default value) ARG ALPINE_REPO_URL -ENV DOCKERIZE_VERSION=v0.9.1 +ENV DOCKERIZE_VERSION=v0.9.3 WORKDIR /go/src/github.com/jwilder # Optionally set corporate mirror for apk @@ -38,8 +38,10 @@ ENV JMX_VERSION=0.20.0 # Upgrade Alpine and base packages # PFP-260: Upgrade Sqlite to >=3.28.0-r0 to fix https://security.snyk.io/vuln/SNYK-ALPINE39-SQLITE-449762 RUN apk --no-cache --update-cache --available upgrade \ - && apk --no-cache add curl bash coreutils gcompat sqlite libc6-compat snappy \ + && apk --no-cache add 'c-ares>1.34.5' --repository=${ALPINE_REPO_URL}/edge/main \ + && apk --no-cache add curl bash coreutils gcompat sqlite libc6-compat \ && apk --no-cache add openjdk17-jre-headless --repository=${ALPINE_REPO_URL}/edge/community \ + && apk --no-cache add snappy=~1.2 --repository=${ALPINE_REPO_URL}/edge/main \ && mkdir -p /datahub/datahub-upgrade/lib \ && wget --no-verbose -P /datahub/datahub-upgrade/lib ${GITHUB_REPO_URL}/open-telemetry/opentelemetry-java-instrumentation/releases/download/v2.15.0/opentelemetry-javaagent.jar \ && wget --no-verbose -P /datahub/datahub-upgrade/lib ${MAVEN_CENTRAL_REPO_URL}/io/prometheus/jmx/jmx_prometheus_javaagent/${JMX_VERSION}/jmx_prometheus_javaagent-${JMX_VERSION}.jar -O jmx_prometheus_javaagent.jar \ @@ -62,7 +64,7 @@ FROM base AS dev-install FROM ${APP_ENV}-install AS final -RUN addgroup -S datahub && adduser -S datahub -G datahub +RUN addgroup -S datahub && adduser -S datahub -G datahub && chmod g-s /home/datahub USER datahub ENV OTEL_EXPORTER_OTLP_MAX_PAYLOAD_SIZE=4194304 \ diff --git a/docker/elasticsearch-setup/Dockerfile b/docker/elasticsearch-setup/Dockerfile index 584007a5fb..274cb4130d 100644 --- a/docker/elasticsearch-setup/Dockerfile +++ b/docker/elasticsearch-setup/Dockerfile @@ -10,14 +10,16 @@ FROM golang:1-alpine3.21 AS binary ARG ALPINE_REPO_URL -ENV DOCKERIZE_VERSION=v0.9.1 +ENV DOCKERIZE_VERSION=v0.9.3 WORKDIR /go/src/github.com/jwilder # Optionally set corporate mirror for apk RUN if [ "${ALPINE_REPO_URL}" != "http://dl-cdn.alpinelinux.org/alpine" ] ; then sed -i "s#http.*://dl-cdn.alpinelinux.org/alpine#${ALPINE_REPO_URL}#g" /etc/apk/repositories ; fi # PFP-260: Upgrade Sqlite to >=3.28.0-r0 to fix https://security.snyk.io/vuln/SNYK-ALPINE39-SQLITE-449762 -RUN apk --no-cache --update add openssl git tar curl sqlite +RUN apk --no-cache --update-cache --available upgrade \ + && apk --no-cache add 'c-ares>1.34.5' --repository=${ALPINE_REPO_URL}/edge/main \ + && apk --no-cache add openssl git tar curl sqlite WORKDIR /go/src/github.com/jwilder/dockerize diff --git a/docker/kafka-setup/Dockerfile b/docker/kafka-setup/Dockerfile index efb2319bd3..749c6693af 100644 --- a/docker/kafka-setup/Dockerfile +++ b/docker/kafka-setup/Dockerfile @@ -33,7 +33,10 @@ RUN if [ "${ALPINE_REPO_URL}" != "http://dl-cdn.alpinelinux.org/alpine" ] ; then RUN apk add --no-cache bash coreutils RUN apk --no-cache add openjdk17-jre-headless --repository=${ALPINE_REPO_URL}/edge/community -RUN apk add --no-cache -t .build-deps git curl ca-certificates jq gcc musl-dev libffi-dev zip +RUN apk --no-cache --update-cache --available upgrade \ + && apk --no-cache add 'c-ares>1.34.5' --repository=${ALPINE_REPO_URL}/edge/main \ + && apk --no-cache add -t .build-deps git curl ca-certificates jq gcc musl-dev libffi-dev zip + RUN mkdir -p /opt \ && if [ "${APACHE_DOWNLOAD_URL}" != "null" ] ; then mirror="${APACHE_DOWNLOAD_URL}/" ; else mirror=$(curl --stderr /dev/null https://www.apache.org/dyn/closer.cgi\?as_json\=1 | jq -r '.preferred'); fi \ && curl -sSL "${mirror}kafka/${KAFKA_VERSION}/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz" \ diff --git a/docker/mysql-setup/Dockerfile b/docker/mysql-setup/Dockerfile index 21b696a1b9..9da4c7754c 100644 --- a/docker/mysql-setup/Dockerfile +++ b/docker/mysql-setup/Dockerfile @@ -5,13 +5,15 @@ FROM golang:1-alpine3.21 AS binary ARG ALPINE_REPO_URL -ENV DOCKERIZE_VERSION=v0.9.1 +ENV DOCKERIZE_VERSION=v0.9.3 WORKDIR /go/src/github.com/jwilder # Optionally set corporate mirror for apk RUN if [ "${ALPINE_REPO_URL}" != "http://dl-cdn.alpinelinux.org/alpine" ] ; then sed -i "s#http.*://dl-cdn.alpinelinux.org/alpine#${ALPINE_REPO_URL}#g" /etc/apk/repositories ; fi -RUN apk --no-cache --update add openssl git tar curl +RUN apk --no-cache --update-cache --available upgrade \ + && apk --no-cache add 'c-ares>1.34.5' --repository=${ALPINE_REPO_URL}/edge/main \ + && apk --no-cache add openssl git tar curl WORKDIR /go/src/github.com/jwilder/dockerize diff --git a/docker/postgres-setup/Dockerfile b/docker/postgres-setup/Dockerfile index 5362e0d787..36fb95e129 100644 --- a/docker/postgres-setup/Dockerfile +++ b/docker/postgres-setup/Dockerfile @@ -5,13 +5,15 @@ FROM golang:1-alpine3.21 AS binary ARG ALPINE_REPO_URL -ENV DOCKERIZE_VERSION=v0.9.1 +ENV DOCKERIZE_VERSION=v0.9.3 WORKDIR /go/src/github.com/jwilder # Optionally set corporate mirror for apk RUN if [ "${ALPINE_REPO_URL}" != "http://dl-cdn.alpinelinux.org/alpine" ] ; then sed -i "s#http.*://dl-cdn.alpinelinux.org/alpine#${ALPINE_REPO_URL}#g" /etc/apk/repositories ; fi -RUN apk --no-cache --update add openssl git tar curl +RUN apk --no-cache --update-cache --available upgrade \ + && apk --no-cache add 'c-ares>1.34.5' --repository=${ALPINE_REPO_URL}/edge/main \ + && apk --no-cache add openssl git tar curl WORKDIR /go/src/github.com/jwilder/dockerize diff --git a/docker/snippets/ubuntu_python_base b/docker/snippets/ubuntu_python_base index b0c153dea0..582c6d5a10 100644 --- a/docker/snippets/ubuntu_python_base +++ b/docker/snippets/ubuntu_python_base @@ -27,8 +27,8 @@ RUN existing_group=$(getent group 1000 | cut -d: -f1) && \ fi && \ # Create and set proper permissions for datahub directories mkdir -p $HOME && \ - chown -R datahub:datahub $HOME - + chown -R datahub:datahub $HOME && \ + chmod g-s $HOME # Setup the PPA for alternative Python versions. # TODO: Eventually we should switch to using uv's support for python-build-standalone. diff --git a/metadata-io/build.gradle b/metadata-io/build.gradle index 2899735650..f076d4191b 100644 --- a/metadata-io/build.gradle +++ b/metadata-io/build.gradle @@ -37,6 +37,7 @@ dependencies { exclude group: 'io.grpc', module: 'grpc-protobuf' } implementation externalDependency.dgraphNetty + implementation externalDependency.dgraphShadedNetty implementation externalDependency.slf4jApi runtimeOnly externalDependency.logbackClassic compileOnly externalDependency.lombok