fix(ci): enforce docker snippet validation in CI (#13163)

.github/workflows/build-and-test.yml

@@ -158,7 +158,7 @@ jobs:
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
 
-  quickstart-compose-validation:
+  docker-codegen-validation:
     runs-on: ubuntu-latest
     needs: setup
     if: ${{ needs.setup.outputs.docker_change == 'true' }}
@@ -170,6 +170,8 @@ jobs:
           python-version: "3.10"
       - name: Quickstart Compose Validation
         run: ./docker/quickstart/generate_and_compare.sh
+      - name: Docker Snippet Validation
+        run: python python-build/generate_ingestion_docker.py --check
 
   event-file:
     runs-on: ubuntu-latest

.github/workflows/docker-unified.yml

@@ -1144,7 +1144,7 @@ jobs:
           docker pull confluentinc/cp-kafka:7.4.0 &
           docker pull mysql:8.2 &
           docker pull opensearchproject/opensearch:2.9.0 &
-          docker pull ${{ env.DATAHUB_INGESTION_IMAGE }}:head  &
+          docker pull ${{ env.DATAHUB_INGESTION_BASE_IMAGE }}:head-slim  &
 
           wait
           docker images
@@ -1153,7 +1153,7 @@ jobs:
         env:
           DATAHUB_TELEMETRY_ENABLED: false
           DATAHUB_VERSION: ${{ needs.setup.outputs.unique_tag }}
-          DATAHUB_ACTIONS_IMAGE: ${{ env.DATAHUB_INGESTION_IMAGE }}
+          DATAHUB_ACTIONS_IMAGE: ${{ env.DATAHUB_INGESTION_BASE_IMAGE }}
           ACTIONS_VERSION: ${{ needs.datahub_ingestion_slim_build.outputs.tag || 'head-slim' }}
           ACTIONS_EXTRA_PACKAGES: "acryl-datahub-actions[executor] acryl-datahub-actions"
           ACTIONS_CONFIG: "https://raw.githubusercontent.com/acryldata/datahub-actions/main/docker/config/executor.yaml"

docker/datahub-ingestion-base/Dockerfile

@@ -97,8 +97,12 @@ WORKDIR $HOME
 RUN uv venv --python "$PYTHON_VERSION"
 ENV VIRTUAL_ENV=$HOME/.venv
 ENV PATH="${VIRTUAL_ENV}/bin:$PATH"
-# Requests comes with it's own CA bundle, but we want to use always use the system CA bundle.
+
+# We always want to use the system CA bundle.
+# Requests comes with it's own CA bundle, which we need to override.
 ENV REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt
+# uv uses a different mechanism. See https://github.com/astral-sh/uv/issues/1474.
+ENV SSL_CERT_FILE="/etc/ssl/certs/ca-certificates.crt"
 
 
 FROM base-empty AS full-deps-prebuild
@@ -153,6 +157,7 @@ RUN --mount=from=full-deps-prebuild,source=$HOME/.venv,target=/venv-full \
 
 FROM base-${APP_ENV} AS final
 
+COPY --from=powerman/dockerize:0.19 /usr/local/bin/dockerize /usr/local/bin
 COPY ./docker/datahub-ingestion-base/entrypoint.sh /entrypoint.sh
 ENTRYPOINT [ "/entrypoint.sh" ]
 ENV PATH="/datahub-ingestion/.local/bin:$PATH"

docker/datahub-ingestion/Dockerfile

@@ -98,8 +98,12 @@ WORKDIR $HOME
 RUN uv venv --python "$PYTHON_VERSION"
 ENV VIRTUAL_ENV=$HOME/.venv
 ENV PATH="${VIRTUAL_ENV}/bin:$PATH"
-# Requests comes with it's own CA bundle, but we want to use always use the system CA bundle.
+
+# We always want to use the system CA bundle.
+# Requests comes with it's own CA bundle, which we need to override.
 ENV REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt
+# uv uses a different mechanism. See https://github.com/astral-sh/uv/issues/1474.
+ENV SSL_CERT_FILE="/etc/ssl/certs/ca-certificates.crt"
 
 
 FROM base-empty AS full-deps-prebuild

docker/snippets/ubuntu_python_base

@@ -76,5 +76,9 @@ WORKDIR $HOME
 RUN uv venv --python "$PYTHON_VERSION"
 ENV VIRTUAL_ENV=$HOME/.venv
 ENV PATH="${VIRTUAL_ENV}/bin:$PATH"
-# Requests comes with it's own CA bundle, but we want to use always use the system CA bundle.
+
+# We always want to use the system CA bundle.
+# Requests comes with it's own CA bundle, which we need to override.
 ENV REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt
+# uv uses a different mechanism. See https://github.com/astral-sh/uv/issues/1474.
+ENV SSL_CERT_FILE="/etc/ssl/certs/ca-certificates.crt"

python-build/generate_ingestion_docker.py

@@ -55,16 +55,16 @@ def update_template(
         flags=re.DOTALL | re.MULTILINE,
     )
 
-    # if subs == 0:
-    #     raise ValueError(f"No templates found in {template_file}")
+    if subs == 0:
+        raise ValueError(f"No templates found in {template_file}")
 
-    output = outfile or template_file
-    if check_only:
-        if output.read_text() != content:
+    if check_only and not outfile:
+        if template_file.read_text() != content:
             print(f"ERROR: {template_file} is out of date")
             sys.exit(1)
     else:
         print(f"Applied {subs} substitutions while processing {template_file}")
+        output = outfile or template_file
         output.write_text(content)