VERSION - Bump ingestion debian to bookworm (#19917)

* VERSION - Bump ingestion debian to bookworm * MINOR - Fix collate app yaml reader with unwanted dropwizard java props
2025-12-24 14:08:45 +00:00 · 2025-02-24 10:45:25 +01:00 · 2025-02-24 10:45:25 +01:00 · 51c86f5a76
commit 51c86f5a76
parent d6511d9da3
5 changed files with 100 additions and 11 deletions
--- a/.github/workflows/py-operator-build-test.yml
+++ b/.github/workflows/py-operator-build-test.yml
@ -0,0 +1,81 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+name: py-operator-build-test
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+      - '0.[0-9]+.[0-9]+'
+    paths-ignore:
+      - 'openmetadata-docs/**'
+      - 'openmetadata-docs-v1/**'
+  pull_request_target:
+    types: [labeled, opened, synchronize, reopened]
+    paths:
+      - "ingestion/**"
+      - "openmetadata-service/**"
+      - "openmetadata-spec/src/main/resources/json/schema/**"
+      - "pom.xml"
+      - "Makefile"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  py-run-build-tests:
+    runs-on: ubuntu-latest
+    steps:
+
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
+        with:
+          tool-cache: false
+          android: true
+          dotnet: true
+          haskell: true
+          large-packages: false
+          swap-storage: true
+          docker-images: false
+
+      - name: Wait for the labeler
+        uses: lewagon/wait-on-check-action@v1.3.4
+        if: ${{ github.event_name == 'pull_request_target' }}
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          check-name: Team Label
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+          wait-interval: 90
+
+      - name: Verify PR labels
+        uses: jesusvasquez333/verify-pr-label-action@v1.4.0
+        if: ${{ github.event_name == 'pull_request_target' }}
+        with:
+          github-token: '${{ secrets.GITHUB_TOKEN }}'
+          valid-labels: 'safe to test'
+          pull-request-number: '${{ github.event.pull_request.number }}'
+          disable-reviews: true  # To not auto approve changes
+
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          fetch-depth: 0
+
+      - name: Build Ingestion Operator & Ingest Sample Data
+        run: |
+          docker build -f ingestion/operators/docker/Dockerfile.ci . -t openmetadata/ingestion-base:test
+          docker run --net=host --rm openmetadata/igestion-base:test metadata ingest -c /openmetadata/pipelines/sample_data.yaml
--- a/ingestion/Dockerfile
+++ b/ingestion/Dockerfile
@ -29,6 +29,8 @@ RUN apt-get -qq update \
    libkrb5-dev \
    default-jdk \
    openssl \
+    # To ensure compatibility with unixodbc package
+    odbcinst=2.3.11-2+deb12u1 \
    postgresql \
    postgresql-contrib \
    tdsodbc \
@ -42,7 +44,7 @@ RUN apt-get -qq update \
    && rm -rf /var/lib/apt/lists/*
 COPY --from=mysql /usr/bin/mysqldump /usr/bin/mysqldump

-RUN if [ $(uname -m) = "arm64" || $(uname -m) = "aarch64" ]; \
+RUN if [ $(uname -m) = "arm64" ] | [ $(uname -m) = "aarch64" ]; \
 then \
 wget -q https://download.oracle.com/otn_software/linux/instantclient/191000/instantclient-basic-linux.arm64-19.10.0.0.0dbru.zip -O /oracle-instantclient.zip && \
 unzip -qq -d /instantclient -j /oracle-instantclient.zip && rm -f /oracle-instantclient.zip; \
--- a/ingestion/Dockerfile.ci
+++ b/ingestion/Dockerfile.ci
@ -29,6 +29,8 @@ RUN dpkg --configure -a \
    libkrb5-dev \
    default-jdk \
    openssl \
+    # To ensure compatibility with unixodbc package
+    odbcinst=2.3.11-2+deb12u1 \
    postgresql \
    postgresql-contrib \
    tdsodbc \
@ -43,7 +45,7 @@ RUN dpkg --configure -a \
    && rm -rf /var/lib/apt/lists/*
 COPY --from=mysql /usr/bin/mysqldump /usr/bin/mysqldump

-RUN if [ $(uname -m) = "arm64" || $(uname -m) = "aarch64" ]; \
+RUN if [ $(uname -m) = "arm64" ] | [ $(uname -m) = "aarch64" ]; \
 then \
 wget https://download.oracle.com/otn_software/linux/instantclient/191000/instantclient-basic-linux.arm64-19.10.0.0.0dbru.zip -O /oracle-instantclient.zip && \
 unzip -d /instantclient -j /oracle-instantclient.zip && rm -f /oracle-instantclient.zip; \
--- a/ingestion/operators/docker/Dockerfile
+++ b/ingestion/operators/docker/Dockerfile
@ -1,4 +1,4 @@
-FROM python:3.10-bullseye
+FROM python:3.10-bookworm

 RUN curl -sS https://packages.microsoft.com/keys/microsoft.asc | apt-key add -
 RUN curl -sS https://packages.microsoft.com/config/debian/11/prod.list > /etc/apt/sources.list.d/mssql-release.list
@ -28,11 +28,13 @@ RUN dpkg --configure -a \
    libkrb5-dev \
    default-jdk \
    openssl \
+    # To ensure compatibility with unixodbc package
+    odbcinst=2.3.11-2+deb12u1 \
    postgresql \
    postgresql-contrib \
    tdsodbc \
-    unixodbc \
-    unixodbc-dev \
+    unixodbc=2.3.11-2+deb12u1 \
+    unixodbc-dev=2.3.11-2+deb12u1 \
    unzip \
    git \
    wget --no-install-recommends \
@ -43,7 +45,7 @@ RUN dpkg --configure -a \
 # Add updated postgres/redshift dependencies based on libq
 ENV DEBIAN_FRONTEND=noninteractive
 RUN curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
-RUN echo "deb https://apt.postgresql.org/pub/repos/apt/ bullseye-pgdg main" > /etc/apt/sources.list.d/pgdg.list; \
+RUN echo "deb https://apt.postgresql.org/pub/repos/apt/ bookworm-pgdg main" > /etc/apt/sources.list.d/pgdg.list; \
    apt-get -qq update; \
    apt-get -qq install --no-install-recommends -y libpq-dev postgresql-client postgresql-common postgresql postgresql-contrib; \
    apt-get -qq autoremove -yqq --purge; \
--- a/ingestion/operators/docker/Dockerfile.ci
+++ b/ingestion/operators/docker/Dockerfile.ci
@ -1,4 +1,4 @@
-FROM python:3.10-bullseye
+FROM python:3.10-bookworm

 RUN curl -sS https://packages.microsoft.com/keys/microsoft.asc | apt-key add -
 RUN curl -sS https://packages.microsoft.com/config/debian/11/prod.list > /etc/apt/sources.list.d/mssql-release.list
@ -27,11 +27,13 @@ RUN apt-get -qq update \
    libkrb5-dev \
    default-jdk \
    openssl \
+    # To ensure compatibility with unixodbc package
+    odbcinst=2.3.11-2+deb12u1 \
    postgresql \
    postgresql-contrib \
    tdsodbc \
-    unixodbc \
-    unixodbc-dev \
+    unixodbc=2.3.11-2+deb12u1 \
+    unixodbc-dev=2.3.11-2+deb12u1 \
    unzip \
    vim \
    git \
@ -43,13 +45,13 @@ RUN apt-get -qq update \
 # Add updated postgres/redshift dependencies based on libq
 ENV DEBIAN_FRONTEND=noninteractive
 RUN curl -sS https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
-RUN echo "deb https://apt.postgresql.org/pub/repos/apt/ bullseye-pgdg main" > /etc/apt/sources.list.d/pgdg.list; \
+RUN echo "deb https://apt.postgresql.org/pub/repos/apt/ bookworm-pgdg main" > /etc/apt/sources.list.d/pgdg.list; \
    apt-get -qq update; \
    apt-get -qq install --no-install-recommends -y libpq-dev postgresql-client postgresql-common postgresql postgresql-contrib; \
    apt-get -qq autoremove -yqq --purge; \
    apt-get -qq clean && rm -rf /var/lib/apt/lists/*

-RUN if [ $(uname -m) = "arm64" || $(uname -m) = "aarch64" ]; \
+RUN if [ $(uname -m) = "arm64" ] | [ $(uname -m) = "aarch64" ]; \
 then \
 wget -q https://download.oracle.com/otn_software/linux/instantclient/191000/instantclient-basic-linux.arm64-19.10.0.0.0dbru.zip -O /oracle-instantclient.zip && \
 unzip -qq -d /instantclient -j /oracle-instantclient.zip && rm -f /oracle-instantclient.zip; \