diff --git a/.github/workflows/docker-unified.yml b/.github/workflows/docker-unified.yml index 5845f83aff..5e206dd95d 100644 --- a/.github/workflows/docker-unified.yml +++ b/.github/workflows/docker-unified.yml @@ -1,5 +1,8 @@ name: Docker Build, Scan, Test on: + workflow_dispatch: + schedule: + - cron: '0 0 * * *' # Run at midnight UTC every day push: branches: - master @@ -22,17 +25,7 @@ concurrency: cancel-in-progress: true env: - DATAHUB_GMS_IMAGE: "acryldata/datahub-gms" - DATAHUB_FRONTEND_IMAGE: "acryldata/datahub-frontend-react" - DATAHUB_MAE_CONSUMER_IMAGE: "acryldata/datahub-mae-consumer" - DATAHUB_MCE_CONSUMER_IMAGE: "acryldata/datahub-mce-consumer" - DATAHUB_KAFKA_SETUP_IMAGE: "acryldata/datahub-kafka-setup" - DATAHUB_ELASTIC_SETUP_IMAGE: "acryldata/datahub-elasticsearch-setup" - DATAHUB_MYSQL_SETUP_IMAGE: "acryldata/datahub-mysql-setup" - DATAHUB_UPGRADE_IMAGE: "acryldata/datahub-upgrade" - DATAHUB_INGESTION_BASE_IMAGE: "acryldata/datahub-ingestion-base" - DATAHUB_INGESTION_IMAGE: "acryldata/datahub-ingestion" - DATAHUB_ACTIONS_IMAGE: "acryldata/datahub-actions" + DOCKER_REGISTRY: "acryldata" DOCKER_CACHE: "DEPOT" DEPOT_PROJECT_ID: "${{ vars.DEPOT_PROJECT_ID }}" @@ -46,6 +39,7 @@ jobs: setup: runs-on: depot-ubuntu-24.04-small outputs: + # TODO: Many of the vars below should not be required anymore. tag: ${{ steps.tag.outputs.tag }} slim_tag: ${{ steps.tag.outputs.slim_tag }} full_tag: ${{ steps.tag.outputs.full_tag }} @@ -74,6 +68,12 @@ jobs: smoke_test_change: ${{ steps.ci-optimize.outputs.smoke-test-change == 'true' }} integrations_service_change: "false" datahub_executor_change: "false" + + run_publish_images: ${{ steps.run-publish-images.outputs.run_publish_images}} + + build_runner_type: ${{ steps.set-runner.outputs.build_runner_type }} + test_runner_type: ${{ steps.set-runner.outputs.test_runner_type }} + use_depot_cache: ${{ steps.set-runner.outputs.use_depot_cache }} steps: - name: Check out the repo uses: acryldata/sane-checkout-action@v3 @@ -125,11 +125,47 @@ jobs: - uses: ./.github/actions/ci-optimization id: ci-optimize + - name: Determine runner type + id: set-runner + # This needs to handle two scenarios: + # 1. Running on a PR from a fork. There are some auth issues that prevent us from using depot in that case. + # So, Its easier to just use the regular github actions cache and build all images for each parallel job running smoke test. + # Note, concurrency is lower when using github runners, queue times can be longer, test time is longer due to fewer parallel jobs. + # 2. Running on a PR from a branch in the datahub-project org and push/schedule events on master. + # Depot is used here for remote container builds in base_build and also for all runners. Depot runners support unlimited concurrency + # and hence short queue times and higher parallelism of smoke tests + + run: | + if [[ "${{ env.DOCKER_CACHE }}" == "DEPOT" && "${{ env.DEPOT_PROJECT_ID }}" != "" ]]; then + echo "build_runner_type=depot-ubuntu-24.04-4" >> "$GITHUB_OUTPUT" + echo "test_runner_type=depot-ubuntu-24.04-4" >> "$GITHUB_OUTPUT" + echo "use_depot_cache=true" >> "$GITHUB_OUTPUT" + else + echo "build_runner_type=ubuntu-latest" >> "$GITHUB_OUTPUT" + echo "test_runner_type=ubuntu-latest" >> "$GITHUB_OUTPUT" + echo "use_depot_cache=false" >> "$GITHUB_OUTPUT" + # publishing is currently only supported via depot + fi + + - name: Check whether to run publishing build + id: run-publish-images + run: | + if [[ "${{ steps.set-runner.outputs.use_depot_cache }}" == 'true' && + ( "${{ steps.publish.outputs.publish }}" == 'true' || + "${{ steps.pr-publish.outputs.pr-publish }}" == 'true' || + "${{ github.event_name }}" == 'workflow_dispatch' || + "${{ github.event_name }}" == 'schedule') ]]; then + echo "run_publish_images=true" >> "$GITHUB_OUTPUT" + else + echo "run_publish_images=false" >> "$GITHUB_OUTPUT" + fi + + smoke_test_lint: name: Lint on smoke tests runs-on: depot-ubuntu-24.04 needs: setup - if: ${{ needs.setup.outputs.smoke_test_change }}" == 'true' }} + if: ${{ needs.setup.outputs.smoke_test_change == 'true' }} steps: - name: Check out the repo uses: acryldata/sane-checkout-action@v3 @@ -158,11 +194,13 @@ jobs: ./gradlew :smoke-test:cypressLint base_build: - name: Prepare all images - runs-on: depot-ubuntu-24.04-4 + name: Build all images + runs-on: ${{ needs.setup.outputs.build_runner_type }} needs: setup + if: ${{ needs.setup.outputs.use_depot_cache == 'true' }} # On fork, smoke test job does the build since depot cache is not available outputs: build_id: ${{ steps.capture-build-id.outputs.build_id }} + matrix: ${{ steps.capture-build-id.outputs.matrix }} steps: - name: Set up JDK 17 uses: actions/setup-java@v4 @@ -170,18 +208,21 @@ jobs: distribution: "zulu" java-version: 17 - #- uses: gradle/actions/setup-gradle@v4 - - uses: actions/cache@v4 + - uses: actions/cache/restore@v3 with: path: | ~/.cache/uv key: ${{ runner.os }}-uv-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-uv- - - uses: actions/cache@v4 + - uses: actions/cache/restore@v3 with: path: | ~/.cache/yarn key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} + restore-keys: | + ${{ runner.os }}-yarn- - name: Set up Depot CLI if: ${{ env.DOCKER_CACHE == 'DEPOT' }} @@ -195,209 +236,66 @@ jobs: python-version: "3.10" cache: "pip" - - name: Build all Docker Contexts - if: ${{ env.DOCKER_CACHE == 'DEPOT' && env.DEPOT_PROJECT_ID != '' }} + - name: Login to DockerHub + uses: docker/login-action@v3 + if: ${{ needs.setup.outputs.docker-login == 'true' }} + with: + username: ${{ secrets.ACRYL_DOCKER_USERNAME }} + password: ${{ secrets.ACRYL_DOCKER_PASSWORD }} + + - name: Build all Images (For Smoke tests) + if: ${{ needs.setup.outputs.run_publish_images == 'false' }} + # If not publishing, just a subset of images required for smoke tests is sufficient. run: | - ./gradlew :docker:buildImagesQuickstartDebugConsumers -PreleaseVersion=pr-build + ./gradlew :docker:buildImagesQuickStartDebugConsumers -Ptag=${{ needs.setup.outputs.tag }} -PpythonDockerVersion=${{ needs.setup.outputs.python_release_version }} -PdockerRegistry=${{ env.DOCKER_REGISTRY }} + + - name: Build all Images (Publish) + if: ${{ needs.setup.outputs.run_publish_images == 'true'}} + run: | + ./gradlew :docker:buildImagesAll -PmatrixBuild=true -Ptag=${{ needs.setup.outputs.tag }} -PshaTag=${{ needs.setup.outputs.short_sha }} -PpythonDockerVersion=${{ needs.setup.outputs.python_release_version }} -PdockerRegistry=${{ env.DOCKER_REGISTRY }} -PdockerPush=true - name: Capture build Id id: capture-build-id - if: ${{ env.DOCKER_CACHE == 'DEPOT' && env.DEPOT_PROJECT_ID != '' }} run: | pip install jq DEPOT_BUILD_ID=$(jq -r '.["depot.build"]?.buildID' ${{ github.workspace }}/build/build-metadata.json) echo "build_id=${DEPOT_BUILD_ID}" >> "$GITHUB_OUTPUT" + echo "matrix=$(jq -c '{"target":.["depot.build"].targets}' ${{ github.workspace }}/build/build-metadata.json)" >> $GITHUB_OUTPUT - gms_build: - name: Build and Push DataHub GMS Docker Image - runs-on: ubuntu-latest - needs: [setup, base_build] - if: ${{ needs.setup.outputs.backend_change == 'true' || needs.setup.outputs.publish == 'true' }} - steps: - - name: Free up disk space - run: | - sudo apt-get remove 'dotnet-*' azure-cli || true - sudo rm -rf /usr/local/lib/android/ || true - sudo docker image prune -a -f || true - - name: Set up JDK 17 - uses: actions/setup-java@v4 + - uses: actions/cache/save@v4 + if: ${{ github.ref == 'refs/heads/master' }} with: - distribution: "zulu" - java-version: 17 - - uses: gradle/actions/setup-gradle@v4 - - name: Check out the repo - uses: acryldata/sane-checkout-action@v3 - - name: Pre-build artifacts for docker image - run: | - ./gradlew :metadata-service:war:dockerPrepare + path: | + ~/.cache/uv + key: ${{ runner.os }}-uv-${{ hashFiles('**/requirements.txt') }} - - name: Build and push - uses: ./.github/actions/docker-custom-build-and-push + - uses: actions/cache/save@v4 + if: ${{ github.ref == 'refs/heads/master' }} with: - images: | - ${{ env.DATAHUB_GMS_IMAGE }} - image_tag: ${{ needs.setup.outputs.tag }} - username: ${{ secrets.ACRYL_DOCKER_USERNAME }} - password: ${{ secrets.ACRYL_DOCKER_PASSWORD }} - publish: ${{ needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }} - context: ./build/dockerBuildContext/metadata-service/war/docker - file: ./docker/datahub-gms/Dockerfile - platforms: linux/amd64,linux/arm64/v8 - gms_scan: - permissions: - contents: read # for actions/checkout to fetch code - security-events: write # for github/codeql-action/upload-sarif to upload SARIF results - actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status - name: "[Monitoring] Scan GMS images for vulnerabilities" - runs-on: ubuntu-latest - needs: [setup, gms_build] - if: ${{ needs.setup.outputs.backend_change == 'true' || needs.setup.outputs.publish == 'true' }} - steps: - - name: Checkout # adding checkout step just to make trivy upload happy - uses: acryldata/sane-checkout-action@v3 - - name: Download image - uses: ishworkh/container-image-artifact-download@v2.0.0 - if: ${{ needs.setup.outputs.publish != 'true' && needs.setup.outputs.pr-publish != 'true' }} - with: - image: ${{ env.DATAHUB_GMS_IMAGE }}:${{ needs.setup.outputs.unique_tag }} - - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@0.30.0 - env: - TRIVY_OFFLINE_SCAN: true - TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2,ghcr.io/aquasecurity/trivy-db:2 - TRIVY_JAVA_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-java-db:1,ghcr.io/aquasecurity/trivy-java-db:1 - with: - image-ref: ${{ env.DATAHUB_GMS_IMAGE }}:${{ needs.setup.outputs.unique_tag }} - format: "template" - template: "@/contrib/sarif.tpl" - output: "trivy-results.sarif" - severity: "CRITICAL,HIGH" - ignore-unfixed: true - vuln-type: "os,library" - - name: Upload Trivy scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@v3 - with: - sarif_file: "trivy-results.sarif" + path: | + ~/.cache/yarn + key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} - mae_consumer_build: - name: Build and Push DataHub MAE Consumer Docker Image - runs-on: ubuntu-latest - needs: [setup, smoke_test_lint, base_build] - if: ${{ needs.setup.outputs.backend_change == 'true' || needs.setup.outputs.publish == 'true' }} - steps: - - name: Free up disk space - run: | - sudo apt-get remove 'dotnet-*' azure-cli || true - sudo rm -rf /usr/local/lib/android/ || true - sudo docker image prune -a -f || true - - name: Set up JDK 17 - uses: actions/setup-java@v4 - with: - distribution: "zulu" - java-version: 17 - - uses: gradle/actions/setup-gradle@v4 - - name: Check out the repo - uses: acryldata/sane-checkout-action@v3 - - name: Pre-build artifacts for docker image - run: | - ./gradlew :metadata-jobs:mae-consumer-job:dockerPrepare - - name: Build and push - uses: ./.github/actions/docker-custom-build-and-push - with: - images: | - ${{ env.DATAHUB_MAE_CONSUMER_IMAGE }} - image_tag: ${{ needs.setup.outputs.tag }} - username: ${{ secrets.ACRYL_DOCKER_USERNAME }} - password: ${{ secrets.ACRYL_DOCKER_PASSWORD }} - publish: ${{ needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }} - context: ./build/dockerBuildContext/metadata-jobs/mae-consumer-job/docker - file: ./docker/datahub-mae-consumer/Dockerfile - platforms: linux/amd64,linux/arm64/v8 - mae_consumer_scan: - name: "[Monitoring] Scan MAE consumer images for vulnerabilities" + + scan_images: + name: Scan images for vulnerabilities runs-on: depot-ubuntu-24.04 - needs: [setup, smoke_test_lint, mae_consumer_build] - if: ${{ needs.setup.outputs.backend_change == 'true' || needs.setup.outputs.publish == 'true' }} - permissions: - contents: read # for actions/checkout to fetch code - security-events: write # for github/codeql-action/upload-sarif to upload SARIF results - actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status - steps: - - name: Checkout # adding checkout step just to make trivy upload happy - uses: acryldata/sane-checkout-action@v3 - - name: Download image - uses: ishworkh/container-image-artifact-download@v2.0.0 - if: ${{ needs.setup.outputs.publish != 'true' && needs.setup.outputs.pr-publish != 'true' }} - with: - image: ${{ env.DATAHUB_MAE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }} - - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@0.30.0 - env: - TRIVY_OFFLINE_SCAN: true - TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2,ghcr.io/aquasecurity/trivy-db:2 - TRIVY_JAVA_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-java-db:1,ghcr.io/aquasecurity/trivy-java-db:1 - with: - image-ref: ${{ env.DATAHUB_MAE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }} - format: "template" - template: "@/contrib/sarif.tpl" - output: "trivy-results.sarif" - severity: "CRITICAL,HIGH" - ignore-unfixed: true - vuln-type: "os,library" - - name: Upload Trivy scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@v3 - with: - sarif_file: "trivy-results.sarif" - - mce_consumer_build: - name: Build and Push DataHub MCE Consumer Docker Image - runs-on: ubuntu-latest needs: [setup, base_build] - if: ${{ needs.setup.outputs.backend_change == 'true' || needs.setup.outputs.publish == 'true' }} + if: ${{ needs.setup.outputs.run_publish_images == 'true' }} + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.base_build.outputs.matrix) }} steps: - - name: Set up JDK 17 - uses: actions/setup-java@v4 - with: - distribution: "zulu" - java-version: 17 - - uses: gradle/actions/setup-gradle@v4 - - name: Check out the repo - uses: acryldata/sane-checkout-action@v3 - - name: Pre-build artifacts for docker image + - id: download_image + name: Download images from depot + if: ${{ needs.setup.outputs.use_depot_cache == 'true' }} run: | - ./gradlew :metadata-jobs:mce-consumer-job:dockerPrepare + depot pull --project ${{ env.DEPOT_PROJECT_ID }} ${{ needs.base_build.outputs.build_id }} --target ${{ matrix.target}} + docker images + echo "docker_image=$(docker images --format '{{.Repository}}:{{.Tag}}' | grep ${{ needs.setup.outputs.tag }} )" >> $GITHUB_OUTPUT - - name: Build and push - uses: ./.github/actions/docker-custom-build-and-push - with: - images: | - ${{ env.DATAHUB_MCE_CONSUMER_IMAGE }} - image_tag: ${{ needs.setup.outputs.tag }} - username: ${{ secrets.ACRYL_DOCKER_USERNAME }} - password: ${{ secrets.ACRYL_DOCKER_PASSWORD }} - publish: ${{ needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }} - context: ./build/dockerBuildContext/metadata-jobs/mce-consumer-job/docker - file: ./docker/datahub-mce-consumer/Dockerfile - platforms: linux/amd64,linux/arm64/v8 - mce_consumer_scan: - name: "[Monitoring] Scan MCE consumer images for vulnerabilities" - runs-on: depot-ubuntu-24.04 - needs: [setup, smoke_test_lint, mce_consumer_build] - if: ${{ needs.setup.outputs.backend_change == 'true' || needs.setup.outputs.publish == 'true' }} - permissions: - contents: read # for actions/checkout to fetch code - security-events: write # for github/codeql-action/upload-sarif to upload SARIF results - actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status - steps: - - name: Checkout # adding checkout step just to make trivy upload happy - uses: acryldata/sane-checkout-action@v3 - - name: Download image - uses: ishworkh/container-image-artifact-download@v2.0.0 - if: ${{ needs.setup.outputs.publish != 'true' && needs.setup.outputs.pr-publish != 'true' }} - with: - image: ${{ env.DATAHUB_MCE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }} - name: Run Trivy vulnerability scanner uses: aquasecurity/trivy-action@0.30.0 env: @@ -405,761 +303,7 @@ jobs: TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2,ghcr.io/aquasecurity/trivy-db:2 TRIVY_JAVA_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-java-db:1,ghcr.io/aquasecurity/trivy-java-db:1 with: - image-ref: ${{ env.DATAHUB_MCE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }} - format: "template" - template: "@/contrib/sarif.tpl" - output: "trivy-results.sarif" - severity: "CRITICAL,HIGH" - ignore-unfixed: true - vuln-type: "os,library" - - name: Upload Trivy scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@v3 - with: - sarif_file: "trivy-results.sarif" - - datahub_upgrade_build: - name: Build and Push DataHub Upgrade Docker Image - runs-on: ubuntu-latest - needs: [setup, base_build] - if: ${{ needs.setup.outputs.backend_change == 'true' || needs.setup.outputs.publish == 'true' }} - steps: - - name: Free up disk space - run: | - sudo apt-get remove 'dotnet-*' azure-cli || true - sudo rm -rf /usr/local/lib/android/ || true - sudo docker image prune -a -f || true - - name: Set up JDK 17 - uses: actions/setup-java@v4 - with: - distribution: "zulu" - java-version: 17 - - uses: gradle/actions/setup-gradle@v4 - - name: Check out the repo - uses: acryldata/sane-checkout-action@v3 - - name: Pre-build artifacts for docker image - run: | - ./gradlew :datahub-upgrade:dockerPrepare - - - name: Build and push - uses: ./.github/actions/docker-custom-build-and-push - with: - images: | - ${{ env.DATAHUB_UPGRADE_IMAGE }} - image_tag: ${{ needs.setup.outputs.tag }} - username: ${{ secrets.ACRYL_DOCKER_USERNAME }} - password: ${{ secrets.ACRYL_DOCKER_PASSWORD }} - publish: ${{ needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }} - context: ./build/dockerBuildContext/datahub-upgrade/docker - file: ./docker/datahub-upgrade/Dockerfile - platforms: linux/amd64,linux/arm64/v8 - datahub_upgrade_scan: - name: "[Monitoring] Scan DataHub Upgrade images for vulnerabilities" - runs-on: ubuntu-latest - needs: [setup, smoke_test_lint, datahub_upgrade_build] - if: ${{ needs.setup.outputs.backend_change == 'true' || needs.setup.outputs.publish == 'true' }} - permissions: - contents: read # for actions/checkout to fetch code - security-events: write # for github/codeql-action/upload-sarif to upload SARIF results - actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status - steps: - - name: Checkout # adding checkout step just to make trivy upload happy - uses: acryldata/sane-checkout-action@v3 - - name: Download image - uses: ishworkh/container-image-artifact-download@v2.0.0 - if: ${{ needs.setup.outputs.publish != 'true' && needs.setup.outputs.pr-publish != 'true' }} - with: - image: ${{ env.DATAHUB_UPGRADE_IMAGE }}:${{ needs.setup.outputs.unique_tag }} - - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@0.30.0 - env: - TRIVY_OFFLINE_SCAN: true - TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2,ghcr.io/aquasecurity/trivy-db:2 - TRIVY_JAVA_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-java-db:1,ghcr.io/aquasecurity/trivy-java-db:1 - with: - image-ref: ${{ env.DATAHUB_UPGRADE_IMAGE }}:${{ needs.setup.outputs.unique_tag }} - format: "template" - template: "@/contrib/sarif.tpl" - output: "trivy-results.sarif" - severity: "CRITICAL,HIGH" - ignore-unfixed: true - vuln-type: "os,library" - - name: Upload Trivy scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@v3 - with: - sarif_file: "trivy-results.sarif" - - frontend_build: - name: Build and Push DataHub Frontend Docker Image - runs-on: ubuntu-latest - needs: [setup, base_build] - if: ${{ needs.setup.outputs.frontend_change == 'true' || needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true'}} - steps: - - name: Free up disk space - run: | - sudo apt-get remove 'dotnet-*' azure-cli || true - sudo rm -rf /usr/local/lib/android/ || true - sudo docker image prune -a -f || true - - name: Set up JDK 17 - uses: actions/setup-java@v4 - with: - distribution: "zulu" - java-version: 17 - - uses: gradle/actions/setup-gradle@v4 - - name: Check out the repo - uses: acryldata/sane-checkout-action@v3 - - name: Pre-build artifacts for docker image - run: | - ./gradlew :datahub-frontend:dockerPrepare -x test -x yarnTest -x yarnLint --parallel - - - name: Build and push - uses: ./.github/actions/docker-custom-build-and-push - with: - images: | - ${{ env.DATAHUB_FRONTEND_IMAGE }} - image_tag: ${{ needs.setup.outputs.tag }} - username: ${{ secrets.ACRYL_DOCKER_USERNAME }} - password: ${{ secrets.ACRYL_DOCKER_PASSWORD }} - publish: ${{ needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }} - context: ./build/dockerBuildContext/datahub-frontend/docker - file: ./docker/datahub-frontend/Dockerfile - platforms: linux/amd64,linux/arm64/v8 - frontend_scan: - name: "[Monitoring] Scan Frontend images for vulnerabilities" - runs-on: ubuntu-24.04 - needs: [setup, smoke_test_lint, frontend_build] - if: ${{ needs.setup.outputs.frontend_change == 'true' || needs.setup.outputs.publish == 'true' }} - permissions: - contents: read # for actions/checkout to fetch code - security-events: write # for github/codeql-action/upload-sarif to upload SARIF results - actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status - steps: - - name: Checkout # adding checkout step just to make trivy upload happy - uses: actions/checkout@v4 - - name: Download image - uses: ishworkh/container-image-artifact-download@v2.0.0 - if: ${{ needs.setup.outputs.publish != 'true' && needs.setup.outputs.pr-publish != 'true' }} - with: - image: ${{ env.DATAHUB_FRONTEND_IMAGE }}:${{ needs.setup.outputs.unique_tag }} - - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@0.30.0 - env: - TRIVY_OFFLINE_SCAN: true - TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2,ghcr.io/aquasecurity/trivy-db:2 - TRIVY_JAVA_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-java-db:1,ghcr.io/aquasecurity/trivy-java-db:1 - with: - image-ref: ${{ env.DATAHUB_FRONTEND_IMAGE }}:${{ needs.setup.outputs.unique_tag }} - format: "template" - template: "@/contrib/sarif.tpl" - output: "trivy-results.sarif" - severity: "CRITICAL,HIGH" - ignore-unfixed: true - vuln-type: "os,library" - - name: Upload Trivy scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@v3 - with: - sarif_file: "trivy-results.sarif" - - kafka_setup_build: - name: Build and Push DataHub Kafka Setup Docker Image - runs-on: ubuntu-latest - needs: [setup, base_build] - if: ${{ needs.setup.outputs.kafka_setup_change == 'true' || (needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true') }} - steps: - - name: Free up disk space - run: | - sudo apt-get remove 'dotnet-*' azure-cli || true - sudo rm -rf /usr/local/lib/android/ || true - sudo docker image prune -a -f || true - - name: Check out the repo - uses: acryldata/sane-checkout-action@v3 - - name: Build and push - uses: ./.github/actions/docker-custom-build-and-push - with: - images: | - ${{ env.DATAHUB_KAFKA_SETUP_IMAGE }} - image_tag: ${{ needs.setup.outputs.tag }} - username: ${{ secrets.ACRYL_DOCKER_USERNAME }} - password: ${{ secrets.ACRYL_DOCKER_PASSWORD }} - publish: ${{ needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }} - context: . - file: ./docker/kafka-setup/Dockerfile - platforms: linux/amd64,linux/arm64/v8 - kafka_setup_scan: - permissions: - contents: read # for actions/checkout to fetch code - security-events: write # for github/codeql-action/upload-sarif to upload SARIF results - actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status - name: "[Monitoring] Scan Kafka Setup images for vulnerabilities" - runs-on: ubuntu-latest - needs: [setup, smoke_test_lint, kafka_setup_build] - if: ${{ needs.setup.outputs.kafka_setup_change == 'true' || (needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true') }} - steps: - - name: Checkout # adding checkout step just to make trivy upload happy - uses: acryldata/sane-checkout-action@v3 - - name: Download image - uses: ishworkh/container-image-artifact-download@v2.0.0 - if: ${{ needs.setup.outputs.publish != 'true' && needs.setup.outputs.pr-publish != 'true' }} - with: - image: ${{ env.DATAHUB_KAFKA_SETUP_IMAGE }}:${{ needs.setup.outputs.unique_tag }} - - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@0.30.0 - env: - TRIVY_OFFLINE_SCAN: true - TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2,ghcr.io/aquasecurity/trivy-db:2 - TRIVY_JAVA_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-java-db:1,ghcr.io/aquasecurity/trivy-java-db:1 - with: - image-ref: ${{ env.DATAHUB_KAFKA_SETUP_IMAGE }}:${{ needs.setup.outputs.unique_tag }} - format: "template" - template: "@/contrib/sarif.tpl" - output: "trivy-results.sarif" - severity: "CRITICAL,HIGH" - ignore-unfixed: true - vuln-type: "os,library" - - name: Upload Trivy scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@v3 - with: - sarif_file: "trivy-results.sarif" - - mysql_setup_build: - name: Build and Push DataHub MySQL Setup Docker Image - runs-on: ubuntu-latest - needs: [setup, base_build] - if: ${{ false || needs.setup.outputs.mysql_setup_change == 'true' || (needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true') }} - steps: - - name: Free up disk space - run: | - sudo apt-get remove 'dotnet-*' azure-cli || true - sudo rm -rf /usr/local/lib/android/ || true - sudo docker image prune -a -f || true - - name: Check out the repo - uses: acryldata/sane-checkout-action@v3 - - - name: Build and push - uses: ./.github/actions/docker-custom-build-and-push - with: - images: | - ${{ env.DATAHUB_MYSQL_SETUP_IMAGE }} - image_tag: ${{ needs.setup.outputs.tag }} - username: ${{ secrets.ACRYL_DOCKER_USERNAME }} - password: ${{ secrets.ACRYL_DOCKER_PASSWORD }} - publish: ${{ needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }} - context: . - file: ./docker/mysql-setup/Dockerfile - platforms: linux/amd64,linux/arm64/v8 - mysql_setup_scan: - permissions: - contents: read # for actions/checkout to fetch code - security-events: write # for github/codeql-action/upload-sarif to upload SARIF results - actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status - name: "[Monitoring] Scan MySQL Setup images for vulnerabilities" - runs-on: ubuntu-latest - needs: [setup, smoke_test_lint, mysql_setup_build] - if: ${{ needs.setup.outputs.mysql_setup_change == 'true' || (needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true') }} - steps: - - name: Checkout # adding checkout step just to make trivy upload happy - uses: acryldata/sane-checkout-action@v3 - - name: Download image - uses: ishworkh/container-image-artifact-download@v2.0.0 - if: ${{ needs.setup.outputs.publish != 'true' && needs.setup.outputs.pr-publish != 'true' }} - with: - image: ${{ env.DATAHUB_MYSQL_SETUP_IMAGE }}:${{ needs.setup.outputs.unique_tag }} - - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@0.30.0 - env: - TRIVY_OFFLINE_SCAN: true - TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2,ghcr.io/aquasecurity/trivy-db:2 - TRIVY_JAVA_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-java-db:1,ghcr.io/aquasecurity/trivy-java-db:1 - with: - image-ref: ${{ env.DATAHUB_MYSQL_SETUP_IMAGE }}:${{ needs.setup.outputs.unique_tag }} - format: "template" - template: "@/contrib/sarif.tpl" - output: "trivy-results.sarif" - severity: "CRITICAL,HIGH" - ignore-unfixed: true - vuln-type: "os,library" - - name: Upload Trivy scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@v3 - with: - sarif_file: "trivy-results.sarif" - - elasticsearch_setup_build: - name: Build and Push DataHub Elasticsearch Setup Docker Image - runs-on: ubuntu-latest - needs: [setup, base_build] - if: ${{ needs.setup.outputs.elasticsearch_setup_change == 'true' || (needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' ) }} - steps: - - name: Free up disk space - run: | - sudo apt-get remove 'dotnet-*' azure-cli || true - sudo rm -rf /usr/local/lib/android/ || true - sudo docker image prune -a -f || true - - name: Check out the repo - uses: acryldata/sane-checkout-action@v3 - - - name: Build and push - uses: ./.github/actions/docker-custom-build-and-push - with: - images: | - ${{ env.DATAHUB_ELASTIC_SETUP_IMAGE }} - image_tag: ${{ needs.setup.outputs.tag }} - username: ${{ secrets.ACRYL_DOCKER_USERNAME }} - password: ${{ secrets.ACRYL_DOCKER_PASSWORD }} - publish: ${{ needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }} - context: . - file: ./docker/elasticsearch-setup/Dockerfile - platforms: linux/amd64,linux/arm64/v8 - elasticsearch_setup_scan: - permissions: - contents: read # for actions/checkout to fetch code - security-events: write # for github/codeql-action/upload-sarif to upload SARIF results - actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status - name: "[Monitoring] Scan ElasticSearch setup images for vulnerabilities" - runs-on: ubuntu-latest - needs: [setup, elasticsearch_setup_build] - if: ${{ needs.setup.outputs.elasticsearch_setup_change == 'true' || (needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' ) }} - steps: - - name: Checkout # adding checkout step just to make trivy upload happy - uses: acryldata/sane-checkout-action@v3 - - name: Download image - uses: ishworkh/container-image-artifact-download@v2.0.0 - if: ${{ needs.setup.outputs.publish != 'true' && needs.setup.outputs.pr-publish != 'true' }} - with: - image: ${{ env.DATAHUB_ELASTIC_SETUP_IMAGE }}:${{ needs.setup.outputs.unique_tag }} - - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@0.30.0 - env: - TRIVY_OFFLINE_SCAN: true - TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2,ghcr.io/aquasecurity/trivy-db:2 - TRIVY_JAVA_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-java-db:1,ghcr.io/aquasecurity/trivy-java-db:1 - with: - image-ref: ${{ env.DATAHUB_ELASTIC_SETUP_IMAGE }}:${{ needs.setup.outputs.unique_tag }} - format: "template" - template: "@/contrib/sarif.tpl" - output: "trivy-results.sarif" - severity: "CRITICAL,HIGH" - ignore-unfixed: true - vuln-type: "os,library" - - name: Upload Trivy scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@v3 - with: - sarif_file: "trivy-results.sarif" - - datahub_ingestion_base_build: - name: Build and Push DataHub Ingestion (Base) Docker Image - runs-on: ubuntu-latest - outputs: - tag: ${{ steps.tag.outputs.tag }} - needs: setup - if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }} - steps: - - name: Free up disk space - run: | - sudo apt-get remove 'dotnet-*' azure-cli || true - sudo rm -rf /usr/local/lib/android/ || true - sudo docker image prune -a -f || true - - name: Check out the repo - uses: acryldata/sane-checkout-action@v3 - - - name: Build and push Base Image - if: ${{ needs.setup.outputs.ingestion_base_change == 'true' }} - uses: ./.github/actions/docker-custom-build-and-push - with: - images: | - ${{ env.DATAHUB_INGESTION_BASE_IMAGE }} - image_tag: ${{ needs.setup.outputs.tag }} - username: ${{ secrets.ACRYL_DOCKER_USERNAME }} - password: ${{ secrets.ACRYL_DOCKER_PASSWORD }} - publish: ${{ needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }} - context: . - file: ./docker/datahub-ingestion-base/Dockerfile - platforms: linux/amd64,linux/arm64/v8 - depot-project: ${{ vars.DEPOT_PROJECT_ID }} - - name: Compute DataHub Ingestion (Base) Tag - id: tag - run: echo "tag=${{ needs.setup.outputs.ingestion_base_change == 'true' && needs.setup.outputs.unique_tag || 'head' }}" >> "$GITHUB_OUTPUT" - datahub_ingestion_base_slim_build: - name: Build and Push DataHub Ingestion (Base-Slim) Docker Image - runs-on: ubuntu-latest - outputs: - tag: ${{ steps.tag.outputs.tag }} - needs: [setup, smoke_test_lint, datahub_ingestion_base_build] - if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }} - steps: - - name: Free up disk space - run: | - sudo apt-get remove 'dotnet-*' azure-cli || true - sudo rm -rf /usr/local/lib/android/ || true - sudo docker image prune -a -f || true - - name: Check out the repo - uses: acryldata/sane-checkout-action@v3 - - name: Download Base Image - uses: ishworkh/container-image-artifact-download@v2.0.0 - if: ${{ needs.setup.outputs.publish != 'true' && needs.setup.outputs.pr-publish != 'true' && needs.setup.outputs.ingestion_base_change == 'true' }} - with: - image: ${{ env.DATAHUB_INGESTION_BASE_IMAGE }}:${{ needs.setup.outputs.ingestion_base_change == 'true' && needs.setup.outputs.unique_tag || 'head' }} - - name: Login to DockerHub - uses: docker/login-action@v3 - if: ${{ needs.setup.outputs.docker-login == 'true' && needs.setup.outputs.publish == 'false' && needs.setup.outputs.pr-publish == 'false' && needs.setup.outputs.ingestion_base_change == 'false' }} - with: - username: ${{ secrets.ACRYL_DOCKER_USERNAME }} - password: ${{ secrets.ACRYL_DOCKER_PASSWORD }} - - name: Build and push Base-Slim Image - if: ${{ needs.setup.outputs.ingestion_base_change == 'true' }} - uses: ./.github/actions/docker-custom-build-and-push - with: - images: | - ${{ env.DATAHUB_INGESTION_BASE_IMAGE }} - image_tag: ${{ needs.setup.outputs.slim_tag }} - flavor: slim - username: ${{ secrets.ACRYL_DOCKER_USERNAME }} - password: ${{ secrets.ACRYL_DOCKER_PASSWORD }} - build-args: | - APP_ENV=slim - BASE_IMAGE=${{ env.DATAHUB_INGESTION_BASE_IMAGE }}:${{ needs.setup.outputs.ingestion_base_change == 'true' && needs.setup.outputs.unique_tag || 'head' }} - publish: ${{ needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }} - context: . - file: ./docker/datahub-ingestion-base/Dockerfile - platforms: linux/amd64,linux/arm64/v8 - depot-project: ${{ vars.DEPOT_PROJECT_ID }} - - name: Compute DataHub Ingestion (Base-Slim) Tag - id: tag - run: echo "tag=${{ needs.setup.outputs.ingestion_base_change == 'true' && needs.setup.outputs.unique_slim_tag || 'head-slim' }}" >> "$GITHUB_OUTPUT" - datahub_ingestion_base_full_build: - name: Build and Push DataHub Ingestion (Base-Full) Docker Image - runs-on: ubuntu-latest - outputs: - tag: ${{ steps.tag.outputs.tag }} - needs: [setup, smoke_test_lint, datahub_ingestion_base_build] - if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }} - steps: - - name: Check out the repo - uses: acryldata/sane-checkout-action@v3 - - name: Download Base Image - uses: ishworkh/container-image-artifact-download@v2.0.0 - if: ${{ needs.setup.outputs.publish != 'true' && needs.setup.outputs.pr-publish != 'true' && needs.setup.outputs.ingestion_base_change == 'true' }} - with: - image: ${{ env.DATAHUB_INGESTION_BASE_IMAGE }}:${{ needs.setup.outputs.ingestion_base_change == 'true' && needs.setup.outputs.unique_tag || 'head' }} - - name: Login to DockerHub - uses: docker/login-action@v3 - if: ${{ needs.setup.outputs.docker-login == 'true' && needs.setup.outputs.publish == 'false' && needs.setup.outputs.pr-publish == 'false' && needs.setup.outputs.ingestion_base_change == 'false' }} - with: - username: ${{ secrets.ACRYL_DOCKER_USERNAME }} - password: ${{ secrets.ACRYL_DOCKER_PASSWORD }} - - name: Build and push (Base-Full) Image - if: ${{ needs.setup.outputs.ingestion_base_change == 'true' }} - uses: ./.github/actions/docker-custom-build-and-push - with: - images: | - ${{ env.DATAHUB_INGESTION_BASE_IMAGE }} - image_tag: ${{ needs.setup.outputs.full_tag }} - username: ${{ secrets.ACRYL_DOCKER_USERNAME }} - password: ${{ secrets.ACRYL_DOCKER_PASSWORD }} - build-args: | - APP_ENV=full - BASE_IMAGE=${{ env.DATAHUB_INGESTION_BASE_IMAGE }}:${{ needs.setup.outputs.ingestion_base_change == 'true' && needs.setup.outputs.unique_tag || 'head' }} - publish: ${{ needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }} - context: . - file: ./docker/datahub-ingestion-base/Dockerfile - platforms: linux/amd64,linux/arm64/v8 - - name: Compute DataHub Ingestion (Base-Full) Tag - id: tag - run: echo "tag=${{ needs.setup.outputs.ingestion_base_change == 'true' && needs.setup.outputs.unique_full_tag || 'head' }}" >> "$GITHUB_OUTPUT" - - datahub_ingestion_slim_build: - name: Build and Push DataHub Ingestion Docker Images - runs-on: ubuntu-latest - outputs: - tag: ${{ steps.tag.outputs.tag }} - needs_artifact_download: ${{ needs.setup.outputs.ingestion_change == 'true' && ( needs.setup.outputs.publish != 'true' && needs.setup.outputs.pr-publish != 'true') }} - needs: [setup, smoke_test_lint] - if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }} - steps: - - name: Check out the repo - uses: acryldata/sane-checkout-action@v3 - - uses: actions/setup-python@v5 - with: - python-version: "3.10" - cache: "pip" - - name: Set up JDK 17 - uses: actions/setup-java@v4 - with: - distribution: "zulu" - java-version: 17 - - name: Build codegen - if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish =='true' }} - run: ./gradlew :metadata-ingestion:codegen - - name: Login to DockerHub - uses: docker/login-action@v3 - if: ${{ needs.setup.outputs.docker-login == 'true' && needs.setup.outputs.publish == 'false' && needs.setup.outputs.pr-publish == 'false' && needs.setup.outputs.ingestion_base_change == 'false' }} - with: - username: ${{ secrets.ACRYL_DOCKER_USERNAME }} - password: ${{ secrets.ACRYL_DOCKER_PASSWORD }} - - name: Build and push Slim Image - if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }} - uses: ./.github/actions/docker-custom-build-and-push - with: - images: | - ${{ env.DATAHUB_INGESTION_IMAGE }} - build-args: | - RELEASE_VERSION=${{ needs.setup.outputs.python_release_version }} - APP_ENV=slim - image_tag: ${{ needs.setup.outputs.slim_tag }} - flavor: slim - username: ${{ secrets.ACRYL_DOCKER_USERNAME }} - password: ${{ secrets.ACRYL_DOCKER_PASSWORD }} - publish: ${{ needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }} - context: . - file: ./docker/datahub-ingestion/Dockerfile - platforms: linux/amd64,linux/arm64/v8 - # Workaround 2025-01-25 - Depot publishing errors - depot-project: ${{ (startsWith(github.ref, 'refs/tags/') || github.event_name == 'release') && '' || vars.DEPOT_PROJECT_ID }} - - name: Compute Tag - id: tag - run: echo "tag=${{ needs.setup.outputs.ingestion_change == 'true' && needs.setup.outputs.unique_slim_tag || 'head-slim' }}" >> "$GITHUB_OUTPUT" - datahub_ingestion_slim_scan: - permissions: - contents: read # for actions/checkout to fetch code - security-events: write # for github/codeql-action/upload-sarif to upload SARIF results - actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status - name: "[Monitoring] Scan Datahub Ingestion Slim images for vulnerabilities" - runs-on: ubuntu-latest - needs: [setup, smoke_test_lint, datahub_ingestion_slim_build] - if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' }} - steps: - - name: Checkout # adding checkout step just to make trivy upload happy - uses: acryldata/sane-checkout-action@v3 - - name: Download image Slim Image - uses: ishworkh/container-image-artifact-download@v2.0.0 - if: ${{ needs.datahub_ingestion_slim_build.outputs.needs_artifact_download == 'true' }} - with: - image: ${{ env.DATAHUB_INGESTION_IMAGE }}:${{ needs.datahub_ingestion_slim_build.outputs.tag }} - - name: Run Trivy vulnerability scanner Slim Image - uses: aquasecurity/trivy-action@0.30.0 - env: - TRIVY_OFFLINE_SCAN: true - TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2,ghcr.io/aquasecurity/trivy-db:2 - TRIVY_JAVA_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-java-db:1,ghcr.io/aquasecurity/trivy-java-db:1 - with: - image-ref: ${{ env.DATAHUB_INGESTION_IMAGE }}:${{ needs.datahub_ingestion_slim_build.outputs.tag }} - format: "template" - template: "@/contrib/sarif.tpl" - output: "trivy-results.sarif" - severity: "CRITICAL,HIGH" - ignore-unfixed: true - vuln-type: "os,library" - timeout: 15m - - name: Upload Trivy scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@v3 - with: - sarif_file: "trivy-results.sarif" - - datahub_ingestion_full_build: - name: Build and Push DataHub Ingestion (Full) Docker Images - runs-on: ubuntu-latest - outputs: - tag: ${{ steps.tag.outputs.tag }} - needs_artifact_download: ${{ needs.setup.outputs.ingestion_change == 'true' && ( needs.setup.outputs.publish != 'true' && needs.setup.outputs.pr-publish != 'true' ) }} - needs: [setup, smoke_test_lint] - if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }} - steps: - - name: Check out the repo - uses: acryldata/sane-checkout-action@v3 - - uses: actions/setup-python@v5 - with: - python-version: "3.10" - cache: "pip" - - name: Set up JDK 17 - uses: actions/setup-java@v4 - with: - distribution: "zulu" - java-version: 17 - - uses: gradle/actions/setup-gradle@v4 - - name: Build codegen - if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }} - run: ./gradlew :metadata-ingestion:codegen - - name: Login to DockerHub - uses: docker/login-action@v3 - if: ${{ needs.setup.outputs.docker-login == 'true' && needs.setup.outputs.publish == 'false' && needs.setup.outputs.pr-publish == 'false' && needs.setup.outputs.ingestion_base_change == 'false' }} - with: - username: ${{ secrets.ACRYL_DOCKER_USERNAME }} - password: ${{ secrets.ACRYL_DOCKER_PASSWORD }} - - name: Build and push Full Image - if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }} - uses: ./.github/actions/docker-custom-build-and-push - with: - images: | - ${{ env.DATAHUB_INGESTION_IMAGE }} - build-args: | - RELEASE_VERSION=${{ needs.setup.outputs.python_release_version }} - image_tag: ${{ needs.setup.outputs.tag }} - username: ${{ secrets.ACRYL_DOCKER_USERNAME }} - password: ${{ secrets.ACRYL_DOCKER_PASSWORD }} - publish: ${{ needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }} - context: . - file: ./docker/datahub-ingestion/Dockerfile - platforms: linux/amd64,linux/arm64/v8 - # Workaround 2025-01-25 - Depot publishing errors - depot-project: ${{ (startsWith(github.ref, 'refs/tags/') || github.event_name == 'release') && '' || vars.DEPOT_PROJECT_ID }} - - name: Compute Tag (Full) - id: tag - run: echo "tag=${{ needs.setup.outputs.ingestion_change == 'true' && needs.setup.outputs.unique_tag || 'head' }}" >> "$GITHUB_OUTPUT" - datahub_ingestion_full_scan: - permissions: - contents: read # for actions/checkout to fetch code - security-events: write # for github/codeql-action/upload-sarif to upload SARIF results - actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status - name: "[Monitoring] Scan Datahub Ingestion images for vulnerabilities" - runs-on: depot-ubuntu-24.04 - needs: [setup, smoke_test_lint, datahub_ingestion_full_build] - if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }} - steps: - - name: Checkout # adding checkout step just to make trivy upload happy - uses: acryldata/sane-checkout-action@v3 - - name: Download image Full Image - uses: ishworkh/container-image-artifact-download@v2.0.0 - if: ${{ needs.datahub_ingestion_full_build.outputs.needs_artifact_download == 'true' }} - with: - image: ${{ env.DATAHUB_INGESTION_IMAGE }}:${{ needs.datahub_ingestion_full_build.outputs.tag }} - - name: Run Trivy vulnerability scanner Full Image - uses: aquasecurity/trivy-action@0.30.0 - env: - TRIVY_OFFLINE_SCAN: true - TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2,ghcr.io/aquasecurity/trivy-db:2 - TRIVY_JAVA_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-java-db:1,ghcr.io/aquasecurity/trivy-java-db:1 - with: - image-ref: ${{ env.DATAHUB_INGESTION_IMAGE }}:${{ needs.datahub_ingestion_full_build.outputs.tag }} - format: "template" - template: "@/contrib/sarif.tpl" - output: "trivy-results.sarif" - severity: "CRITICAL,HIGH" - ignore-unfixed: true - vuln-type: "os,library" - timeout: 15m - - name: Upload Trivy scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@v3 - with: - sarif_file: "trivy-results.sarif" - - datahub_actions_build: - name: Build and Push DataHub Actions Docker Image - runs-on: depot-ubuntu-24.04 - needs: [setup] - if: ${{ needs.setup.outputs.actions_change == 'true' || needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true'}} - steps: - - name: Check out the repo - uses: acryldata/sane-checkout-action@v3 - - - name: Set up Depot CLI - if: ${{ env.DOCKER_CACHE == 'DEPOT' }} - uses: depot/setup-action@v1 - - - name: Build and push - uses: ./.github/actions/docker-custom-build-and-push - with: - images: | - ${{ env.DATAHUB_ACTIONS_IMAGE }} - image_tag: ${{ needs.setup.outputs.tag }} - username: ${{ secrets.ACRYL_DOCKER_USERNAME }} - password: ${{ secrets.ACRYL_DOCKER_PASSWORD }} - build-args: | - RELEASE_VERSION=${{ needs.setup.outputs.python_release_version }} - publish: ${{ needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }} - context: . - file: ./docker/datahub-actions/Dockerfile - platforms: linux/amd64,linux/arm64/v8 - datahub_actions_scan: - name: "[Monitoring] Scan Datahub Actions images for vulnerabilities" - runs-on: depot-ubuntu-24.04 - needs: [setup, smoke_test_lint, datahub_actions_build] - if: ${{ needs.setup.outputs.actions_change == 'true' || needs.setup.outputs.publish == 'true' }} - permissions: - contents: read # for actions/checkout to fetch code - security-events: write # for github/codeql-action/upload-sarif to upload SARIF results - actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status - steps: - - name: Checkout # adding checkout step just to make trivy upload happy - uses: actions/checkout@v4 - - name: Download image - uses: ishworkh/container-image-artifact-download@v2.0.0 - if: ${{ needs.setup.outputs.publish != 'true' && needs.setup.outputs.pr-publish != 'true' }} - with: - image: ${{ env.DATAHUB_ACTIONS_IMAGE }}:${{ needs.setup.outputs.unique_tag }} - - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@0.30.0 - env: - TRIVY_OFFLINE_SCAN: true - TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2,ghcr.io/aquasecurity/trivy-db:2 - TRIVY_JAVA_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-java-db:1,ghcr.io/aquasecurity/trivy-java-db:1 - with: - image-ref: ${{ env.DATAHUB_ACTIONS_IMAGE }}:${{ needs.setup.outputs.unique_tag }} - format: "template" - template: "@/contrib/sarif.tpl" - output: "trivy-results.sarif" - severity: "CRITICAL,HIGH" - ignore-unfixed: true - vuln-type: "os,library" - - name: Upload Trivy scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@v3 - with: - sarif_file: "trivy-results.sarif" - datahub_actions_slim_build: - name: Build and Push DataHub Actions Slim Docker Image - runs-on: depot-ubuntu-24.04 - needs: [setup] - outputs: - tag: ${{ steps.tag.outputs.tag }} - needs_artifact_download: ${{ needs.setup.outputs.actions_change == 'true' && ( needs.setup.outputs.publish != 'true' && needs.setup.outputs.pr-publish != 'true') }} - if: ${{ needs.setup.outputs.actions_change == 'true' || needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true'}} - steps: - - name: Check out the repo - uses: acryldata/sane-checkout-action@v3 - - name: Build and push Slim image - if: ${{ needs.setup.outputs.actions_change == 'true' }} - uses: ./.github/actions/docker-custom-build-and-push - with: - images: | - ${{ env.DATAHUB_ACTIONS_IMAGE }} - image_tag: ${{ needs.setup.outputs.slim_tag }} - flavor: slim - username: ${{ secrets.ACRYL_DOCKER_USERNAME }} - password: ${{ secrets.ACRYL_DOCKER_PASSWORD }} - build-args: | - APP_ENV=slim - RELEASE_VERSION=${{ needs.setup.outputs.python_release_version }} - publish: ${{ needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }} - context: . - file: ./docker/datahub-actions/Dockerfile - platforms: linux/amd64,linux/arm64/v8 - depot-project: ${{ vars.DEPOT_PROJECT_ID }} - - name: Compute Tag - id: tag - run: echo "tag=${{ needs.setup.outputs.actions_change == 'true' && needs.setup.outputs.unique_slim_tag || 'head-slim' }}" >> "$GITHUB_OUTPUT" - - datahub_actions_slim_scan: - name: "[Monitoring] Scan Datahub Actions Slim images for vulnerabilities" - runs-on: depot-ubuntu-24.04 - needs: [setup, smoke_test_lint, datahub_actions_slim_build] - if: ${{ needs.setup.outputs.actions_change == 'true' || needs.setup.outputs.publish == 'true' }} - permissions: - contents: read # for actions/checkout to fetch code - security-events: write # for github/codeql-action/upload-sarif to upload SARIF results - actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status - steps: - - name: Checkout # adding checkout step just to make trivy upload happy - uses: actions/checkout@v4 - - name: Download image - uses: ishworkh/container-image-artifact-download@v2.0.0 - if: ${{ needs.datahub_actions_slim_build.outputs.needs_artifact_download == 'true' }} - with: - image: ${{ env.DATAHUB_ACTIONS_IMAGE }}:${{ needs.setup.outputs.slim_tag }} - - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@0.30.0 - env: - TRIVY_OFFLINE_SCAN: true - TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2,ghcr.io/aquasecurity/trivy-db:2 - TRIVY_JAVA_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-java-db:1,ghcr.io/aquasecurity/trivy-java-db:1 - with: - image-ref: ${{ env.DATAHUB_ACTIONS_IMAGE }}:${{ needs.setup.outputs.slim_tag }} + image-ref: ${{ steps.download_image.outputs.docker_image }} format: "template" template: "@/contrib/sarif.tpl" output: "trivy-results.sarif" @@ -1172,8 +316,9 @@ jobs: sarif_file: "trivy-results.sarif" smoke_test_matrix: - runs-on: depot-ubuntu-24.04-small - needs: [setup, determine_runner] + runs-on: ${{ needs.setup.outputs.test_runner_type }} + needs: setup + if: ${{ (github.event_name == 'pull_request' || github.event_name == 'push') }} outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} cypress_batch_count: ${{ steps.set-batch-count.outputs.cypress_batch_count }} @@ -1185,7 +330,7 @@ jobs: # python_batch_count is used to split pytests in the smoke-test (batches of actual test functions) # cypress_batch_count is used to split the collection of cypress test specs into batches. run: | - if [[ "${{ needs.determine_runner.outputs.runner_type }}" == "ubuntu-latest" ]]; then + if [[ "${{ needs.setup.outputs.test_runner_type }}" == "ubuntu-latest" ]]; then echo "cypress_batch_count=5" >> "$GITHUB_OUTPUT" echo "python_batch_count=3" >> "$GITHUB_OUTPUT" else @@ -1221,31 +366,14 @@ jobs: fi echo "matrix={\"include\":[$includes] }" >> "$GITHUB_OUTPUT" - determine_runner: - name: Determine Runner Type - runs-on: depot-ubuntu-24.04-small - needs: setup - outputs: - runner_type: ${{ steps.set-runner.outputs.runner_type }} - steps: - - id: set-runner - run: | - if [[ "${{ env.DOCKER_CACHE }}" == "DEPOT" && "${{ env.DEPOT_PROJECT_ID }}" != "" ]]; then - echo "runner_type=depot-ubuntu-24.04-4" >> "$GITHUB_OUTPUT" - else - echo "runner_type=ubuntu-latest" >> "$GITHUB_OUTPUT" - fi - smoke_test: name: Run Smoke Tests (${{ matrix.test_strategy }}, Batch ${{ matrix.batch }}/${{ matrix.batch_count }}) - runs-on: ${{ needs.determine_runner.outputs.runner_type }} + runs-on: ${{ needs.setup.outputs.test_runner_type }} needs: [ setup, smoke_test_matrix, base_build, - determine_runner, - datahub_actions_slim_build, ] strategy: fail-fast: false @@ -1253,31 +381,34 @@ jobs: if: ${{ always() && !failure() && !cancelled() && needs.smoke_test_matrix.outputs.matrix != '[]' }} steps: - name: Free up disk space - if: ${{ needs.determine_runner.outputs.runner_type == 'ubuntu-latest' }} + if: ${{ !contains(needs.setup.outputs.test_runner_type, 'depot') }} run: | sudo apt-get remove 'dotnet-*' azure-cli || true sudo rm -rf /usr/local/lib/android/ || true sudo docker image prune -a -f || true - - uses: actions/cache@v4 + - uses: actions/cache/restore@v3 with: path: | ~/.cache/uv key: ${{ runner.os }}-uv-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-uv- - - uses: actions/cache@v4 + - uses: actions/cache/restore@v3 with: path: | ~/.npm ~/.cache/Cypress ~/.cache/yarn key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} - + restore-keys: | + ${{ runner.os }}-yarn- - name: Check out the repo uses: acryldata/sane-checkout-action@v3 - name: Set up Depot CLI - if: ${{ env.DOCKER_CACHE == 'DEPOT' && env.DEPOT_PROJECT_ID != '' }} + if: ${{ needs.setup.outputs.use_depot_cache == 'true' }} uses: depot/setup-action@v1 - uses: actions/setup-python@v5 @@ -1286,30 +417,33 @@ jobs: cache: "pip" - uses: gradle/actions/setup-gradle@v4 - if: ${{ env.DEPOT_PROJECT_ID == '' }} + if: ${{ needs.setup.outputs.use_depot_cache != 'true' }} + + - name: Login to DockerHub + uses: docker/login-action@v3 + if: ${{ needs.setup.outputs.docker-login == 'true' }} + with: + username: ${{ secrets.ACRYL_DOCKER_USERNAME }} + password: ${{ secrets.ACRYL_DOCKER_PASSWORD }} - name: build images - if: ${{ env.DEPOT_PROJECT_ID == '' }} + if: ${{ needs.setup.outputs.use_depot_cache != 'true' }} run: | - docker pull confluentinc/cp-kafka:7.4.0 & - docker pull mysql:8.2 & - docker pull opensearchproject/opensearch:2.9.0 & - docker pull ${{ env.DATAHUB_INGESTION_BASE_IMAGE }}:head-slim & - ./gradlew :docker:buildImagesQuickstartDebugConsumers -PreleaseVersion=pr-build & - - wait + ./gradlew :docker:buildImagesQuickstartDebugConsumers -Ptag=${{ needs.setup.outputs.tag }} -PpythonDockerVersion=${{ needs.setup.outputs.python_release_version }} -PdockerRegistry=${{ env.DOCKER_REGISTRY }} docker images env: DOCKER_CACHE: GITHUB + - name: pull images from depot - if: ${{ env.DOCKER_CACHE == 'DEPOT' && env.DEPOT_PROJECT_ID != '' }} + if: ${{ needs.setup.outputs.use_depot_cache == 'true' }} run: | depot pull --project ${{ env.DEPOT_PROJECT_ID }} ${{ needs.base_build.outputs.build_id }} docker images + - name: run quickstart env: DATAHUB_TELEMETRY_ENABLED: false - DATAHUB_VERSION: pr-build + DATAHUB_VERSION: ${{ needs.setup.outputs.tag }} DATAHUB_ACTIONS_IMAGE: ${{ env.DATAHUB_ACTIONS_IMAGE }} ACTIONS_EXTRA_PACKAGES: "acryl-datahub-actions[executor] acryl-datahub-actions" ACTIONS_CONFIG: "https://raw.githubusercontent.com/acryldata/datahub-actions/main/docker/config/executor.yaml" @@ -1342,7 +476,7 @@ jobs: - name: Smoke test env: RUN_QUICKSTART: false - DATAHUB_VERSION: ${{ needs.setup.outputs.unique_tag }} + DATAHUB_VERSION: ${{ needs.setup.outputs.tag }} CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }} CLEANUP_DATA: "false" TEST_STRATEGY: ${{ matrix.test_strategy }} @@ -1390,6 +524,21 @@ jobs: with: token: ${{ secrets.CODECOV_TOKEN }} + - uses: actions/cache/save@v4 + if: ${{ github.ref == 'refs/heads/master' }} + with: + path: | + ~/.cache/uv + key: ${{ runner.os }}-uv-${{ hashFiles('**/requirements.txt') }} + + - uses: actions/cache/save@v4 + if: ${{ github.ref == 'refs/heads/master' }} + with: + path: | + ~/.cache/yarn + key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} + + deploy_datahub_head: name: Deploy to Datahub HEAD runs-on: ubuntu-latest diff --git a/build.gradle b/build.gradle index 5e9d65920d..4226cc4c6f 100644 --- a/build.gradle +++ b/build.gradle @@ -65,7 +65,7 @@ buildscript { ext.logbackClassicJava8 = '1.2.12' ext.awsSdk2Version = '2.30.33' - ext.docker_registry = 'acryldata' + ext.docker_registry = project.getProperties().getOrDefault("dockerRegistry", 'acryldata') apply from: './repositories.gradle' buildscript.repositories.addAll(project.repositories) diff --git a/datahub-actions/build.gradle b/datahub-actions/build.gradle index d1edaf526a..f3fbad1fa6 100644 --- a/datahub-actions/build.gradle +++ b/datahub-actions/build.gradle @@ -28,7 +28,7 @@ ext { docker_registry = 'acryldata' docker_repo = 'datahub-actions' docker_target = project.getProperties().getOrDefault("dockerTarget", "slim") - docker_version = "v${version}${docker_target == 'slim' ? '-slim' : ''}" + python_docker_version = project.getProperties().getOrDefault("pythonDockerVersion", "1!0.0.0+docker.${version}") } @@ -143,7 +143,7 @@ task cleanPythonCache(type: Exec) { docker { //dependsOn(build) - name "${docker_registry}/${docker_repo}:${versionTag}${docker_target == 'slim' ? '-slim' : ''}" + name "${docker_registry}/${docker_repo}:${versionTag}" dockerfile file("${rootProject.projectDir}/docker/datahub-actions/Dockerfile") files fileTree(rootProject.projectDir) { exclude "datahub-actions/scripts/**" @@ -159,10 +159,15 @@ docker { }.exclude { i -> (!i.file.name.endsWith(".dockerignore") && i.file.isHidden()) } - buildArgs([APP_ENV: docker_target, RELEASE_VERSION: python_docker_version]) - + additionalTag("Debug", "${docker_registry}/${docker_repo}:debug") + defaultVariant = "slim" + variants = [ + "slim": [suffix: "-slim", args: [APP_ENV: "slim", RELEASE_VERSION: python_docker_version]], + "full": [suffix: "", args: [APP_ENV: "full", RELEASE_VERSION: python_docker_version]] + ] + } build.dependsOn install diff --git a/docker/build.gradle b/docker/build.gradle index e9609528ef..0d5cb9f174 100644 --- a/docker/build.gradle +++ b/docker/build.gradle @@ -83,6 +83,16 @@ ext { 'quickstartStorage': [ profile: 'quickstart-storage', preserveVolumes: true + ], + 'allImages': [ //This is a special task just to include all images as dependencies - and is useful when CI needs to publish all images + profile: 'quickstart-consumers', + modules: python_services_modules + backend_profile_modules + [ + ':datahub-frontend', + ':metadata-jobs:mce-consumer-job', + ':metadata-jobs:mae-consumer-job', + ':datahub-actions', + ':docker:datahub-ingestion' + ] ] ] @@ -194,6 +204,9 @@ quickstart_configs.each { taskName, config -> buildCmd << "depot" buildCmd += bakeCmdArgs buildCmd += ['--save', '--metadata-file', "${rootProject.buildDir}/build-metadata.json"] + if (project.properties.getOrDefault("dockerPush", false)){ + buildCmd << "--push" + } } else { buildCmd += ["docker", "buildx" ] buildCmd +=bakeCmdArgs @@ -221,64 +234,6 @@ quickstart_configs.each { taskName, config -> } } -//TODO This is a copy of buildImages* tasks but without the prepareAll dependency. Need to refactor to avoid code duplication. -/* -quickstart_configs.each { taskName, config -> - tasks.register("buildImagesFromCache${taskName}", Exec) { - ext{ - bakeSpec = [:] - } - group = 'quickstart-ci' - dependsOn(config.modules.collect { it + ':generateBakeSnippet' }) - - def jsonFile = new File(rootProject.buildDir, "bake-spec-${taskName}.json") - - bakeCmdArgs = ["bake", "-f", "${jsonFile.absolutePath}"] - def buildCmd = [] - if (System.getenv("DOCKER_CACHE") == "GITHUB") { - buildCmd += ["docker", "buildx" ] - buildCmd += bakeCmdArgs - def githubToken = System.getenv("GITHUB_TOKEN") - if (githubToken) { - dockerCmd += ["--cache-from", "type=gha,token=${githubToken}"] - dockerCmd += ["--cache-to", "type=gha,mode=max,token=${githubToken}"] - } else { - dockerCmd += ["--cache-from", "type=gha"] - dockerCmd += ["--cache-to", "type=gha,mode=max"] - } - } else if (System.getenv("DOCKER_CACHE") == "DEPOT") { - buildCmd << "depot" - buildCmd += bakeCmdArgs - buildCmd += ['--save', '--metadata-file', "${rootProject.buildDir}/build-metadata.json"] - } else { - buildCmd += ["docker", "buildx" ] - buildCmd +=bakeCmdArgs - } - println(buildCmd.join(" ")) - commandLine buildCmd - workingDir rootProject.projectDir - - doFirst { - def bakeSnippets = [:] - def targets = [] - - config.modules.each { module -> - def moduleProject = project.project(module) - def generateBakeSnippetsTask = moduleProject.tasks.getByName("generateBakeSnippet") - bakeSnippets.putAll(generateBakeSnippetsTask.bakeSpec.target) - targets.addAll(generateBakeSnippetsTask.bakeSpec.target.keySet()) - } - - ext.bakeSpec.group = [ "default": ["targets": targets] ] - ext.bakeSpec.target = bakeSnippets - - jsonFile.parentFile.mkdirs() - jsonFile.text = groovy.json.JsonOutput.prettyPrint(groovy.json.JsonOutput.toJson(ext.bakeSpec)) - } - } -} -*/ - // Configure dependencies for ComposeUp tasks quickstart_configs.each { taskName, config -> if (config.modules) { diff --git a/docker/datahub-gms/Dockerfile b/docker/datahub-gms/Dockerfile index d607c13699..5ead5b209b 100644 --- a/docker/datahub-gms/Dockerfile +++ b/docker/datahub-gms/Dockerfile @@ -63,7 +63,7 @@ FROM base AS dev-install FROM ${APP_ENV}-install AS final -RUN mkdir -p /etc/datahub/plugins/auth/resources +RUN mkdir -p /etc/datahub/plugins/auth/resources /datahub RUN addgroup -S datahub && adduser -S datahub -G datahub && chmod g-s /home/datahub RUN chown -R datahub:datahub /etc/datahub /datahub diff --git a/docker/datahub-ingestion/build.gradle b/docker/datahub-ingestion/build.gradle index e6854b4d7a..3d07067e39 100644 --- a/docker/datahub-ingestion/build.gradle +++ b/docker/datahub-ingestion/build.gradle @@ -10,7 +10,7 @@ ext { docker_repo = 'datahub-ingestion' docker_dir = 'datahub-ingestion' docker_target = project.getProperties().getOrDefault("dockerTarget", "slim") - docker_version = "v${version}${docker_target == 'slim' ? '-slim' : ''}" + docker_version = "${versionTag}" // Variants will add suffix to this tag python_docker_version = project.getProperties().getOrDefault("pythonDockerVersion", "1!0.0.0+docker.${version}") revision = 12 // increment to trigger rebuild @@ -36,9 +36,16 @@ docker { } version "${docker_version}" + + defaultVariant = "slim" + + variants = [ + "slim": [suffix: "", args: [RELEASE_VERSION: python_docker_version, APP_ENV: "slim"]], + "full": [suffix: "full", args: [RELEASE_VERSION: python_docker_version, APP_ENV: "full"]] + ] // This task is intended to build the slim image - target 'ingestion-base-slim' - def dockerBuildArgs = [RELEASE_VERSION: python_docker_version, APP_ENV: docker_target] + //target 'ingestion-base-slim' //Review if this needs to be handled by bake + def dockerBuildArgs = [RELEASE_VERSION: python_docker_version] // Add build args if they are defined (needed for some CI or enterprise environments) if (project.hasProperty('pipMirrorUrl')) { diff --git a/gradle/docker/docker.gradle b/gradle/docker/docker.gradle index 6cc9fd72c7..9feee1d83d 100644 --- a/gradle/docker/docker.gradle +++ b/gradle/docker/docker.gradle @@ -53,6 +53,15 @@ class DockerPluginExtension { ListProperty platforms ListProperty dependencies // May contain tasks or task names Property target + MapProperty> variants + + Property defaultVariant + List defaultPlatforms = ["linux/amd64", "linux/arm64/v8"] + + // For quickStart debug builds that use APP_ENV=dev like pattern. Not used in matrix builds. + MapProperty debugBuildArgs + + DockerPluginExtension(Project project) { this.project = project @@ -63,6 +72,13 @@ class DockerPluginExtension { platforms = project.objects.listProperty(String) dependencies = project.objects.listProperty(Object) target = project.objects.property(String) + variants = project.objects.mapProperty(String, Map) + // quickstart builds does not require all variants to be built. So, if default variant is specified, and + // projectProperty matrixBuild is not set to true, then only default variant is built to save time. + // The defaultVariant does not use a suffix in the tag. + defaultVariant = project.objects.property(String) + + debugBuildArgs = project.objects.mapProperty(String, String) } def files(Object... files) { @@ -81,6 +97,10 @@ class DockerPluginExtension { buildArgs.putAll(values) } + def debugBuildArgs(Map values) { + debugBuildArgs.putAll(values) + } + def platform(String... platforms) { this.platforms.addAll(platforms) } @@ -229,21 +249,73 @@ project.afterEvaluate { description "Generates bake snippets for the project" doLast { + // if matrixBuild is true, this is to publish images, so all variants must be built. + def matrixBuild = project.getProperties().getOrDefault("matrixBuild", false) + def bake_spec_target = [ context: "${buildContext}", dockerfile: "${extension.dockerfile.get().toPath()}", - tags: extension.tags.get().values() ] + + if (project.hasProperty("tag")) { + def dockerTag = project.property("tag") + bake_spec_target.tags = extension.tags.get().values().findAll({ tag -> tag.contains(dockerTag) }).toList() + } else { + bake_spec_target.tags = extension.tags.get().values().toList() + } if (extension.buildArgs.get()) { bake_spec_target.args = extension.buildArgs.get() - } - if (extension.platforms.get()) { - bake_spec_target.platforms = extension.platforms.get() + } else { + bake_spec_target.args = [:] } + if (extension.debugBuildArgs.get() && bake_spec_target.tags.findAll { tag -> tag.contains("debug") }.size() > 0) { + bake_spec_target.args.putAll(extension.debugBuildArgs.get()) + } + + if (matrixBuild) { + if (extension.platforms.get()) { + bake_spec_target.platforms = extension.platforms.get() + } else { + bake_spec_target.platforms = extension.defaultPlatforms + } + } + + if (extension.variants.get()) { + bake_spec_target.name = "${project.name}-\${variants.tagSuffix}" + bake_spec_target.matrix = [variants: []] + + extension.variants.get().each { variant, variantSpec -> + if (matrixBuild || variant == extension.defaultVariant.get()) { + // Its not easy to merge common buildArgs and variant buildArgs with json format. + // So, we just add common buildArgs to matrix. + bake_spec_target.matrix.variants.add([ 'tagSuffix': "${variantSpec.suffix}", 'args': variantSpec.args + bake_spec_target.args]) + } + } + // When variants are present, fix up tags to include the variant suffix + // A bit of a hack that for quickstart builds, we need a tag named debug that contains the default variant without suffixes + bake_spec_target.tags = bake_spec_target.tags.collect( + { tag -> tag + (tag.contains("debug") ? "" : "\${variants.tagSuffix}") } + ).toList() + if (project.hasProperty("shaTag")) { + def shaTag = project.property("shaTag") + def dockerRepo = extension.tags.get().get("").split(":")[0] //Extract the repo name from the default tag + bake_spec_target.tags.add("${dockerRepo}:${shaTag}\${variants.tagSuffix}") + } + + bake_spec_target.args = "\${variants.args}" + + } else { + if (project.hasProperty("shaTag")) { + def shaTag = project.property("shaTag") + def dockerRepo = extension.tags.get().get("").split(":")[0] + bake_spec_target.tags.add("${dockerRepo}:${shaTag}") + } + } ext.bakeSpec = [ target: [ "${project.name}": bake_spec_target] ] + } } diff --git a/gradle/versioning/versioning-global.gradle b/gradle/versioning/versioning-global.gradle index e7b92cf654..4c89d3c95a 100644 --- a/gradle/versioning/versioning-global.gradle +++ b/gradle/versioning/versioning-global.gradle @@ -23,14 +23,15 @@ def detailedVersionString = "0.0.0-unknown-SNAPSHOT" def cliMajorVersion = "0.15.0" // base default cli major version def snapshotVersion = false def javaVersion = "" -// tag for docker images. the prefix v is used in tag only if the tag is computed from a version. If a releaseVersion is supplied, use it as is. -// This enables pr tags to be used without the v prefix. This variance was previouslyhandled in the CI steps build images without using gradle. + +// Used to tag docker images. If the project property tag is set, use it as is (posssibly add suffixes like slim, pythonVersion if applicable). +// Otherwise, compute the tag from the version. def versionTag = "" + if (project.hasProperty("releaseVersion")) { version = releaseVersion detailedVersionString = releaseVersion - versionTag = releaseVersion } else { try { // apply this plugin in a try-catch block so that we can handle cases without .git directory @@ -39,7 +40,6 @@ if (project.hasProperty("releaseVersion")) { detailedVersionString = gitVersion() version = details.lastTag version = version.startsWith("v")? version.substring(1): version - versionTag = "v" + version def suffix = details.isCleanTag? "": "-SNAPSHOT" snapshotVersion = ! details.isCleanTag } @@ -82,10 +82,15 @@ if (project.hasProperty("releaseVersion")) { // we are unable to part the last token as an integer, so we just append SNAPSHOT to this version javaVersion = versionParts[0..versionParts.size()-1].join('.') + '-SNAPSHOT' } - versionTag = "v" + version } } +if (project.hasProperty("tag")) { + versionTag = tag +} else { + versionTag = "v" + version +} + // Note: No task, we want this executed during config phase, once for rootProject. def data = [ fullVersion: detailedVersionString, @@ -105,4 +110,9 @@ rootProject.buildDir.mkdirs() // Write to file outputFile.text = jsonBuilder.toPrettyString() -println "git.properties JSON data written to ${outputFile}" +task printVersionDetails() { + println("fullVersion=" + detailedVersionString) + println("cliMajorVersion=" + cliMajorVersion) + println("version=" + version) + println("versionTag=" + versionTag) +} diff --git a/metadata-service/war/build.gradle b/metadata-service/war/build.gradle index ff91dd2036..20d6c50fff 100644 --- a/metadata-service/war/build.gradle +++ b/metadata-service/war/build.gradle @@ -120,6 +120,8 @@ docker { if (dockerBuildArgs.size() > 0) { buildArgs(dockerBuildArgs) } + + debugBuildArgs = [APP_ENV: 'dev'] } test {