yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-09-09 17:20:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ci): publish head images only after passing tests (#14600)

Browse Source
This commit is contained in:
Chakru 2025-09-01 18:52:29 +05:30 committed by GitHub
parent 6044d8d298
commit 8a839b8077
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 67 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

74
.github/workflows/docker-unified.yml vendored
View File

@ -1,8 +1,6 @@
name: Docker Build, Scan, Test
on:
workflow_dispatch:
schedule:
- cron: "0 0 * * *" # Run at midnight UTC every day
push:
branches:
- master
@ -107,7 +105,7 @@ jobs:
env:
ENABLE_PUBLISH: >-
${{
(github.event_name == 'release' || ((github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' || github.event_name == 'push') && github.ref == 'refs/heads/master'))
(github.event_name == 'release' || ((github.event_name == 'workflow_dispatch' || github.event_name == 'push') && github.ref == 'refs/heads/master'))
&& ( secrets.ACRYL_DOCKER_PASSWORD != '' )
}}
run: |
@ -265,8 +263,9 @@ jobs:
- name: Build all Images (Publish)
if: ${{ needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }}
# since this is for publishing, we will build all images, not just those for smoke tests. But will publish only if tests pass.
run: |
./gradlew :docker:buildImagesAll -PmatrixBuild=true -Ptag=${{ needs.setup.outputs.tag }} -PshaTag=${{ needs.setup.outputs.short_sha }} -PpythonDockerVersion=${{ needs.setup.outputs.python_release_version }} -PdockerRegistry=${{ env.DOCKER_REGISTRY }} -PdockerPush=true
./gradlew :docker:buildImagesAll -PmatrixBuild=true -Ptag=${{ needs.setup.outputs.tag }} -PshaTag=${{ needs.setup.outputs.short_sha }} -PpythonDockerVersion=${{ needs.setup.outputs.python_release_version }} -PdockerRegistry=${{ env.DOCKER_REGISTRY }}
- name: Capture build Id
id: capture-build-id
@ -275,7 +274,16 @@ jobs:
DEPOT_BUILD_ID=$(jq -r '.["depot.build"]?.buildID' ${{ github.workspace }}/build/build-metadata.json)
echo "build_id=${DEPOT_BUILD_ID}" >> "$GITHUB_OUTPUT"
echo "matrix=$(jq -c '{"target":.["depot.build"].targets}' ${{ github.workspace }}/build/build-metadata.json)" >> $GITHUB_OUTPUT
echo "matrix=$(depot bake -f ${{ github.workspace }}/build/bake-spec-allImages.json --print | jq -c '{"target":.target | keys}')" >> $GITHUB_OUTPUT
- name: Save build Metadata
if: ${{ needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }}
uses: actions/upload-artifact@v4
with:
name: build-metadata-${{ needs.setup.outputs.tag }}
path: |
${{ github.workspace }}/build/build-metadata.json
${{ github.workspace }}/build/bake-spec-allImages.json
- uses: actions/cache/save@v4
if: ${{ github.ref == 'refs/heads/master' }}
@ -299,7 +307,7 @@ jobs:
name: Scan images for vulnerabilities
runs-on: depot-ubuntu-24.04
needs: [setup, base_build]
if: ${{ needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }}
if: ${{ needs.setup.outputs.publish == 'true' }}
strategy:
fail-fast: false
matrix: ${{ fromJson(needs.base_build.outputs.matrix) }}
@ -557,10 +565,62 @@ jobs:
~/.cache/yarn
key: ${{ needs.setup.outputs.yarn_cache_key }}
publish_images:
name: Push images after tests pass
runs-on: ${{ needs.setup.outputs.test_runner_type_small }}
needs: [setup, smoke_test, base_build]
if: ${{ always() && !failure() && !cancelled() }}
steps:
- name: Check if tests have passed
id: tests_passed
run: |
# Check the overall result of the matrix job
# Matrix jobs can have mixed results, so we check for any failures
if [[ "${{ needs.smoke_test.result }}" == "failure" ]]; then
echo "Smoke tests failed, skipping image pushing"
echo "tests_passed=false" >> "$GITHUB_OUTPUT"
exit 1
elif [[ "${{ needs.smoke_test.result }}" == "cancelled" ]]; then
echo "Smoke tests were cancelled, skipping image pushing"
echo "tests_passed=false" >> "$GITHUB_OUTPUT"
exit 1
else
echo "Smoke tests completed successfully, proceeding with image pushing"
echo "tests_passed=true" >> "$GITHUB_OUTPUT"
fi
- name: Set up Depot CLI
if: ${{ steps.tests_passed.outputs.tests_passed == 'true' && needs.setup.outputs.use_depot_cache == 'true' }}
uses: depot/setup-action@v1
- name: Login to DockerHub
uses: docker/login-action@v3
if: ${{ steps.tests_passed.outputs.tests_passed == 'true' && needs.setup.outputs.docker-login == 'true' }}
with:
username: ${{ secrets.ACRYL_DOCKER_USERNAME }}
password: ${{ secrets.ACRYL_DOCKER_PASSWORD }}
- name: Download build Metadata
if: ${{ needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }}
uses: actions/download-artifact@v4
with:
name: build-metadata-${{ needs.setup.outputs.tag }}
path: ${{ github.workspace }}/build
- name: Push images from depot builder
if: ${{ steps.tests_passed.outputs.tests_passed == 'true' && needs.setup.outputs.use_depot_cache == 'true' && (needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' ) }}
run: |
depot bake -f ${{ github.workspace }}/build/bake-spec-allImages.json --print | jq -c '.target | to_entries | map({target: .key, tags: .value.tags[]})'| jq -c '.[]' | while IFS= read -r line; do
TARGET=$(echo "$line" | jq -r '.target')
TAG=$(echo "$line" | jq -r '.tags')
depot push --project ${{ env.DEPOT_PROJECT_ID }} ${{ needs.base_build.outputs.build_id }} --target $TARGET --tag $TAG
done
deploy_datahub_head:
name: Deploy to Datahub HEAD
runs-on: ubuntu-latest
needs: [setup, smoke_test_lint, smoke_test]
needs: [setup, smoke_test_lint, smoke_test, publish_images]
steps:
- uses: aws-actions/configure-aws-credentials@v4
if: ${{ needs.setup.outputs.publish != 'false' && github.repository_owner == 'datahub-project' && needs.setup.outputs.repository_name == 'datahub' }}