feat(build): conditional ci (#9673)

.github/actions/ci-optimization/action.yml

@@ -0,0 +1,79 @@
+name: 'Identify CI Optimizations'
+description: 'Determine if code changes are specific to certain modules.'
+
+outputs:
+  frontend-only:
+    description: "Frontend only change"
+    value: ${{ steps.filter.outputs.frontend == 'true' && steps.filter.outputs.ingestion == 'false' && steps.filter.outputs.backend == 'false' }}
+  ingestion-only:
+    description: "Ingestion only change"
+    value: ${{ steps.filter.outputs.frontend == 'false' && steps.filter.outputs.ingestion == 'true' && steps.filter.outputs.backend == 'false' }}
+  backend-only:
+    description: "Backend only change"
+    value: ${{ steps.filter.outputs.frontend == 'false' && steps.filter.outputs.ingestion == 'false' && steps.filter.outputs.backend == 'true' }}
+  backend-change:
+    description: "Backend code has changed"
+    value: ${{ steps.filter.outputs.backend == 'true' }}
+  ingestion-change:
+    description: "Ingestion code has changed"
+    value: ${{ steps.filter.outputs.ingestion == 'true' }}
+  frontend-change:
+    description: "Frontend code has changed"
+    value: ${{ steps.filter.outputs.frontend == 'true' }}
+  docker-change:
+    description: "Docker code has changed"
+    value: ${{ steps.filter.outputs.docker == 'true' }}
+  kafka-setup-change:
+    description: "Kafka setup docker change"
+    value: ${{ steps.filter.outputs.kafka-setup == 'true' }}
+  mysql-setup-change:
+    description: "Mysql setup docker change"
+    value: ${{ steps.filter.outputs.mysql-setup == 'true' }}
+  postgres-setup-change:
+    description: "Postgres setup docker change"
+    value: ${{ steps.filter.outputs.postgres-setup == 'true' }}
+  elasticsearch-setup-change:
+    description: "Elasticsearch setup docker change"
+    value: ${{ steps.filter.outputs.elasticsearch-setup == 'true' }}
+runs:
+  using: "composite"
+  steps:
+    - uses: dorny/paths-filter@v2
+      id: filter
+      with:
+        filters: |
+          frontend:
+            - "datahub-frontend/**"
+            - "datahub-web-react/**"
+            - "smoke-test/tests/cypress/**"
+            - "docker/datahub-frontend/**"
+          ingestion:
+            - "metadata-ingestion-modules/airflow-plugin/**"
+            - "metadata-ingestion/**"
+            - "metadata-models/**"
+            - "smoke-test/**"
+            - "docker/datahub-ingestion-**"
+          docker:
+            - "docker/**"
+          backend:
+            - "metadata-models/**"
+            - "datahub-upgrade/**"
+            - "entity-registry/**"
+            - "li-utils/**"
+            - "metadata-auth/**"
+            - "metadata-dao-impl/**"
+            - "metadata-events/**"
+            - "metadata-io/**"
+            - "metadata-jobs/**"
+            - "metadata-service/**"
+            - "metadata-utils/**"
+            - "smoke-test/**"
+            - "docker/**"
+          kafka-setup:
+            - "docker/kafka-setup/**"
+          mysql-setup:
+            - "docker/mysql-setup/**"
+          postgres-setup:
+            - "docker/postgres-setup/**"
+          elasticsearch-setup:
+            - "docker/elasticsearch-setup/**"

.github/workflows/build-and-test.yml

@@ -20,6 +20,25 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  setup:
+    runs-on: ubuntu-latest
+    outputs:
+      frontend_change: ${{ steps.ci-optimize.outputs.frontend-change == 'true' }}
+      ingestion_change: ${{ steps.ci-optimize.outputs.ingestion-change == 'true' }}
+      backend_change: ${{ steps.ci-optimize.outputs.backend-change == 'true' }}
+      docker_change: ${{ steps.ci-optimize.outputs.docker-change == 'true' }}
+      frontend_only: ${{ steps.ci-optimize.outputs.frontend-only == 'true' }}
+      ingestion_only: ${{ steps.ci-optimize.outputs.ingestion-only == 'true' }}
+      kafka_setup_change: ${{ steps.ci-optimize.outputs.kafka-setup-change == 'true' }}
+      mysql_setup_change: ${{ steps.ci-optimize.outputs.mysql-setup-change == 'true' }}
+      postgres_setup_change: ${{ steps.ci-optimize.outputs.postgres-setup-change == 'true' }}
+      elasticsearch_setup_change: ${{ steps.ci-optimize.outputs.elasticsearch-setup-change == 'true' }}
+    steps:
+      - name: Check out the repo
+        uses: hsheth2/sane-checkout-action@v1
+      - uses: ./.github/actions/ci-optimization
+        id: ci-optimize
+
   build:
     strategy:
       fail-fast: false
@@ -36,11 +55,13 @@ jobs:
             timezone: "America/New_York"
     runs-on: ubuntu-latest
     timeout-minutes: 60
+    needs: setup
     steps:
       - uses: szenius/set-timezone@v1.2
         with:
           timezoneLinux: ${{ matrix.timezone }}
-      - uses: hsheth2/sane-checkout-action@v1
+      - name: Check out the repo
+        uses: hsheth2/sane-checkout-action@v1
       - name: Set up JDK 17
         uses: actions/setup-java@v3
         with:
@@ -51,12 +72,12 @@ jobs:
         with:
           python-version: "3.10"
           cache: pip
-      - name: Gradle build (and test) for metadata ingestion
+      - name: Gradle build (and test) for NOT metadata ingestion
-        if: ${{  matrix.command == 'except_metadata_ingestion' }}
+        if: ${{  matrix.command == 'except_metadata_ingestion' && needs.setup.outputs.backend_change == 'true' }}
         run: |
           ./gradlew build -x :metadata-ingestion:build -x :metadata-ingestion:check -x docs-website:build -x :metadata-integration:java:spark-lineage:test -x :metadata-io:test -x :metadata-ingestion-modules:airflow-plugin:build -x :metadata-ingestion-modules:airflow-plugin:check -x :datahub-frontend:build -x :datahub-web-react:build --parallel
       - name: Gradle build (and test) for frontend
-        if: ${{  matrix.command == 'frontend' }}
+        if: ${{  matrix.command == 'frontend' && needs.setup.outputs.frontend_change == 'true' }}
         run: |
           ./gradlew :datahub-frontend:build :datahub-web-react:build --parallel
         env:
@@ -75,8 +96,11 @@ jobs:
 
   quickstart-compose-validation:
     runs-on: ubuntu-latest
+    needs: setup
+    if: ${{ needs.setup.outputs.docker_change == 'true' }}
     steps:
-      - uses: actions/checkout@v3
+      - name: Check out the repo
+        uses: hsheth2/sane-checkout-action@v1
       - uses: actions/setup-python@v4
         with:
           python-version: "3.10"

.github/workflows/docker-unified.yml

@@ -51,9 +51,19 @@ jobs:
       short_sha: ${{ steps.tag.outputs.short_sha }}
       branch_name: ${{ steps.tag.outputs.branch_name }}
       repository_name: ${{ steps.tag.outputs.repository_name }}
+      frontend_change: ${{ steps.ci-optimize.outputs.frontend-change == 'true' }}
+      ingestion_change: ${{ steps.ci-optimize.outputs.ingestion-change == 'true' }}
+      backend_change: ${{ steps.ci-optimize.outputs.backend-change == 'true' }}
+      frontend_only: ${{ steps.ci-optimize.outputs.frontend-only == 'true' }}
+      ingestion_only: ${{ steps.ci-optimize.outputs.ingestion-only == 'true' }}
+      backend_only: ${{ steps.ci-optimize.outputs.backend-only == 'true' }}
+      kafka_setup_change: ${{ steps.ci-optimize.outputs.kafka-setup-change == 'true' }}
+      mysql_setup_change: ${{ steps.ci-optimize.outputs.mysql-setup-change == 'true' }}
+      postgres_setup_change: ${{ steps.ci-optimize.outputs.postgres-setup-change == 'true' }}
+      elasticsearch_setup_change: ${{ steps.ci-optimize.outputs.elasticsearch-setup-change == 'true' }}
     steps:
-      - name: Checkout
+      - name: Check out the repo
-        uses: actions/checkout@v3
+        uses: hsheth2/sane-checkout-action@v1
       - name: Compute Tag
         id: tag
         run: |
@@ -75,11 +85,14 @@ jobs:
         run: |
           echo "Enable publish: ${{ env.ENABLE_PUBLISH }}"
           echo "publish=${{ env.ENABLE_PUBLISH }}" >> $GITHUB_OUTPUT
+      - uses: ./.github/actions/ci-optimization
+        id: ci-optimize
 
   gms_build:
     name: Build and Push DataHub GMS Docker Image
     runs-on: ubuntu-latest
     needs: setup
+    if: ${{ needs.setup.outputs.backend_change == 'true' || needs.setup.outputs.publish == 'true' }}
     steps:
       - name: Set up JDK 17
         uses: actions/setup-java@v3
@@ -113,6 +126,7 @@ jobs:
     name: "[Monitoring] Scan GMS images for vulnerabilities"
     runs-on: ubuntu-latest
     needs: [setup, gms_build]
+    if: ${{ needs.setup.outputs.backend_change == 'true' || needs.setup.outputs.publish == 'true' }}
     steps:
       - name: Checkout # adding checkout step just to make trivy upload happy
         uses: actions/checkout@v3
@@ -142,6 +156,7 @@ jobs:
     name: Build and Push DataHub MAE Consumer Docker Image
     runs-on: ubuntu-latest
     needs: setup
+    if: ${{ needs.setup.outputs.backend_change == 'true' || needs.setup.outputs.publish == 'true' }}
     steps:
       - name: Set up JDK 17
         uses: actions/setup-java@v3
@@ -171,6 +186,7 @@ jobs:
     name: "[Monitoring] Scan MAE consumer images for vulnerabilities"
     runs-on: ubuntu-latest
     needs: [setup, mae_consumer_build]
+    if: ${{ needs.setup.outputs.backend_change == 'true' || needs.setup.outputs.publish == 'true' }}
     permissions:
       contents: read # for actions/checkout to fetch code
       security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
@@ -204,6 +220,7 @@ jobs:
     name: Build and Push DataHub MCE Consumer Docker Image
     runs-on: ubuntu-latest
     needs: setup
+    if: ${{ needs.setup.outputs.backend_change == 'true' || needs.setup.outputs.publish == 'true' }}
     steps:
       - name: Set up JDK 17
         uses: actions/setup-java@v3
@@ -233,6 +250,7 @@ jobs:
     name: "[Monitoring] Scan MCE consumer images for vulnerabilities"
     runs-on: ubuntu-latest
     needs: [setup, mce_consumer_build]
+    if: ${{ needs.setup.outputs.backend_change == 'true' || needs.setup.outputs.publish == 'true' }}
     permissions:
       contents: read # for actions/checkout to fetch code
       security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
@@ -266,6 +284,7 @@ jobs:
     name: Build and Push DataHub Upgrade Docker Image
     runs-on: ubuntu-latest
     needs: setup
+    if: ${{ needs.setup.outputs.backend_change == 'true' || needs.setup.outputs.publish == 'true' }}
     steps:
       - name: Set up JDK 17
         uses: actions/setup-java@v3
@@ -295,6 +314,7 @@ jobs:
     name: "[Monitoring] Scan DataHub Upgrade images for vulnerabilities"
     runs-on: ubuntu-latest
     needs: [setup, datahub_upgrade_build]
+    if: ${{ needs.setup.outputs.backend_change == 'true' || needs.setup.outputs.publish == 'true' }}
     permissions:
       contents: read # for actions/checkout to fetch code
       security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
@@ -328,6 +348,7 @@ jobs:
     name: Build and Push DataHub Frontend Docker Image
     runs-on: ubuntu-latest
     needs: setup
+    if: ${{ needs.setup.outputs.frontend_change == 'true' || needs.setup.outputs.publish == 'true' }}
     steps:
       - name: Set up JDK 17
         uses: actions/setup-java@v3
@@ -359,6 +380,7 @@ jobs:
     name: "[Monitoring] Scan Frontend images for vulnerabilities"
     runs-on: ubuntu-latest
     needs: [setup, frontend_build]
+    if: ${{ needs.setup.outputs.frontend_change == 'true' || needs.setup.outputs.publish == 'true' }}
     permissions:
       contents: read # for actions/checkout to fetch code
       security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
@@ -392,6 +414,7 @@ jobs:
     name: Build and Push DataHub Kafka Setup Docker Image
     runs-on: ubuntu-latest
     needs: setup
+    if: ${{ needs.setup.outputs.kafka_setup_change == 'true' || needs.setup.outputs.publish == 'true' }}
     steps:
       - name: Check out the repo
         uses: hsheth2/sane-checkout-action@v1
@@ -412,6 +435,7 @@ jobs:
     name: Build and Push DataHub MySQL Setup Docker Image
     runs-on: ubuntu-latest
     needs: setup
+    if: ${{ needs.setup.outputs.mysql_setup_change == 'true' || needs.setup.outputs.publish == 'true' }}
     steps:
       - name: Check out the repo
         uses: hsheth2/sane-checkout-action@v1
@@ -432,6 +456,7 @@ jobs:
     name: Build and Push DataHub Elasticsearch Setup Docker Image
     runs-on: ubuntu-latest
     needs: setup
+    if: ${{ needs.setup.outputs.elasticsearch_setup_change == 'true' || needs.setup.outputs.publish == 'true' }}
     steps:
       - name: Check out the repo
         uses: hsheth2/sane-checkout-action@v1
@@ -454,6 +479,7 @@ jobs:
     outputs:
       tag: ${{ steps.tag.outputs.tag }}
     needs: setup
+    if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' }}
     steps:
       - name: Check out the repo
         uses: hsheth2/sane-checkout-action@v1
@@ -486,6 +512,7 @@ jobs:
     outputs:
       tag: ${{ steps.tag.outputs.tag }}
     needs: [setup, datahub_ingestion_base_build]
+    if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' }}
     steps:
       - name: Check out the repo
         uses: hsheth2/sane-checkout-action@v1
@@ -526,6 +553,7 @@ jobs:
     outputs:
       tag: ${{ steps.tag.outputs.tag }}
     needs: [setup, datahub_ingestion_base_build]
+    if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' }}
     steps:
       - name: Check out the repo
         uses: hsheth2/sane-checkout-action@v1
@@ -568,6 +596,7 @@ jobs:
       tag: ${{ steps.tag.outputs.tag }}
       needs_artifact_download: ${{ (steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true') && needs.setup.outputs.publish != 'true' }}
     needs: [setup, datahub_ingestion_base_slim_build]
+    if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' }}
     steps:
       - name: Set up JDK 17
         uses: actions/setup-java@v3
@@ -623,6 +652,7 @@ jobs:
     name: "[Monitoring] Scan Datahub Ingestion Slim images for vulnerabilities"
     runs-on: ubuntu-latest
     needs: [setup, datahub_ingestion_slim_build]
+    if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' }}
     steps:
       - name: Checkout # adding checkout step just to make trivy upload happy
         uses: actions/checkout@v3
@@ -655,6 +685,7 @@ jobs:
       tag: ${{ steps.tag.outputs.tag }}
       needs_artifact_download: ${{ (steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true') && needs.setup.outputs.publish != 'true' }}
     needs: [setup, datahub_ingestion_base_full_build]
+    if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' }}
     steps:
       - name: Set up JDK 17
         uses: actions/setup-java@v3
@@ -709,6 +740,7 @@ jobs:
     name: "[Monitoring] Scan Datahub Ingestion images for vulnerabilities"
     runs-on: ubuntu-latest
     needs: [setup, datahub_ingestion_full_build]
+    if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' }}
     steps:
       - name: Checkout # adding checkout step just to make trivy upload happy
         uses: actions/checkout@v3
@@ -734,22 +766,31 @@ jobs:
         with:
           sarif_file: "trivy-results.sarif"
 
+  smoke_test_matrix:
+    runs-on: ubuntu-latest
+    needs: setup
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    steps:
+      - id: set-matrix
+        run: |
+          if [ '${{ needs.setup.outputs.frontend_only }}' == 'true' ]; then
+             echo 'matrix=["cypress_suite1","cypress_rest"]' >> $GITHUB_OUTPUT
+          elif [ '${{ needs.setup.outputs.ingestion_only }}' == 'true' ]; then
+             echo 'matrix=["no_cypress_suite0","no_cypress_suite1"]' >> $GITHUB_OUTPUT
+          elif [ '${{ needs.setup.outputs.backend_change }}' == 'true' ]; then
+             echo 'matrix=["no_cypress_suite0","no_cypress_suite1","cypress_suite1","cypress_rest"]' >> $GITHUB_OUTPUT
+          else
+             echo 'matrix=[]' >> $GITHUB_OUTPUT
+          fi
+
   smoke_test:
     name: Run Smoke Tests
     runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        test_strategy:
-          [
-            "no_cypress_suite0",
-            "no_cypress_suite1",
-            "cypress_suite1",
-            "cypress_rest",
-          ]
     needs:
       [
         setup,
+        smoke_test_matrix,
         gms_build,
         frontend_build,
         kafka_setup_build,
@@ -760,6 +801,11 @@ jobs:
         datahub_upgrade_build,
         datahub_ingestion_slim_build,
       ]
+    strategy:
+      fail-fast: false
+      matrix:
+        test_strategy: ${{ fromJson(needs.smoke_test_matrix.outputs.matrix) }}
+    if: ${{ always() && !failure() && !cancelled() && needs.smoke_test_matrix.outputs.matrix != '[]' }}
     steps:
       - name: Disk Check
         run: df -h . && docker images
@@ -788,57 +834,99 @@ jobs:
         run: df -h . && docker images
       - name: Download GMS image
         uses: ishworkh/docker-image-artifact-download@v1
-        if: ${{ needs.setup.outputs.publish != 'true' }}
+        if: ${{ needs.setup.outputs.publish != 'true' && needs.gms_build.result == 'success' }}
         with:
           image: ${{ env.DATAHUB_GMS_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
       - name: Download Frontend image
         uses: ishworkh/docker-image-artifact-download@v1
-        if: ${{ needs.setup.outputs.publish != 'true' }}
+        if: ${{ needs.setup.outputs.publish != 'true' && needs.frontend_build.result == 'success' }}
         with:
           image: ${{ env.DATAHUB_FRONTEND_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
       - name: Download Kafka Setup image
         uses: ishworkh/docker-image-artifact-download@v1
-        if: ${{ needs.setup.outputs.publish != 'true' }}
+        if: ${{ needs.setup.outputs.publish != 'true' && needs.kafka_setup_build.result == 'success' }}
         with:
           image: ${{ env.DATAHUB_KAFKA_SETUP_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
       - name: Download Mysql Setup image
         uses: ishworkh/docker-image-artifact-download@v1
-        if: ${{ needs.setup.outputs.publish != 'true' }}
+        if: ${{ needs.setup.outputs.publish != 'true' && needs.mysql_setup_build.result == 'success' }}
         with:
           image: ${{ env.DATAHUB_MYSQL_SETUP_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
       - name: Download Elastic Setup image
         uses: ishworkh/docker-image-artifact-download@v1
-        if: ${{ needs.setup.outputs.publish != 'true' }}
+        if: ${{ needs.setup.outputs.publish != 'true' && needs.elasticsearch_setup_build.result == 'success' }}
         with:
           image: ${{ env.DATAHUB_ELASTIC_SETUP_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
       - name: Download MCE Consumer image
         uses: ishworkh/docker-image-artifact-download@v1
-        if: ${{ needs.setup.outputs.publish != 'true' }}
+        if: ${{ needs.setup.outputs.publish != 'true' && needs.mce_consumer_build.result == 'success' }}
         with:
           image: ${{ env.DATAHUB_MCE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
       - name: Download MAE Consumer image
         uses: ishworkh/docker-image-artifact-download@v1
-        if: ${{ needs.setup.outputs.publish != 'true' }}
+        if: ${{ needs.setup.outputs.publish != 'true' && needs.mae_consumer_build.result == 'success' }}
         with:
           image: ${{ env.DATAHUB_MAE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
       - name: Download upgrade image
         uses: ishworkh/docker-image-artifact-download@v1
-        if: ${{ needs.setup.outputs.publish != 'true' }}
+        if: ${{ needs.setup.outputs.publish != 'true' && needs.datahub_upgrade_build.result == 'success' }}
         with:
           image: ${{ env.DATAHUB_UPGRADE_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
       - name: Download datahub-ingestion-slim image
         uses: ishworkh/docker-image-artifact-download@v1
-        if: ${{ needs.datahub_ingestion_slim_build.outputs.needs_artifact_download == 'true' }}
+        if: ${{ needs.datahub_ingestion_slim_build.outputs.needs_artifact_download == 'true' && needs.datahub_ingestion_slim_build.result == 'success' }}
         with:
           image: ${{ env.DATAHUB_INGESTION_IMAGE }}:${{ needs.datahub_ingestion_slim_build.outputs.tag }}
       - name: Disk Check
         run: df -h . && docker images
+      - name: CI Optimization Head Images
+        # When publishing all tests/images are built (no optimizations)
+        if: ${{ needs.setup.outputs.publish != 'true' }}
+        run: |
+          if [ '${{ needs.setup.outputs.backend_change }}' == 'false' ]; then
+             echo 'GMS/Upgrade/MCE/MAE head images'
+             docker pull '${{ env.DATAHUB_GMS_IMAGE }}:head'
+             docker pull '${{ env.DATAHUB_MCE_CONSUMER_IMAGE }}:head'
+             docker pull '${{ env.DATAHUB_MAE_CONSUMER_IMAGE }}:head'
+             docker pull '${{ env.DATAHUB_UPGRADE_IMAGE }}:head'
+             docker tag '${{ env.DATAHUB_GMS_IMAGE }}:head' '${{ env.DATAHUB_GMS_IMAGE }}:${{ needs.setup.outputs.unique_tag }}'
+             docker tag '${{ env.DATAHUB_MCE_CONSUMER_IMAGE }}:head' '${{ env.DATAHUB_MCE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }}'
+             docker tag '${{ env.DATAHUB_MAE_CONSUMER_IMAGE }}:head' '${{ env.DATAHUB_MAE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }}'
+             docker tag '${{ env.DATAHUB_UPGRADE_IMAGE }}:head' '${{ env.DATAHUB_UPGRADE_IMAGE }}:${{ needs.setup.outputs.unique_tag }}'
+          fi
+          if [ '${{ needs.setup.outputs.frontend_change }}' == 'false' ]; then
+             echo 'Frontend head images'
+             docker pull '${{ env.DATAHUB_FRONTEND_IMAGE }}:head'
+             docker tag '${{ env.DATAHUB_FRONTEND_IMAGE }}:head' '${{ env.DATAHUB_FRONTEND_IMAGE }}:${{ needs.setup.outputs.unique_tag }}'
+          fi
+          if [ '${{ needs.setup.outputs.kafka_setup_change }}' == 'false' ]; then
+            echo 'kafka-setup head images'
+            docker pull '${{ env.DATAHUB_KAFKA_SETUP_IMAGE }}:head'
+            docker tag '${{ env.DATAHUB_KAFKA_SETUP_IMAGE }}:head' '${{ env.DATAHUB_KAFKA_SETUP_IMAGE }}:${{ needs.setup.outputs.unique_tag }}'
+          fi
+          if [ '${{ needs.setup.outputs.mysql_setup_change }}' == 'false' ]; then
+            echo 'mysql-setup head images'
+            docker pull '${{ env.DATAHUB_MYSQL_SETUP_IMAGE }}:head'
+            docker tag '${{ env.DATAHUB_MYSQL_SETUP_IMAGE }}:head' '${{ env.DATAHUB_MYSQL_SETUP_IMAGE }}:${{ needs.setup.outputs.unique_tag }}'
+          fi
+          if [ '${{ needs.setup.outputs.elasticsearch_setup_change }}' == 'false' ]; then
+            echo 'elasticsearch-setup head images'
+            docker pull '${{ env.DATAHUB_ELASTIC_SETUP_IMAGE }}:head'
+            docker tag '${{ env.DATAHUB_ELASTIC_SETUP_IMAGE }}:head' '${{ env.DATAHUB_ELASTIC_SETUP_IMAGE }}:${{ needs.setup.outputs.unique_tag }}'
+          fi
+          if [ '${{ needs.setup.outputs.ingestion_change }}' == 'false' ]; then
+            echo 'datahub-ingestion head-slim images'
+            docker pull '${{ env.DATAHUB_INGESTION_IMAGE }}:head-slim'
+            if [ '${{ needs.datahub_ingestion_slim_build.outputs.tag || 'head-slim' }}' != 'head-slim' ]; then
+               docker tag '${{ env.DATAHUB_INGESTION_IMAGE }}:head-slim' '${{ env.DATAHUB_INGESTION_IMAGE }}:${{ needs.datahub_ingestion_slim_build.outputs.tag }}'
+            fi
+          fi
       - name: run quickstart
         env:
           DATAHUB_TELEMETRY_ENABLED: false
           DATAHUB_VERSION: ${{ needs.setup.outputs.unique_tag }}
           DATAHUB_ACTIONS_IMAGE: ${{ env.DATAHUB_INGESTION_IMAGE }}
-          ACTIONS_VERSION: ${{ needs.datahub_ingestion_slim_build.outputs.tag }}
+          ACTIONS_VERSION: ${{ needs.datahub_ingestion_slim_build.outputs.tag || 'head-slim' }}
           ACTIONS_EXTRA_PACKAGES: "acryl-datahub-actions[executor]==0.0.13 acryl-datahub-actions==0.0.13 acryl-datahub==0.10.5"
           ACTIONS_CONFIG: "https://raw.githubusercontent.com/acryldata/datahub-actions/main/docker/config/executor.yaml"
         run: |