From 087d3fdb0dba1fccd802161e44a74a22edc211ac Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Fri, 19 Jan 2024 23:27:44 -0600 Subject: [PATCH] feat(build): conditional ci (#9673) --- .github/actions/ci-optimization/action.yml | 79 ++++++++++++ .github/workflows/build-and-test.yml | 34 +++++- .github/workflows/docker-unified.yml | 132 +++++++++++++++++---- 3 files changed, 218 insertions(+), 27 deletions(-) create mode 100644 .github/actions/ci-optimization/action.yml diff --git a/.github/actions/ci-optimization/action.yml b/.github/actions/ci-optimization/action.yml new file mode 100644 index 0000000000..404e0bab81 --- /dev/null +++ b/.github/actions/ci-optimization/action.yml @@ -0,0 +1,79 @@ +name: 'Identify CI Optimizations' +description: 'Determine if code changes are specific to certain modules.' + +outputs: + frontend-only: + description: "Frontend only change" + value: ${{ steps.filter.outputs.frontend == 'true' && steps.filter.outputs.ingestion == 'false' && steps.filter.outputs.backend == 'false' }} + ingestion-only: + description: "Ingestion only change" + value: ${{ steps.filter.outputs.frontend == 'false' && steps.filter.outputs.ingestion == 'true' && steps.filter.outputs.backend == 'false' }} + backend-only: + description: "Backend only change" + value: ${{ steps.filter.outputs.frontend == 'false' && steps.filter.outputs.ingestion == 'false' && steps.filter.outputs.backend == 'true' }} + backend-change: + description: "Backend code has changed" + value: ${{ steps.filter.outputs.backend == 'true' }} + ingestion-change: + description: "Ingestion code has changed" + value: ${{ steps.filter.outputs.ingestion == 'true' }} + frontend-change: + description: "Frontend code has changed" + value: ${{ steps.filter.outputs.frontend == 'true' }} + docker-change: + description: "Docker code has changed" + value: ${{ steps.filter.outputs.docker == 'true' }} + kafka-setup-change: + description: "Kafka setup docker change" + value: ${{ steps.filter.outputs.kafka-setup == 'true' }} + mysql-setup-change: + description: "Mysql setup docker change" + value: ${{ steps.filter.outputs.mysql-setup == 'true' }} + postgres-setup-change: + description: "Postgres setup docker change" + value: ${{ steps.filter.outputs.postgres-setup == 'true' }} + elasticsearch-setup-change: + description: "Elasticsearch setup docker change" + value: ${{ steps.filter.outputs.elasticsearch-setup == 'true' }} +runs: + using: "composite" + steps: + - uses: dorny/paths-filter@v2 + id: filter + with: + filters: | + frontend: + - "datahub-frontend/**" + - "datahub-web-react/**" + - "smoke-test/tests/cypress/**" + - "docker/datahub-frontend/**" + ingestion: + - "metadata-ingestion-modules/airflow-plugin/**" + - "metadata-ingestion/**" + - "metadata-models/**" + - "smoke-test/**" + - "docker/datahub-ingestion-**" + docker: + - "docker/**" + backend: + - "metadata-models/**" + - "datahub-upgrade/**" + - "entity-registry/**" + - "li-utils/**" + - "metadata-auth/**" + - "metadata-dao-impl/**" + - "metadata-events/**" + - "metadata-io/**" + - "metadata-jobs/**" + - "metadata-service/**" + - "metadata-utils/**" + - "smoke-test/**" + - "docker/**" + kafka-setup: + - "docker/kafka-setup/**" + mysql-setup: + - "docker/mysql-setup/**" + postgres-setup: + - "docker/postgres-setup/**" + elasticsearch-setup: + - "docker/elasticsearch-setup/**" \ No newline at end of file diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 64493e9921..180e0472a8 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -20,6 +20,25 @@ concurrency: cancel-in-progress: true jobs: + setup: + runs-on: ubuntu-latest + outputs: + frontend_change: ${{ steps.ci-optimize.outputs.frontend-change == 'true' }} + ingestion_change: ${{ steps.ci-optimize.outputs.ingestion-change == 'true' }} + backend_change: ${{ steps.ci-optimize.outputs.backend-change == 'true' }} + docker_change: ${{ steps.ci-optimize.outputs.docker-change == 'true' }} + frontend_only: ${{ steps.ci-optimize.outputs.frontend-only == 'true' }} + ingestion_only: ${{ steps.ci-optimize.outputs.ingestion-only == 'true' }} + kafka_setup_change: ${{ steps.ci-optimize.outputs.kafka-setup-change == 'true' }} + mysql_setup_change: ${{ steps.ci-optimize.outputs.mysql-setup-change == 'true' }} + postgres_setup_change: ${{ steps.ci-optimize.outputs.postgres-setup-change == 'true' }} + elasticsearch_setup_change: ${{ steps.ci-optimize.outputs.elasticsearch-setup-change == 'true' }} + steps: + - name: Check out the repo + uses: hsheth2/sane-checkout-action@v1 + - uses: ./.github/actions/ci-optimization + id: ci-optimize + build: strategy: fail-fast: false @@ -36,11 +55,13 @@ jobs: timezone: "America/New_York" runs-on: ubuntu-latest timeout-minutes: 60 + needs: setup steps: - uses: szenius/set-timezone@v1.2 with: timezoneLinux: ${{ matrix.timezone }} - - uses: hsheth2/sane-checkout-action@v1 + - name: Check out the repo + uses: hsheth2/sane-checkout-action@v1 - name: Set up JDK 17 uses: actions/setup-java@v3 with: @@ -51,12 +72,12 @@ jobs: with: python-version: "3.10" cache: pip - - name: Gradle build (and test) for metadata ingestion - if: ${{ matrix.command == 'except_metadata_ingestion' }} + - name: Gradle build (and test) for NOT metadata ingestion + if: ${{ matrix.command == 'except_metadata_ingestion' && needs.setup.outputs.backend_change == 'true' }} run: | ./gradlew build -x :metadata-ingestion:build -x :metadata-ingestion:check -x docs-website:build -x :metadata-integration:java:spark-lineage:test -x :metadata-io:test -x :metadata-ingestion-modules:airflow-plugin:build -x :metadata-ingestion-modules:airflow-plugin:check -x :datahub-frontend:build -x :datahub-web-react:build --parallel - name: Gradle build (and test) for frontend - if: ${{ matrix.command == 'frontend' }} + if: ${{ matrix.command == 'frontend' && needs.setup.outputs.frontend_change == 'true' }} run: | ./gradlew :datahub-frontend:build :datahub-web-react:build --parallel env: @@ -75,8 +96,11 @@ jobs: quickstart-compose-validation: runs-on: ubuntu-latest + needs: setup + if: ${{ needs.setup.outputs.docker_change == 'true' }} steps: - - uses: actions/checkout@v3 + - name: Check out the repo + uses: hsheth2/sane-checkout-action@v1 - uses: actions/setup-python@v4 with: python-version: "3.10" diff --git a/.github/workflows/docker-unified.yml b/.github/workflows/docker-unified.yml index d246bf329b..5e9112726b 100644 --- a/.github/workflows/docker-unified.yml +++ b/.github/workflows/docker-unified.yml @@ -51,9 +51,19 @@ jobs: short_sha: ${{ steps.tag.outputs.short_sha }} branch_name: ${{ steps.tag.outputs.branch_name }} repository_name: ${{ steps.tag.outputs.repository_name }} + frontend_change: ${{ steps.ci-optimize.outputs.frontend-change == 'true' }} + ingestion_change: ${{ steps.ci-optimize.outputs.ingestion-change == 'true' }} + backend_change: ${{ steps.ci-optimize.outputs.backend-change == 'true' }} + frontend_only: ${{ steps.ci-optimize.outputs.frontend-only == 'true' }} + ingestion_only: ${{ steps.ci-optimize.outputs.ingestion-only == 'true' }} + backend_only: ${{ steps.ci-optimize.outputs.backend-only == 'true' }} + kafka_setup_change: ${{ steps.ci-optimize.outputs.kafka-setup-change == 'true' }} + mysql_setup_change: ${{ steps.ci-optimize.outputs.mysql-setup-change == 'true' }} + postgres_setup_change: ${{ steps.ci-optimize.outputs.postgres-setup-change == 'true' }} + elasticsearch_setup_change: ${{ steps.ci-optimize.outputs.elasticsearch-setup-change == 'true' }} steps: - - name: Checkout - uses: actions/checkout@v3 + - name: Check out the repo + uses: hsheth2/sane-checkout-action@v1 - name: Compute Tag id: tag run: | @@ -75,11 +85,14 @@ jobs: run: | echo "Enable publish: ${{ env.ENABLE_PUBLISH }}" echo "publish=${{ env.ENABLE_PUBLISH }}" >> $GITHUB_OUTPUT + - uses: ./.github/actions/ci-optimization + id: ci-optimize gms_build: name: Build and Push DataHub GMS Docker Image runs-on: ubuntu-latest needs: setup + if: ${{ needs.setup.outputs.backend_change == 'true' || needs.setup.outputs.publish == 'true' }} steps: - name: Set up JDK 17 uses: actions/setup-java@v3 @@ -113,6 +126,7 @@ jobs: name: "[Monitoring] Scan GMS images for vulnerabilities" runs-on: ubuntu-latest needs: [setup, gms_build] + if: ${{ needs.setup.outputs.backend_change == 'true' || needs.setup.outputs.publish == 'true' }} steps: - name: Checkout # adding checkout step just to make trivy upload happy uses: actions/checkout@v3 @@ -142,6 +156,7 @@ jobs: name: Build and Push DataHub MAE Consumer Docker Image runs-on: ubuntu-latest needs: setup + if: ${{ needs.setup.outputs.backend_change == 'true' || needs.setup.outputs.publish == 'true' }} steps: - name: Set up JDK 17 uses: actions/setup-java@v3 @@ -171,6 +186,7 @@ jobs: name: "[Monitoring] Scan MAE consumer images for vulnerabilities" runs-on: ubuntu-latest needs: [setup, mae_consumer_build] + if: ${{ needs.setup.outputs.backend_change == 'true' || needs.setup.outputs.publish == 'true' }} permissions: contents: read # for actions/checkout to fetch code security-events: write # for github/codeql-action/upload-sarif to upload SARIF results @@ -204,6 +220,7 @@ jobs: name: Build and Push DataHub MCE Consumer Docker Image runs-on: ubuntu-latest needs: setup + if: ${{ needs.setup.outputs.backend_change == 'true' || needs.setup.outputs.publish == 'true' }} steps: - name: Set up JDK 17 uses: actions/setup-java@v3 @@ -233,6 +250,7 @@ jobs: name: "[Monitoring] Scan MCE consumer images for vulnerabilities" runs-on: ubuntu-latest needs: [setup, mce_consumer_build] + if: ${{ needs.setup.outputs.backend_change == 'true' || needs.setup.outputs.publish == 'true' }} permissions: contents: read # for actions/checkout to fetch code security-events: write # for github/codeql-action/upload-sarif to upload SARIF results @@ -266,6 +284,7 @@ jobs: name: Build and Push DataHub Upgrade Docker Image runs-on: ubuntu-latest needs: setup + if: ${{ needs.setup.outputs.backend_change == 'true' || needs.setup.outputs.publish == 'true' }} steps: - name: Set up JDK 17 uses: actions/setup-java@v3 @@ -295,6 +314,7 @@ jobs: name: "[Monitoring] Scan DataHub Upgrade images for vulnerabilities" runs-on: ubuntu-latest needs: [setup, datahub_upgrade_build] + if: ${{ needs.setup.outputs.backend_change == 'true' || needs.setup.outputs.publish == 'true' }} permissions: contents: read # for actions/checkout to fetch code security-events: write # for github/codeql-action/upload-sarif to upload SARIF results @@ -328,6 +348,7 @@ jobs: name: Build and Push DataHub Frontend Docker Image runs-on: ubuntu-latest needs: setup + if: ${{ needs.setup.outputs.frontend_change == 'true' || needs.setup.outputs.publish == 'true' }} steps: - name: Set up JDK 17 uses: actions/setup-java@v3 @@ -359,6 +380,7 @@ jobs: name: "[Monitoring] Scan Frontend images for vulnerabilities" runs-on: ubuntu-latest needs: [setup, frontend_build] + if: ${{ needs.setup.outputs.frontend_change == 'true' || needs.setup.outputs.publish == 'true' }} permissions: contents: read # for actions/checkout to fetch code security-events: write # for github/codeql-action/upload-sarif to upload SARIF results @@ -392,6 +414,7 @@ jobs: name: Build and Push DataHub Kafka Setup Docker Image runs-on: ubuntu-latest needs: setup + if: ${{ needs.setup.outputs.kafka_setup_change == 'true' || needs.setup.outputs.publish == 'true' }} steps: - name: Check out the repo uses: hsheth2/sane-checkout-action@v1 @@ -412,6 +435,7 @@ jobs: name: Build and Push DataHub MySQL Setup Docker Image runs-on: ubuntu-latest needs: setup + if: ${{ needs.setup.outputs.mysql_setup_change == 'true' || needs.setup.outputs.publish == 'true' }} steps: - name: Check out the repo uses: hsheth2/sane-checkout-action@v1 @@ -432,6 +456,7 @@ jobs: name: Build and Push DataHub Elasticsearch Setup Docker Image runs-on: ubuntu-latest needs: setup + if: ${{ needs.setup.outputs.elasticsearch_setup_change == 'true' || needs.setup.outputs.publish == 'true' }} steps: - name: Check out the repo uses: hsheth2/sane-checkout-action@v1 @@ -454,6 +479,7 @@ jobs: outputs: tag: ${{ steps.tag.outputs.tag }} needs: setup + if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' }} steps: - name: Check out the repo uses: hsheth2/sane-checkout-action@v1 @@ -486,6 +512,7 @@ jobs: outputs: tag: ${{ steps.tag.outputs.tag }} needs: [setup, datahub_ingestion_base_build] + if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' }} steps: - name: Check out the repo uses: hsheth2/sane-checkout-action@v1 @@ -526,6 +553,7 @@ jobs: outputs: tag: ${{ steps.tag.outputs.tag }} needs: [setup, datahub_ingestion_base_build] + if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' }} steps: - name: Check out the repo uses: hsheth2/sane-checkout-action@v1 @@ -568,6 +596,7 @@ jobs: tag: ${{ steps.tag.outputs.tag }} needs_artifact_download: ${{ (steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true') && needs.setup.outputs.publish != 'true' }} needs: [setup, datahub_ingestion_base_slim_build] + if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' }} steps: - name: Set up JDK 17 uses: actions/setup-java@v3 @@ -623,6 +652,7 @@ jobs: name: "[Monitoring] Scan Datahub Ingestion Slim images for vulnerabilities" runs-on: ubuntu-latest needs: [setup, datahub_ingestion_slim_build] + if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' }} steps: - name: Checkout # adding checkout step just to make trivy upload happy uses: actions/checkout@v3 @@ -655,6 +685,7 @@ jobs: tag: ${{ steps.tag.outputs.tag }} needs_artifact_download: ${{ (steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true') && needs.setup.outputs.publish != 'true' }} needs: [setup, datahub_ingestion_base_full_build] + if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' }} steps: - name: Set up JDK 17 uses: actions/setup-java@v3 @@ -709,6 +740,7 @@ jobs: name: "[Monitoring] Scan Datahub Ingestion images for vulnerabilities" runs-on: ubuntu-latest needs: [setup, datahub_ingestion_full_build] + if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' }} steps: - name: Checkout # adding checkout step just to make trivy upload happy uses: actions/checkout@v3 @@ -734,22 +766,31 @@ jobs: with: sarif_file: "trivy-results.sarif" + smoke_test_matrix: + runs-on: ubuntu-latest + needs: setup + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - id: set-matrix + run: | + if [ '${{ needs.setup.outputs.frontend_only }}' == 'true' ]; then + echo 'matrix=["cypress_suite1","cypress_rest"]' >> $GITHUB_OUTPUT + elif [ '${{ needs.setup.outputs.ingestion_only }}' == 'true' ]; then + echo 'matrix=["no_cypress_suite0","no_cypress_suite1"]' >> $GITHUB_OUTPUT + elif [ '${{ needs.setup.outputs.backend_change }}' == 'true' ]; then + echo 'matrix=["no_cypress_suite0","no_cypress_suite1","cypress_suite1","cypress_rest"]' >> $GITHUB_OUTPUT + else + echo 'matrix=[]' >> $GITHUB_OUTPUT + fi + smoke_test: name: Run Smoke Tests runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - test_strategy: - [ - "no_cypress_suite0", - "no_cypress_suite1", - "cypress_suite1", - "cypress_rest", - ] needs: [ setup, + smoke_test_matrix, gms_build, frontend_build, kafka_setup_build, @@ -760,6 +801,11 @@ jobs: datahub_upgrade_build, datahub_ingestion_slim_build, ] + strategy: + fail-fast: false + matrix: + test_strategy: ${{ fromJson(needs.smoke_test_matrix.outputs.matrix) }} + if: ${{ always() && !failure() && !cancelled() && needs.smoke_test_matrix.outputs.matrix != '[]' }} steps: - name: Disk Check run: df -h . && docker images @@ -788,57 +834,99 @@ jobs: run: df -h . && docker images - name: Download GMS image uses: ishworkh/docker-image-artifact-download@v1 - if: ${{ needs.setup.outputs.publish != 'true' }} + if: ${{ needs.setup.outputs.publish != 'true' && needs.gms_build.result == 'success' }} with: image: ${{ env.DATAHUB_GMS_IMAGE }}:${{ needs.setup.outputs.unique_tag }} - name: Download Frontend image uses: ishworkh/docker-image-artifact-download@v1 - if: ${{ needs.setup.outputs.publish != 'true' }} + if: ${{ needs.setup.outputs.publish != 'true' && needs.frontend_build.result == 'success' }} with: image: ${{ env.DATAHUB_FRONTEND_IMAGE }}:${{ needs.setup.outputs.unique_tag }} - name: Download Kafka Setup image uses: ishworkh/docker-image-artifact-download@v1 - if: ${{ needs.setup.outputs.publish != 'true' }} + if: ${{ needs.setup.outputs.publish != 'true' && needs.kafka_setup_build.result == 'success' }} with: image: ${{ env.DATAHUB_KAFKA_SETUP_IMAGE }}:${{ needs.setup.outputs.unique_tag }} - name: Download Mysql Setup image uses: ishworkh/docker-image-artifact-download@v1 - if: ${{ needs.setup.outputs.publish != 'true' }} + if: ${{ needs.setup.outputs.publish != 'true' && needs.mysql_setup_build.result == 'success' }} with: image: ${{ env.DATAHUB_MYSQL_SETUP_IMAGE }}:${{ needs.setup.outputs.unique_tag }} - name: Download Elastic Setup image uses: ishworkh/docker-image-artifact-download@v1 - if: ${{ needs.setup.outputs.publish != 'true' }} + if: ${{ needs.setup.outputs.publish != 'true' && needs.elasticsearch_setup_build.result == 'success' }} with: image: ${{ env.DATAHUB_ELASTIC_SETUP_IMAGE }}:${{ needs.setup.outputs.unique_tag }} - name: Download MCE Consumer image uses: ishworkh/docker-image-artifact-download@v1 - if: ${{ needs.setup.outputs.publish != 'true' }} + if: ${{ needs.setup.outputs.publish != 'true' && needs.mce_consumer_build.result == 'success' }} with: image: ${{ env.DATAHUB_MCE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }} - name: Download MAE Consumer image uses: ishworkh/docker-image-artifact-download@v1 - if: ${{ needs.setup.outputs.publish != 'true' }} + if: ${{ needs.setup.outputs.publish != 'true' && needs.mae_consumer_build.result == 'success' }} with: image: ${{ env.DATAHUB_MAE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }} - name: Download upgrade image uses: ishworkh/docker-image-artifact-download@v1 - if: ${{ needs.setup.outputs.publish != 'true' }} + if: ${{ needs.setup.outputs.publish != 'true' && needs.datahub_upgrade_build.result == 'success' }} with: image: ${{ env.DATAHUB_UPGRADE_IMAGE }}:${{ needs.setup.outputs.unique_tag }} - name: Download datahub-ingestion-slim image uses: ishworkh/docker-image-artifact-download@v1 - if: ${{ needs.datahub_ingestion_slim_build.outputs.needs_artifact_download == 'true' }} + if: ${{ needs.datahub_ingestion_slim_build.outputs.needs_artifact_download == 'true' && needs.datahub_ingestion_slim_build.result == 'success' }} with: image: ${{ env.DATAHUB_INGESTION_IMAGE }}:${{ needs.datahub_ingestion_slim_build.outputs.tag }} - name: Disk Check run: df -h . && docker images + - name: CI Optimization Head Images + # When publishing all tests/images are built (no optimizations) + if: ${{ needs.setup.outputs.publish != 'true' }} + run: | + if [ '${{ needs.setup.outputs.backend_change }}' == 'false' ]; then + echo 'GMS/Upgrade/MCE/MAE head images' + docker pull '${{ env.DATAHUB_GMS_IMAGE }}:head' + docker pull '${{ env.DATAHUB_MCE_CONSUMER_IMAGE }}:head' + docker pull '${{ env.DATAHUB_MAE_CONSUMER_IMAGE }}:head' + docker pull '${{ env.DATAHUB_UPGRADE_IMAGE }}:head' + docker tag '${{ env.DATAHUB_GMS_IMAGE }}:head' '${{ env.DATAHUB_GMS_IMAGE }}:${{ needs.setup.outputs.unique_tag }}' + docker tag '${{ env.DATAHUB_MCE_CONSUMER_IMAGE }}:head' '${{ env.DATAHUB_MCE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }}' + docker tag '${{ env.DATAHUB_MAE_CONSUMER_IMAGE }}:head' '${{ env.DATAHUB_MAE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }}' + docker tag '${{ env.DATAHUB_UPGRADE_IMAGE }}:head' '${{ env.DATAHUB_UPGRADE_IMAGE }}:${{ needs.setup.outputs.unique_tag }}' + fi + if [ '${{ needs.setup.outputs.frontend_change }}' == 'false' ]; then + echo 'Frontend head images' + docker pull '${{ env.DATAHUB_FRONTEND_IMAGE }}:head' + docker tag '${{ env.DATAHUB_FRONTEND_IMAGE }}:head' '${{ env.DATAHUB_FRONTEND_IMAGE }}:${{ needs.setup.outputs.unique_tag }}' + fi + if [ '${{ needs.setup.outputs.kafka_setup_change }}' == 'false' ]; then + echo 'kafka-setup head images' + docker pull '${{ env.DATAHUB_KAFKA_SETUP_IMAGE }}:head' + docker tag '${{ env.DATAHUB_KAFKA_SETUP_IMAGE }}:head' '${{ env.DATAHUB_KAFKA_SETUP_IMAGE }}:${{ needs.setup.outputs.unique_tag }}' + fi + if [ '${{ needs.setup.outputs.mysql_setup_change }}' == 'false' ]; then + echo 'mysql-setup head images' + docker pull '${{ env.DATAHUB_MYSQL_SETUP_IMAGE }}:head' + docker tag '${{ env.DATAHUB_MYSQL_SETUP_IMAGE }}:head' '${{ env.DATAHUB_MYSQL_SETUP_IMAGE }}:${{ needs.setup.outputs.unique_tag }}' + fi + if [ '${{ needs.setup.outputs.elasticsearch_setup_change }}' == 'false' ]; then + echo 'elasticsearch-setup head images' + docker pull '${{ env.DATAHUB_ELASTIC_SETUP_IMAGE }}:head' + docker tag '${{ env.DATAHUB_ELASTIC_SETUP_IMAGE }}:head' '${{ env.DATAHUB_ELASTIC_SETUP_IMAGE }}:${{ needs.setup.outputs.unique_tag }}' + fi + if [ '${{ needs.setup.outputs.ingestion_change }}' == 'false' ]; then + echo 'datahub-ingestion head-slim images' + docker pull '${{ env.DATAHUB_INGESTION_IMAGE }}:head-slim' + if [ '${{ needs.datahub_ingestion_slim_build.outputs.tag || 'head-slim' }}' != 'head-slim' ]; then + docker tag '${{ env.DATAHUB_INGESTION_IMAGE }}:head-slim' '${{ env.DATAHUB_INGESTION_IMAGE }}:${{ needs.datahub_ingestion_slim_build.outputs.tag }}' + fi + fi - name: run quickstart env: DATAHUB_TELEMETRY_ENABLED: false DATAHUB_VERSION: ${{ needs.setup.outputs.unique_tag }} DATAHUB_ACTIONS_IMAGE: ${{ env.DATAHUB_INGESTION_IMAGE }} - ACTIONS_VERSION: ${{ needs.datahub_ingestion_slim_build.outputs.tag }} + ACTIONS_VERSION: ${{ needs.datahub_ingestion_slim_build.outputs.tag || 'head-slim' }} ACTIONS_EXTRA_PACKAGES: "acryl-datahub-actions[executor]==0.0.13 acryl-datahub-actions==0.0.13 acryl-datahub==0.10.5" ACTIONS_CONFIG: "https://raw.githubusercontent.com/acryldata/datahub-actions/main/docker/config/executor.yaml" run: |