From c43d7dd9d24a36792f9bfee66c5e6e4484a3922d Mon Sep 17 00:00:00 2001 From: Aseem Bansal Date: Fri, 17 Mar 2023 09:34:00 +0530 Subject: [PATCH] chore(ci): fix flakiness, misc improvements (#7605) --- .github/workflows/check-datahub-jars.yml | 23 +++++++++-------- .github/workflows/docker-unified.yml | 14 ---------- .github/workflows/metadata-ingestion.yml | 22 +++------------- metadata-ingestion/.gitignore | 1 + metadata-ingestion/build.gradle | 33 ++++++++++++++++++------ 5 files changed, 41 insertions(+), 52 deletions(-) diff --git a/.github/workflows/check-datahub-jars.yml b/.github/workflows/check-datahub-jars.yml index 2d3f0ebfa3..7af888f34c 100644 --- a/.github/workflows/check-datahub-jars.yml +++ b/.github/workflows/check-datahub-jars.yml @@ -24,6 +24,15 @@ concurrency: jobs: check_jars: + strategy: + fail-fast: false + matrix: + command: + [ + "datahub-client", + "datahub-protobuf", + "spark-lineage" + ] runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -37,15 +46,7 @@ jobs: - uses: actions/setup-python@v4 with: python-version: "3.7" - - name: check datahub-client jar + - name: check ${{ matrix.command }} jar run: | - ./gradlew :metadata-integration:java:datahub-client:build - ./gradlew :metadata-integration:java:datahub-client:javadoc - - name: check spark-lineage jar - run: | - ./gradlew :metadata-integration:java:spark-lineage:build - ./gradlew :metadata-integration:java:spark-lineage:javadoc - - name: check datahub-protobuf jar - run: | - ./gradlew :metadata-integration:java:datahub-protobuf:build - ./gradlew :metadata-integration:java:datahub-protobuf:javadoc + ./gradlew :metadata-integration:java:${{ matrix.command }}:build --info + ./gradlew :metadata-integration:java:${{ matrix.command }}:javadoc diff --git a/.github/workflows/docker-unified.yml b/.github/workflows/docker-unified.yml index 7d722ddfa4..3224173693 100644 --- a/.github/workflows/docker-unified.yml +++ b/.github/workflows/docker-unified.yml @@ -490,20 +490,6 @@ jobs: if: ${{ needs.setup.outputs.publish != 'true' }} with: image: ${{ env.DATAHUB_UPGRADE_IMAGE }}:${{ needs.setup.outputs.unique_tag }} - - name: Calculate pip install plan - id: "packages_checksum" - run: | - pip install pip -U # only 22.2 and above contains the --dry-run flag - - # only the last line of the output is the packages that will be installed - pip install --dry-run -r ./smoke-test/requirements.txt | tail -n 1 > /tmp/would_be_installed.txt - CHECKSUM=$(shasum /tmp/would_be_installed.txt | awk '{print $1}') - echo "packages_checksum=$CHECKSUM" >> $GITHUB_OUTPUT - - uses: actions/cache@v3 - id: cache-venv - with: - path: ./smoke-test/venv/ - key: smoke-venv-${{ runner.os }}-${{ steps.packages_checksum.outputs.packages_checksum }} - name: run quickstart env: DATAHUB_TELEMETRY_ENABLED: false diff --git a/.github/workflows/metadata-ingestion.yml b/.github/workflows/metadata-ingestion.yml index 53aed77ab9..b576809c3e 100644 --- a/.github/workflows/metadata-ingestion.yml +++ b/.github/workflows/metadata-ingestion.yml @@ -54,26 +54,10 @@ jobs: cache: 'pip' - name: Install dependencies run: ./metadata-ingestion/scripts/install_deps.sh - - name: Calculate pip install plan - id: "packages_checksum" - run: | - cd metadata-ingestion - pip install pip -U # only 22.2 and above contains the --dry-run flag - - # only the last line of the output is the packages that will be installed - pip install --dry-run -e .[dev] ${{ matrix.extraPythonRequirement }} | tail -n 1 > /tmp/would_be_installed.txt - CHECKSUM=$(shasum /tmp/would_be_installed.txt | awk '{print $1}') - echo "packages_checksum=$CHECKSUM" >> $GITHUB_OUTPUT - - name: print dependencies - id: print-dependency - run: cat /tmp/would_be_installed.txt - - uses: actions/cache@v3 - id: cache-venv - with: - path: ./metadata-ingestion/venv/ - key: ${{ runner.os }}-venv-${{ steps.packages_checksum.outputs.packages_checksum }}-${{ matrix.python-version }} + - name: Install package + run: ./gradlew :metadata-ingestion:installPackageOnly - name: Run metadata-ingestion tests (extras ${{ matrix.extraPythonRequirement }}) - run: ./gradlew -Pextra_pip_requirements='${{ matrix.extraPythonRequirement }}' :metadata-ingestion:${{ matrix.command }} + run: ./gradlew -Pextra_pip_requirements='${{ matrix.extraPythonRequirement }}' :metadata-ingestion:${{ matrix.command }} -x :metadata-ingestion:installPackageOnly - name: pip freeze show list installed if: always() run: source metadata-ingestion/venv/bin/activate && pip freeze diff --git a/metadata-ingestion/.gitignore b/metadata-ingestion/.gitignore index 3fb2d89b2b..301391fd9c 100644 --- a/metadata-ingestion/.gitignore +++ b/metadata-ingestion/.gitignore @@ -140,3 +140,4 @@ dmypy.json # Generated classes src/datahub/metadata/ +.preflight_sentinel \ No newline at end of file diff --git a/metadata-ingestion/build.gradle b/metadata-ingestion/build.gradle index c0e1dfccc2..5565f9e2be 100644 --- a/metadata-ingestion/build.gradle +++ b/metadata-ingestion/build.gradle @@ -12,7 +12,8 @@ if (!project.hasProperty("extra_pip_requirements")) { } task checkPythonVersion(type: Exec) { - commandLine python_executable, '-c', 'import sys; assert sys.version_info >= (3, 7)' + commandLine python_executable, '-c', + 'import sys; assert (3, 11) > sys.version_info >= (3, 7), f"Python version {sys.version_info[:2]} not allowed"' } task environmentSetup(type: Exec, dependsOn: checkPythonVersion) { @@ -22,10 +23,22 @@ task environmentSetup(type: Exec, dependsOn: checkPythonVersion) { } task runPreFlightScript(type: Exec, dependsOn: environmentSetup) { + def sentinel_file = ".preflight_sentinel" + outputs.file(sentinel_file) commandLine "scripts/datahub_preflight.sh" + commandLine 'bash', '-c', "touch ${sentinel_file}" } -task installPackage(type: Exec, dependsOn: runPreFlightScript) { +task installPackageOnly(type: Exec, dependsOn: runPreFlightScript) { + def sentinel_file = "${venv_name}/.build_install_package_only_sentinel" + inputs.file file('setup.py') + outputs.dir("${venv_name}") + outputs.file(sentinel_file) + commandLine 'bash', '-x', '-c', "${venv_name}/bin/pip install -e ." + commandLine 'bash', '-c', "touch ${sentinel_file}" +} + +task installPackage(type: Exec, dependsOn: installPackageOnly) { inputs.file file('setup.py') outputs.dir("${venv_name}") commandLine 'bash', '-x', '-c', "${venv_name}/bin/pip install -e . ${extra_pip_requirements}" @@ -40,25 +53,27 @@ task codegen(type: Exec, dependsOn: [environmentSetup, installPackage, ':metadat task install(dependsOn: [installPackage, codegen]) task installDev(type: Exec, dependsOn: [install]) { + def sentinel_file = "${venv_name}/.build_install_dev_sentinel" inputs.file file('setup.py') outputs.dir("${venv_name}") - outputs.file("${venv_name}/.build_install_dev_sentinel") + outputs.file(sentinel_file) commandLine 'bash', '-c', "source ${venv_name}/bin/activate && set -x && " + "${venv_name}/bin/pip install -e .[dev] ${extra_pip_requirements} && " + "./scripts/install-sqlalchemy-stubs.sh && " + - "touch ${venv_name}/.build_install_dev_sentinel" + "touch ${sentinel_file}" } task installAll(type: Exec, dependsOn: [install]) { + def sentinel_file = "${venv_name}/.build_install_all_sentinel" inputs.file file('setup.py') outputs.dir("${venv_name}") - outputs.file("${venv_name}/.build_install_all_sentinel") + outputs.file(sentinel_file) commandLine 'bash', '-c', "source ${venv_name}/bin/activate && set -x && " + "${venv_name}/bin/pip install -e .[all] ${extra_pip_requirements} && " + "./scripts/install-sqlalchemy-stubs.sh && " + - "touch ${venv_name}/.build_install_all_sentinel" + "touch ${sentinel_file}" } @@ -111,11 +126,12 @@ task testQuick(type: Exec, dependsOn: [installDev, ':metadata-models:generateJso } task installDevTest(type: Exec, dependsOn: [install]) { + def sentinel_file = "${venv_name}/.build_install_dev_test_sentinel" inputs.file file('setup.py') outputs.dir("${venv_name}") - outputs.file("${venv_name}/.build_install_dev_test_sentinel") + outputs.file(sentinel_file) commandLine 'bash', '-c', - "${venv_name}/bin/pip install -e .[dev,integration-tests] ${extra_pip_requirements} && touch ${venv_name}/.build_install_dev_test_sentinel" + "${venv_name}/bin/pip install -e .[dev,integration-tests] ${extra_pip_requirements} && touch ${sentinel_file}" } def testFile = hasProperty('testFile') ? testFile : 'unknown' @@ -176,6 +192,7 @@ clean { delete 'generated' delete '.mypy_cache' delete '.pytest_cache' + delete '.preflight_sentinel' } clean.dependsOn cleanPythonCache