chore(ci): fix flakiness, misc improvements (#7605)

.github/workflows/check-datahub-jars.yml

@@ -24,6 +24,15 @@ concurrency:
 
 jobs:
   check_jars:
+    strategy:
+      fail-fast: false
+      matrix:
+        command:
+          [
+            "datahub-client",
+            "datahub-protobuf",
+            "spark-lineage"
+          ]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
@@ -37,15 +46,7 @@ jobs:
       - uses: actions/setup-python@v4
         with:
           python-version: "3.7"
-      - name: check datahub-client jar
+      - name: check ${{ matrix.command }} jar
         run: |
-          ./gradlew :metadata-integration:java:datahub-client:build
-          ./gradlew :metadata-integration:java:datahub-client:javadoc
-      - name: check spark-lineage jar
-        run: |
-          ./gradlew :metadata-integration:java:spark-lineage:build
-          ./gradlew :metadata-integration:java:spark-lineage:javadoc
-      - name: check datahub-protobuf jar
-        run: |
-          ./gradlew :metadata-integration:java:datahub-protobuf:build
-          ./gradlew :metadata-integration:java:datahub-protobuf:javadoc
+          ./gradlew :metadata-integration:java:${{ matrix.command }}:build --info
+          ./gradlew :metadata-integration:java:${{ matrix.command }}:javadoc

.github/workflows/docker-unified.yml

@@ -490,20 +490,6 @@ jobs:
         if: ${{ needs.setup.outputs.publish != 'true' }}
         with:
           image: ${{ env.DATAHUB_UPGRADE_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
-      - name: Calculate pip install plan
-        id: "packages_checksum"
-        run: |
-          pip install pip -U # only 22.2 and above contains the --dry-run flag
-
-          # only the last line of the output is the packages that will be installed
-          pip install --dry-run -r ./smoke-test/requirements.txt  | tail -n 1 > /tmp/would_be_installed.txt
-          CHECKSUM=$(shasum /tmp/would_be_installed.txt | awk '{print $1}')
-          echo "packages_checksum=$CHECKSUM" >> $GITHUB_OUTPUT
-      - uses: actions/cache@v3
-        id: cache-venv
-        with:
-          path: ./smoke-test/venv/
-          key: smoke-venv-${{ runner.os }}-${{ steps.packages_checksum.outputs.packages_checksum }}
       - name: run quickstart
         env:
           DATAHUB_TELEMETRY_ENABLED: false

.github/workflows/metadata-ingestion.yml

@@ -54,26 +54,10 @@ jobs:
           cache: 'pip'
       - name: Install dependencies
         run: ./metadata-ingestion/scripts/install_deps.sh
-      - name: Calculate pip install plan
-        id: "packages_checksum"
-        run: |
-          cd metadata-ingestion
-          pip install pip -U # only 22.2 and above contains the --dry-run flag
-
-          # only the last line of the output is the packages that will be installed
-          pip install --dry-run -e .[dev]  ${{ matrix.extraPythonRequirement }} | tail -n 1 > /tmp/would_be_installed.txt
-          CHECKSUM=$(shasum /tmp/would_be_installed.txt | awk '{print $1}')
-          echo "packages_checksum=$CHECKSUM" >> $GITHUB_OUTPUT
-      - name: print dependencies
-        id: print-dependency
-        run: cat /tmp/would_be_installed.txt
-      - uses: actions/cache@v3
-        id: cache-venv
-        with:
-          path: ./metadata-ingestion/venv/
-          key: ${{ runner.os }}-venv-${{ steps.packages_checksum.outputs.packages_checksum }}-${{ matrix.python-version }}
+      - name: Install package
+        run: ./gradlew :metadata-ingestion:installPackageOnly
       - name: Run metadata-ingestion tests (extras ${{ matrix.extraPythonRequirement }})
-        run: ./gradlew -Pextra_pip_requirements='${{ matrix.extraPythonRequirement }}' :metadata-ingestion:${{ matrix.command }}
+        run: ./gradlew -Pextra_pip_requirements='${{ matrix.extraPythonRequirement }}' :metadata-ingestion:${{ matrix.command }} -x :metadata-ingestion:installPackageOnly
       - name: pip freeze show list installed
         if: always()
         run: source metadata-ingestion/venv/bin/activate && pip freeze

metadata-ingestion/.gitignore

@@ -140,3 +140,4 @@ dmypy.json
 
 # Generated classes
 src/datahub/metadata/
+.preflight_sentinel

metadata-ingestion/build.gradle

@@ -12,7 +12,8 @@ if (!project.hasProperty("extra_pip_requirements")) {
 }
 
 task checkPythonVersion(type: Exec) {
-  commandLine python_executable, '-c', 'import sys; assert sys.version_info >= (3, 7)'
+  commandLine python_executable, '-c',
+    'import sys; assert (3, 11) > sys.version_info >= (3, 7), f"Python version {sys.version_info[:2]} not allowed"'
 }
 
 task environmentSetup(type: Exec, dependsOn: checkPythonVersion) {
@@ -22,10 +23,22 @@ task environmentSetup(type: Exec, dependsOn: checkPythonVersion) {
 }
 
 task runPreFlightScript(type: Exec, dependsOn: environmentSetup) {
+  def sentinel_file = ".preflight_sentinel"
+  outputs.file(sentinel_file)
   commandLine "scripts/datahub_preflight.sh"
+  commandLine 'bash', '-c', "touch ${sentinel_file}"
 }
 
-task installPackage(type: Exec, dependsOn: runPreFlightScript) {
+task installPackageOnly(type: Exec, dependsOn: runPreFlightScript) {
+  def sentinel_file = "${venv_name}/.build_install_package_only_sentinel"
+  inputs.file file('setup.py')
+  outputs.dir("${venv_name}")
+  outputs.file(sentinel_file)
+  commandLine 'bash', '-x', '-c', "${venv_name}/bin/pip install -e ."
+  commandLine 'bash', '-c', "touch ${sentinel_file}"
+}
+
+task installPackage(type: Exec, dependsOn: installPackageOnly) {
   inputs.file file('setup.py')
   outputs.dir("${venv_name}")
   commandLine 'bash', '-x', '-c', "${venv_name}/bin/pip install -e . ${extra_pip_requirements}"
@@ -40,25 +53,27 @@ task codegen(type: Exec, dependsOn: [environmentSetup, installPackage, ':metadat
 task install(dependsOn: [installPackage, codegen])
 
 task installDev(type: Exec, dependsOn: [install]) {
+  def sentinel_file = "${venv_name}/.build_install_dev_sentinel"
   inputs.file file('setup.py')
   outputs.dir("${venv_name}")
-  outputs.file("${venv_name}/.build_install_dev_sentinel")
+  outputs.file(sentinel_file)
   commandLine 'bash', '-c',
     "source ${venv_name}/bin/activate && set -x && " +
     "${venv_name}/bin/pip install -e .[dev] ${extra_pip_requirements} && " +
     "./scripts/install-sqlalchemy-stubs.sh && " +
-    "touch ${venv_name}/.build_install_dev_sentinel"
+    "touch ${sentinel_file}"
 }
 
 task installAll(type: Exec, dependsOn: [install]) {
+  def sentinel_file = "${venv_name}/.build_install_all_sentinel"
   inputs.file file('setup.py')
   outputs.dir("${venv_name}")
-  outputs.file("${venv_name}/.build_install_all_sentinel")
+  outputs.file(sentinel_file)
   commandLine 'bash', '-c',
     "source ${venv_name}/bin/activate && set -x && " +
     "${venv_name}/bin/pip install -e .[all] ${extra_pip_requirements} && " +
     "./scripts/install-sqlalchemy-stubs.sh && " +
-    "touch ${venv_name}/.build_install_all_sentinel"
+    "touch ${sentinel_file}"
 }
 
 
@@ -111,11 +126,12 @@ task testQuick(type: Exec, dependsOn: [installDev, ':metadata-models:generateJso
 }
 
 task installDevTest(type: Exec, dependsOn: [install]) {
+  def sentinel_file = "${venv_name}/.build_install_dev_test_sentinel"
   inputs.file file('setup.py')
   outputs.dir("${venv_name}")
-  outputs.file("${venv_name}/.build_install_dev_test_sentinel")
+  outputs.file(sentinel_file)
   commandLine 'bash', '-c',
-    "${venv_name}/bin/pip install -e .[dev,integration-tests] ${extra_pip_requirements} && touch ${venv_name}/.build_install_dev_test_sentinel"
+    "${venv_name}/bin/pip install -e .[dev,integration-tests] ${extra_pip_requirements} && touch ${sentinel_file}"
 }
 
 def testFile = hasProperty('testFile') ? testFile : 'unknown'
@@ -176,6 +192,7 @@ clean {
   delete 'generated'
   delete '.mypy_cache'
   delete '.pytest_cache'
+  delete '.preflight_sentinel'
 }
 clean.dependsOn cleanPythonCache