From 82fafceba47d29304ff71ba4b70b917178531fe2 Mon Sep 17 00:00:00 2001
From: Harshal Sheth <hsheth2@gmail.com>
Date: Tue, 22 Apr 2025 16:54:15 -0400
Subject: [PATCH] ci(airflow): separate airflow constraints from deps (#13291)

---
 .github/workflows/airflow-plugin.yml          | 20 ++++++++++++-------
 .../airflow-plugin/build.gradle               |  9 ++++++---
 2 files changed, 19 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/airflow-plugin.yml b/.github/workflows/airflow-plugin.yml
index 684d3e2cff..53b5623ff3 100644
--- a/.github/workflows/airflow-plugin.yml
+++ b/.github/workflows/airflow-plugin.yml
@@ -35,17 +35,23 @@ jobs:
           # Note: this should be kept in sync with tox.ini.
           - python-version: "3.8"
             extra_pip_extras: test-airflow25
-            # extra_pip_requirements: "apache-airflow~=2.5.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.5.3/constraints-3.8.txt"
+            extra_pip_requirements: "apache-airflow~=2.5.3"
+            # extra_pip_constraints: "-c https://raw.githubusercontent.com/apache/airflow/constraints-2.5.3/constraints-3.8.txt"
           - python-version: "3.10"
-            extra_pip_requirements: "apache-airflow~=2.6.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.6.3/constraints-3.10.txt"
+            extra_pip_requirements: "apache-airflow~=2.6.3"
+            extra_pip_constraints: "-c https://raw.githubusercontent.com/apache/airflow/constraints-2.6.3/constraints-3.10.txt"
           - python-version: "3.10"
-            extra_pip_requirements: "apache-airflow~=2.7.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.7.3/constraints-3.10.txt"
+            extra_pip_requirements: "apache-airflow~=2.7.3"
+            extra_pip_constraints: "-c https://raw.githubusercontent.com/apache/airflow/constraints-2.7.3/constraints-3.10.txt"
           - python-version: "3.10"
-            extra_pip_requirements: "apache-airflow~=2.8.1 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.8.1/constraints-3.10.txt"
+            extra_pip_requirements: "apache-airflow~=2.8.1"
+            extra_pip_constraints: "-c https://raw.githubusercontent.com/apache/airflow/constraints-2.8.1/constraints-3.10.txt"
           - python-version: "3.11"
-            extra_pip_requirements: "apache-airflow~=2.9.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.9.3/constraints-3.11.txt"
+            extra_pip_requirements: "apache-airflow~=2.9.3"
+            extra_pip_constraints: "-c https://raw.githubusercontent.com/apache/airflow/constraints-2.9.3/constraints-3.11.txt"
           - python-version: "3.11"
-            extra_pip_requirements: "apache-airflow~=2.10.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.10.3/constraints-3.11.txt"
+            extra_pip_requirements: "apache-airflow~=2.10.3"
+            extra_pip_constraints: "-c https://raw.githubusercontent.com/apache/airflow/constraints-2.10.3/constraints-3.11.txt"
       fail-fast: false
     steps:
       - name: Set up JDK 17
@@ -62,7 +68,7 @@ jobs:
       - name: Install dependencies
         run: ./metadata-ingestion/scripts/install_deps.sh
       - name: Install airflow package and test (extras ${{ matrix.extra_pip_requirements }})
-        run: ./gradlew -Pextra_pip_requirements='${{ matrix.extra_pip_requirements }}' -Pextra_pip_extras='${{ matrix.extra_pip_extras }}' :metadata-ingestion-modules:airflow-plugin:build
+        run: ./gradlew -Pextra_pip_requirements='${{ matrix.extra_pip_requirements }}' -Pextra_pip_constraints='${{ matrix.extra_pip_constraints }}' -Pextra_pip_extras='${{ matrix.extra_pip_extras }}' :metadata-ingestion-modules:airflow-plugin:build
       - name: pip freeze show list installed
         if: always()
         run: source metadata-ingestion-modules/airflow-plugin/venv/bin/activate && uv pip freeze
diff --git a/metadata-ingestion-modules/airflow-plugin/build.gradle b/metadata-ingestion-modules/airflow-plugin/build.gradle
index dc494d840f..5f1240f7fd 100644
--- a/metadata-ingestion-modules/airflow-plugin/build.gradle
+++ b/metadata-ingestion-modules/airflow-plugin/build.gradle
@@ -14,6 +14,9 @@ ext.venv_activate_command = "set +x && source ${venv_name}/bin/activate && set -
 if (!project.hasProperty("extra_pip_requirements")) {
     ext.extra_pip_requirements = ""
 }
+if (!project.hasProperty("extra_pip_constraints")) {
+    ext.extra_pip_constraints = ""
+}
 if (!project.hasProperty("extra_pip_extras")) {
     ext.extra_pip_extras = ""
 }
@@ -22,7 +25,7 @@ if (extra_pip_extras != "") {
     ext.extra_pip_extras = "," + extra_pip_extras
 }
 
-def pip_install_command = "VIRTUAL_ENV=${venv_name} ${venv_name}/bin/uv pip install -e ../../metadata-ingestion"
+def pip_install_command = "VIRTUAL_ENV=${venv_name} ${venv_name}/bin/uv pip install -e ../../metadata-ingestion ${extra_pip_requirements}"
 
 task environmentSetup(type: Exec) {
   def sentinel_file = "${venv_name}/.venv_environment_sentinel"
@@ -40,7 +43,7 @@ task installPackage(type: Exec, dependsOn: [environmentSetup, ':metadata-ingesti
   outputs.file(sentinel_file)
   commandLine 'bash', '-c',
     venv_activate_command +
-    "${pip_install_command} -e .[ignore${extra_pip_extras}] ${extra_pip_requirements} &&" +
+    "${pip_install_command} -e .[ignore${extra_pip_extras}] ${extra_pip_constraints} &&" +
     "touch ${sentinel_file}"
 }
 
@@ -63,7 +66,7 @@ task installTest(type: Exec, dependsOn: [installDev]) {
   outputs.file(sentinel_file)
   commandLine 'bash', '-c',
     venv_activate_command +
-    "${pip_install_command} -e .[integration-tests${extra_pip_extras}] ${extra_pip_requirements} && " +
+    "${pip_install_command} -e .[integration-tests${extra_pip_extras}] ${extra_pip_constraints} && " +
     "touch ${sentinel_file}"
 }