yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-11-28 19:56:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

ci(airflow): separate airflow constraints from deps (#13291)

Browse Source
This commit is contained in:
Harshal Sheth 2025-04-22 16:54:15 -04:00 committed by GitHub
parent 45c5a620e7
commit 82fafceba4
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 19 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
.github/workflows/airflow-plugin.yml vendored
View File

@ -35,17 +35,23 @@ jobs:
# Note: this should be kept in sync with tox.ini. # Note: this should be kept in sync with tox.ini.
- python-version: "3.8" - python-version: "3.8"
extra_pip_extras: test-airflow25 extra_pip_extras: test-airflow25
# extra_pip_requirements: "apache-airflow~=2.5.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.5.3/constraints-3.8.txt" extra_pip_requirements: "apache-airflow~=2.5.3"
# extra_pip_constraints: "-c https://raw.githubusercontent.com/apache/airflow/constraints-2.5.3/constraints-3.8.txt"
- python-version: "3.10" - python-version: "3.10"
extra_pip_requirements: "apache-airflow~=2.6.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.6.3/constraints-3.10.txt" extra_pip_requirements: "apache-airflow~=2.6.3"
extra_pip_constraints: "-c https://raw.githubusercontent.com/apache/airflow/constraints-2.6.3/constraints-3.10.txt"
- python-version: "3.10" - python-version: "3.10"
extra_pip_requirements: "apache-airflow~=2.7.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.7.3/constraints-3.10.txt" extra_pip_requirements: "apache-airflow~=2.7.3"
extra_pip_constraints: "-c https://raw.githubusercontent.com/apache/airflow/constraints-2.7.3/constraints-3.10.txt"
- python-version: "3.10" - python-version: "3.10"
extra_pip_requirements: "apache-airflow~=2.8.1 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.8.1/constraints-3.10.txt" extra_pip_requirements: "apache-airflow~=2.8.1"
extra_pip_constraints: "-c https://raw.githubusercontent.com/apache/airflow/constraints-2.8.1/constraints-3.10.txt"
- python-version: "3.11" - python-version: "3.11"
extra_pip_requirements: "apache-airflow~=2.9.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.9.3/constraints-3.11.txt" extra_pip_requirements: "apache-airflow~=2.9.3"
extra_pip_constraints: "-c https://raw.githubusercontent.com/apache/airflow/constraints-2.9.3/constraints-3.11.txt"
- python-version: "3.11" - python-version: "3.11"
extra_pip_requirements: "apache-airflow~=2.10.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.10.3/constraints-3.11.txt" extra_pip_requirements: "apache-airflow~=2.10.3"
extra_pip_constraints: "-c https://raw.githubusercontent.com/apache/airflow/constraints-2.10.3/constraints-3.11.txt"
fail-fast: false fail-fast: false
steps: steps:
- name: Set up JDK 17 - name: Set up JDK 17
@ -62,7 +68,7 @@ jobs:
- name: Install dependencies - name: Install dependencies
run: ./metadata-ingestion/scripts/install_deps.sh run: ./metadata-ingestion/scripts/install_deps.sh
- name: Install airflow package and test (extras ${{ matrix.extra_pip_requirements }}) - name: Install airflow package and test (extras ${{ matrix.extra_pip_requirements }})
run: ./gradlew -Pextra_pip_requirements='${{ matrix.extra_pip_requirements }}' -Pextra_pip_extras='${{ matrix.extra_pip_extras }}' :metadata-ingestion-modules:airflow-plugin:build run: ./gradlew -Pextra_pip_requirements='${{ matrix.extra_pip_requirements }}' -Pextra_pip_constraints='${{ matrix.extra_pip_constraints }}' -Pextra_pip_extras='${{ matrix.extra_pip_extras }}' :metadata-ingestion-modules:airflow-plugin:build
- name: pip freeze show list installed - name: pip freeze show list installed
if: always() if: always()
run: source metadata-ingestion-modules/airflow-plugin/venv/bin/activate && uv pip freeze run: source metadata-ingestion-modules/airflow-plugin/venv/bin/activate && uv pip freeze

9
metadata-ingestion-modules/airflow-plugin/build.gradle
View File

@ -14,6 +14,9 @@ ext.venv_activate_command = "set +x && source ${venv_name}/bin/activate && set -
if (!project.hasProperty("extra_pip_requirements")) { if (!project.hasProperty("extra_pip_requirements")) {
ext.extra_pip_requirements = "" ext.extra_pip_requirements = ""
} }
if (!project.hasProperty("extra_pip_constraints")) {
ext.extra_pip_constraints = ""
}
if (!project.hasProperty("extra_pip_extras")) { if (!project.hasProperty("extra_pip_extras")) {
ext.extra_pip_extras = "" ext.extra_pip_extras = ""
} }
@ -22,7 +25,7 @@ if (extra_pip_extras != "") {
ext.extra_pip_extras = "," + extra_pip_extras ext.extra_pip_extras = "," + extra_pip_extras
} }
def pip_install_command = "VIRTUAL_ENV=${venv_name} ${venv_name}/bin/uv pip install -e ../../metadata-ingestion" def pip_install_command = "VIRTUAL_ENV=${venv_name} ${venv_name}/bin/uv pip install -e ../../metadata-ingestion ${extra_pip_requirements}"
task environmentSetup(type: Exec) { task environmentSetup(type: Exec) {
def sentinel_file = "${venv_name}/.venv_environment_sentinel" def sentinel_file = "${venv_name}/.venv_environment_sentinel"
@ -40,7 +43,7 @@ task installPackage(type: Exec, dependsOn: [environmentSetup, ':metadata-ingesti
outputs.file(sentinel_file) outputs.file(sentinel_file)
commandLine 'bash', '-c', commandLine 'bash', '-c',
venv_activate_command + venv_activate_command +
"${pip_install_command} -e .[ignore${extra_pip_extras}] ${extra_pip_requirements} &&" + "${pip_install_command} -e .[ignore${extra_pip_extras}] ${extra_pip_constraints} &&" +
"touch ${sentinel_file}" "touch ${sentinel_file}"
} }
@ -63,7 +66,7 @@ task installTest(type: Exec, dependsOn: [installDev]) {
outputs.file(sentinel_file) outputs.file(sentinel_file)
commandLine 'bash', '-c', commandLine 'bash', '-c',
venv_activate_command + venv_activate_command +
"${pip_install_command} -e .[integration-tests${extra_pip_extras}] ${extra_pip_requirements} && " + "${pip_install_command} -e .[integration-tests${extra_pip_extras}] ${extra_pip_constraints} && " +
"touch ${sentinel_file}" "touch ${sentinel_file}"
} }