yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-09-27 01:55:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(ingest/airflow): allow plugin to load on listener exception (#10152)

Browse Source
This commit is contained in:
Harshal Sheth 2024-03-28 12:43:37 -07:00 committed by GitHub
parent 4e328c38a7
commit 32a2de4dfc
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 22 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/airflow-plugin.yml vendored
View File

@ -67,7 +67,7 @@ jobs:
- name: Install dependencies - name: Install dependencies
run: ./metadata-ingestion/scripts/install_deps.sh run: ./metadata-ingestion/scripts/install_deps.sh
- name: Install airflow package and test (extras ${{ matrix.extra_pip_requirements }}) - name: Install airflow package and test (extras ${{ matrix.extra_pip_requirements }})
run: ./gradlew -Pextra_pip_requirements='${{ matrix.extra_pip_requirements }}' -Pextra_pip_extras='${{ matrix.extra_pip_extras }}' :metadata-ingestion-modules:airflow-plugin:lint :metadata-ingestion-modules:airflow-plugin:testQuick run: ./gradlew -Pextra_pip_requirements='${{ matrix.extra_pip_requirements }}' -Pextra_pip_extras='${{ matrix.extra_pip_extras }}' :metadata-ingestion-modules:airflow-plugin:build
- name: pip freeze show list installed - name: pip freeze show list installed
if: always() if: always()
run: source metadata-ingestion-modules/airflow-plugin/venv/bin/activate && pip freeze run: source metadata-ingestion-modules/airflow-plugin/venv/bin/activate && pip freeze

7
metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_plugin.py
View File

@ -50,6 +50,7 @@ class DatahubPlugin(AirflowPlugin):
name = "datahub_plugin" name = "datahub_plugin"
if _USE_AIRFLOW_LISTENER_INTERFACE: if _USE_AIRFLOW_LISTENER_INTERFACE:
try:
if not NEEDS_AIRFLOW_LISTENER_MODULE: if not NEEDS_AIRFLOW_LISTENER_MODULE:
from datahub_airflow_plugin.datahub_listener import ( # type: ignore[misc] from datahub_airflow_plugin.datahub_listener import ( # type: ignore[misc]
get_airflow_plugin_listener, get_airflow_plugin_listener,
@ -65,6 +66,12 @@ class DatahubPlugin(AirflowPlugin):
import datahub_airflow_plugin._datahub_listener_module as _listener_module # type: ignore[misc] import datahub_airflow_plugin._datahub_listener_module as _listener_module # type: ignore[misc]
listeners = [_listener_module] listeners = [_listener_module]
except Exception as e:
logger.warning(
f"Failed to load the DataHub plugin's event listener: {e}",
exc_info=True,
)
listeners = []
if not _USE_AIRFLOW_LISTENER_INTERFACE: if not _USE_AIRFLOW_LISTENER_INTERFACE: