yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-09-25 09:00:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(ingest/airflow): allow plugin to load on listener exception (#10152)

Browse Source
This commit is contained in:
Harshal Sheth 2024-03-28 12:43:37 -07:00 committed by GitHub
parent 4e328c38a7
commit 32a2de4dfc
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 22 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/airflow-plugin.yml vendored
View File

@ -67,7 +67,7 @@ jobs:
- name: Install dependencies
run: ./metadata-ingestion/scripts/install_deps.sh
- name: Install airflow package and test (extras ${{ matrix.extra_pip_requirements }})
run: ./gradlew -Pextra_pip_requirements='${{ matrix.extra_pip_requirements }}' -Pextra_pip_extras='${{ matrix.extra_pip_extras }}' :metadata-ingestion-modules:airflow-plugin:lint :metadata-ingestion-modules:airflow-plugin:testQuick
run: ./gradlew -Pextra_pip_requirements='${{ matrix.extra_pip_requirements }}' -Pextra_pip_extras='${{ matrix.extra_pip_extras }}' :metadata-ingestion-modules:airflow-plugin:build
- name: pip freeze show list installed
if: always()
run: source metadata-ingestion-modules/airflow-plugin/venv/bin/activate && pip freeze

35
metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_plugin.py
View File

@ -50,21 +50,28 @@ class DatahubPlugin(AirflowPlugin):
name = "datahub_plugin"
if _USE_AIRFLOW_LISTENER_INTERFACE:
if not NEEDS_AIRFLOW_LISTENER_MODULE:
from datahub_airflow_plugin.datahub_listener import ( # type: ignore[misc]
get_airflow_plugin_listener,
try:
if not NEEDS_AIRFLOW_LISTENER_MODULE:
from datahub_airflow_plugin.datahub_listener import ( # type: ignore[misc]
get_airflow_plugin_listener,
)
listeners: list = list(filter(None, [get_airflow_plugin_listener()]))
else:
# On Airflow < 2.5, we need the listener to be a module.
# This is just a quick shim layer to make that work.
#
# Related Airflow change: https://github.com/apache/airflow/pull/27113.
import datahub_airflow_plugin._datahub_listener_module as _listener_module # type: ignore[misc]
listeners = [_listener_module]
except Exception as e:
logger.warning(
f"Failed to load the DataHub plugin's event listener: {e}",
exc_info=True,
)
listeners: list = list(filter(None, [get_airflow_plugin_listener()]))
else:
# On Airflow < 2.5, we need the listener to be a module.
# This is just a quick shim layer to make that work.
#
# Related Airflow change: https://github.com/apache/airflow/pull/27113.
import datahub_airflow_plugin._datahub_listener_module as _listener_module # type: ignore[misc]
listeners = [_listener_module]
listeners = []
if not _USE_AIRFLOW_LISTENER_INTERFACE: