diff --git a/.github/workflows/airflow-plugin.yml b/.github/workflows/airflow-plugin.yml index 68074e626b..21fd547114 100644 --- a/.github/workflows/airflow-plugin.yml +++ b/.github/workflows/airflow-plugin.yml @@ -67,7 +67,7 @@ jobs: - name: Install dependencies run: ./metadata-ingestion/scripts/install_deps.sh - name: Install airflow package and test (extras ${{ matrix.extra_pip_requirements }}) - run: ./gradlew -Pextra_pip_requirements='${{ matrix.extra_pip_requirements }}' -Pextra_pip_extras='${{ matrix.extra_pip_extras }}' :metadata-ingestion-modules:airflow-plugin:lint :metadata-ingestion-modules:airflow-plugin:testQuick + run: ./gradlew -Pextra_pip_requirements='${{ matrix.extra_pip_requirements }}' -Pextra_pip_extras='${{ matrix.extra_pip_extras }}' :metadata-ingestion-modules:airflow-plugin:build - name: pip freeze show list installed if: always() run: source metadata-ingestion-modules/airflow-plugin/venv/bin/activate && pip freeze diff --git a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_plugin.py b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_plugin.py index 2b0b751bd7..437c42713e 100644 --- a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_plugin.py +++ b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_plugin.py @@ -50,21 +50,28 @@ class DatahubPlugin(AirflowPlugin): name = "datahub_plugin" if _USE_AIRFLOW_LISTENER_INTERFACE: - if not NEEDS_AIRFLOW_LISTENER_MODULE: - from datahub_airflow_plugin.datahub_listener import ( # type: ignore[misc] - get_airflow_plugin_listener, + try: + if not NEEDS_AIRFLOW_LISTENER_MODULE: + from datahub_airflow_plugin.datahub_listener import ( # type: ignore[misc] + get_airflow_plugin_listener, + ) + + listeners: list = list(filter(None, [get_airflow_plugin_listener()])) + + else: + # On Airflow < 2.5, we need the listener to be a module. + # This is just a quick shim layer to make that work. + # + # Related Airflow change: https://github.com/apache/airflow/pull/27113. + import datahub_airflow_plugin._datahub_listener_module as _listener_module # type: ignore[misc] + + listeners = [_listener_module] + except Exception as e: + logger.warning( + f"Failed to load the DataHub plugin's event listener: {e}", + exc_info=True, ) - - listeners: list = list(filter(None, [get_airflow_plugin_listener()])) - - else: - # On Airflow < 2.5, we need the listener to be a module. - # This is just a quick shim layer to make that work. - # - # Related Airflow change: https://github.com/apache/airflow/pull/27113. - import datahub_airflow_plugin._datahub_listener_module as _listener_module # type: ignore[misc] - - listeners = [_listener_module] + listeners = [] if not _USE_AIRFLOW_LISTENER_INTERFACE: