yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-12-17 13:04:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ingest): make gms url configuration resilient in rest emitter (#10316)

Browse Source
This commit is contained in:
Aseem Bansal 2024-04-18 14:46:32 +05:30 committed by GitHub
parent 91e3dc829e
commit d3fb698d8d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 6 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/lineage/airflow.md
View File

@ -45,6 +45,8 @@ Set up a DataHub connection in Airflow, either via command line or the Airflow U
airflow connections add --conn-type 'datahub-rest' 'datahub_rest_default' --conn-host 'http://datahub-gms:8080' --conn-password '<optional datahub auth token>' airflow connections add --conn-type 'datahub-rest' 'datahub_rest_default' --conn-host 'http://datahub-gms:8080' --conn-password '<optional datahub auth token>'
``` ```
If you are using hosted Acryl Datahub then please use `https://YOUR_PREFIX.acryl.io/gms` as the `--conn-host` parameter.
#### Airflow UI #### Airflow UI
On the Airflow UI, go to Admin -> Connections and click the "+" symbol to create a new connection. Select "DataHub REST Server" from the dropdown for "Connection Type" and enter the appropriate values. On the Airflow UI, go to Admin -> Connections and click the "+" symbol to create a new connection. Select "DataHub REST Server" from the dropdown for "Connection Type" and enter the appropriate values.

4
metadata-ingestion-modules/airflow-plugin/tests/unit/test_airflow.py
View File

@ -41,13 +41,13 @@ lineage_mce = builder.make_lineage_mce(
datahub_rest_connection_config = Connection( datahub_rest_connection_config = Connection(
conn_id="datahub_rest_test", conn_id="datahub_rest_test",
conn_type="datahub_rest", conn_type="datahub_rest",
host="http://test_host:8080/", host="http://test_host:8080",
extra=None, extra=None,
) )
datahub_rest_connection_config_with_timeout = Connection( datahub_rest_connection_config_with_timeout = Connection(
conn_id="datahub_rest_test", conn_id="datahub_rest_test",
conn_type="datahub_rest", conn_type="datahub_rest",
host="http://test_host:8080/", host="http://test_host:8080",
extra=json.dumps({"timeout_sec": 5}), extra=json.dumps({"timeout_sec": 5}),
) )

4
metadata-ingestion/src/datahub/emitter/rest_emitter.py
View File

@ -10,7 +10,7 @@ from deprecated import deprecated
from requests.adapters import HTTPAdapter, Retry from requests.adapters import HTTPAdapter, Retry
from requests.exceptions import HTTPError, RequestException from requests.exceptions import HTTPError, RequestException
from datahub.cli.cli_utils import get_system_auth from datahub.cli.cli_utils import fixup_gms_url, get_system_auth
from datahub.configuration.common import ConfigurationError, OperationalError from datahub.configuration.common import ConfigurationError, OperationalError
from datahub.emitter.generic_emitter import Emitter from datahub.emitter.generic_emitter import Emitter
from datahub.emitter.mcp import MetadataChangeProposalWrapper from datahub.emitter.mcp import MetadataChangeProposalWrapper
@ -72,7 +72,7 @@ class DataHubRestEmitter(Closeable, Emitter):
): ):
if not gms_server: if not gms_server:
raise ConfigurationError("gms server is required") raise ConfigurationError("gms server is required")
self._gms_server = gms_server self._gms_server = fixup_gms_url(gms_server)
self._token = token self._token = token
self.server_config: Dict[str, Any] = {} self.server_config: Dict[str, Any] = {}