yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-12-14 03:26:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(ingest): looker - optional external_base_url config for chart and dashboard urls (#3985)

Browse Source 
Co-authored-by: Shirshanka Das <shirshanka@apache.org>
This commit is contained in:
John Joyce 2022-01-26 15:06:34 -08:00 committed by GitHub
parent aeabeec133
commit cf38b61a7f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 12 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
metadata-ingestion/source_docs/looker.md
View File

@ -72,7 +72,8 @@ Note that a `.` is used to denote nested fields in the YAML recipe.
| ------------------------- | -------- | ----------------------- | ------------------------------------------------------------------------------------------------------------ | | ------------------------- | -------- | ----------------------- | ------------------------------------------------------------------------------------------------------------ |
| `client_id` | ✅ | | Looker API3 client ID. | | `client_id` | ✅ | | Looker API3 client ID. |
| `client_secret` | ✅ | | Looker API3 client secret. | | `client_secret` | ✅ | | Looker API3 client secret. |
| `base_url` | ✅ | | Url to your Looker instance: `https://company.looker.com:19999` or `https://looker.company.com`, or similar. | | `base_url` | ✅ | | Url to your Looker instance: `https://company.looker.com:19999` or `https://looker.company.com`, or similar. Used for making API calls to Looker and constructing clickable dashboard and chart urls. |
| `external_base_url` | | value of `base_url` | Optional URL to use when constructing external URLs to Looker if the `base_url` is not the correct one to use. For example, `https://looker-public.company.com`. If not provided, the external base URL will default to `base_url`. |
| `platform_name` | | `"looker"` | Platform to use in namespace when constructing URNs. | | `platform_name` | | `"looker"` | Platform to use in namespace when constructing URNs. |
| `extract_owners` | | `True` | When enabled, extracts ownership from Looker directly. When disabled, ownership is left empty for dashboards and charts. | | `extract_owners` | | `True` | When enabled, extracts ownership from Looker directly. When disabled, ownership is left empty for dashboards and charts. |
| `strip_user_ids_from_email` | | `False` | When enabled, converts Looker user emails of the form name@domain.com to urn:li:corpuser:name when assigning ownership | | `strip_user_ids_from_email` | | `False` | When enabled, converts Looker user emails of the form name@domain.com to urn:li:corpuser:name when assigning ownership |

13
metadata-ingestion/src/datahub/ingestion/source/looker.py
View File

@ -29,6 +29,7 @@ from looker_sdk.sdk.api31.models import (
Query, Query,
User, User,
) )
from pydantic import validator
import datahub.emitter.mce_builder as builder import datahub.emitter.mce_builder as builder
from datahub.configuration import ConfigModel from datahub.configuration import ConfigModel
@ -102,6 +103,13 @@ class LookerDashboardSourceConfig(LookerAPIConfig, LookerCommonConfig):
strip_user_ids_from_email: bool = False strip_user_ids_from_email: bool = False
skip_personal_folders: bool = False skip_personal_folders: bool = False
max_threads: int = os.cpu_count() or 40 max_threads: int = os.cpu_count() or 40
external_base_url: Optional[str]
@validator("external_base_url", pre=True, always=True)
def external_url_defaults_to_api_config_base_url(
cls, v: Optional[str], *, values: Dict[str, Any], **kwargs: Dict[str, Any]
) -> str:
return v or values["base_url"]
@dataclass @dataclass
@ -526,13 +534,12 @@ class LookerDashboardSource(Source):
) )
chart_type = self._get_chart_type(dashboard_element) chart_type = self._get_chart_type(dashboard_element)
chart_info = ChartInfoClass( chart_info = ChartInfoClass(
type=chart_type, type=chart_type,
description=dashboard_element.description or "", description=dashboard_element.description or "",
title=dashboard_element.title or "", title=dashboard_element.title or "",
lastModified=ChangeAuditStamps(), lastModified=ChangeAuditStamps(),
chartUrl=dashboard_element.url(self.source_config.base_url), chartUrl=dashboard_element.url(self.source_config.external_base_url or ""),
inputs=dashboard_element.get_view_urns(self.source_config), inputs=dashboard_element.get_view_urns(self.source_config),
customProperties={ customProperties={
"upstream_fields": ",".join( "upstream_fields": ",".join(
@ -618,7 +625,7 @@ class LookerDashboardSource(Source):
title=looker_dashboard.title, title=looker_dashboard.title,
charts=[mce.proposedSnapshot.urn for mce in chart_mces], charts=[mce.proposedSnapshot.urn for mce in chart_mces],
lastModified=ChangeAuditStamps(), lastModified=ChangeAuditStamps(),
dashboardUrl=looker_dashboard.url(self.source_config.base_url), dashboardUrl=looker_dashboard.url(self.source_config.external_base_url),
) )
dashboard_snapshot.aspects.append(dashboard_info) dashboard_snapshot.aspects.append(dashboard_info)