yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-11-04 12:51:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ingest/looker): add user stats to report (#9505)

Browse Source
This commit is contained in:
Harshal Sheth 2024-01-03 14:28:32 -05:00 committed by GitHub
parent c9613043c8
commit 83b904e379
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 10 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
metadata-ingestion/src/datahub/ingestion/source/looker/looker_common.py
View File

@ -1059,6 +1059,7 @@ class LookerDashboardSourceReport(StaleEntityRemovalSourceReport):
dashboards_scanned_for_usage: int = 0 dashboards_scanned_for_usage: int = 0
charts_scanned_for_usage: int = 0 charts_scanned_for_usage: int = 0
charts_with_activity: LossySet[str] = dataclasses_field(default_factory=LossySet) charts_with_activity: LossySet[str] = dataclasses_field(default_factory=LossySet)
accessed_dashboards: int = 0
dashboards_with_activity: LossySet[str] = dataclasses_field( dashboards_with_activity: LossySet[str] = dataclasses_field(
default_factory=LossySet default_factory=LossySet
) )
@ -1066,6 +1067,10 @@ class LookerDashboardSourceReport(StaleEntityRemovalSourceReport):
_looker_explore_registry: Optional[LookerExploreRegistry] = None _looker_explore_registry: Optional[LookerExploreRegistry] = None
total_explores: int = 0 total_explores: int = 0
explores_scanned: int = 0 explores_scanned: int = 0
resolved_user_ids: int = 0
email_ids_missing: int = 0 # resolved users with missing email addresses
_looker_api: Optional[LookerAPI] = None _looker_api: Optional[LookerAPI] = None
query_latency: Dict[str, datetime.timedelta] = dataclasses_field( query_latency: Dict[str, datetime.timedelta] = dataclasses_field(
default_factory=dict default_factory=dict

5
metadata-ingestion/src/datahub/ingestion/source/looker/looker_config.py
View File

@ -160,11 +160,6 @@ class LookerDashboardSourceConfig(
description="When enabled, extracts ownership from Looker directly. When disabled, ownership is left empty " description="When enabled, extracts ownership from Looker directly. When disabled, ownership is left empty "
"for dashboards and charts.", "for dashboards and charts.",
) )
actor: Optional[str] = Field(
None,
description="This config is deprecated in favor of `extract_owners`. Previously, was the actor to use in "
"ownership properties of ingested metadata.",
)
strip_user_ids_from_email: bool = Field( strip_user_ids_from_email: bool = Field(
False, False,
description="When enabled, converts Looker user emails of the form name@domain.com to urn:li:corpuser:name " description="When enabled, converts Looker user emails of the form name@domain.com to urn:li:corpuser:name "

13
metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py
View File

@ -129,9 +129,6 @@ class LookerDashboardSource(TestableSource, StatefulIngestionSourceBase):
source_config: LookerDashboardSourceConfig source_config: LookerDashboardSourceConfig
reporter: LookerDashboardSourceReport reporter: LookerDashboardSourceReport
user_registry: LookerUserRegistry user_registry: LookerUserRegistry
accessed_dashboards: int = 0
resolved_user_ids: int = 0
email_ids_missing: int = 0 # resolved users with missing email addresses
reachable_look_registry: Set[ reachable_look_registry: Set[
str str
] # Keep track of look-id which are reachable from Dashboard ] # Keep track of look-id which are reachable from Dashboard
@ -866,7 +863,7 @@ class LookerDashboardSource(TestableSource, StatefulIngestionSourceBase):
def _get_looker_dashboard( def _get_looker_dashboard(
self, dashboard: Dashboard, client: LookerAPI self, dashboard: Dashboard, client: LookerAPI
) -> LookerDashboard: ) -> LookerDashboard:
self.accessed_dashboards += 1 self.reporter.accessed_dashboards += 1
if dashboard.folder is None: if dashboard.folder is None:
logger.debug(f"{dashboard.id} has no folder") logger.debug(f"{dashboard.id} has no folder")
dashboard_folder_path = None dashboard_folder_path = None
@ -928,9 +925,9 @@ class LookerDashboardSource(TestableSource, StatefulIngestionSourceBase):
if user is not None and self.source_config.extract_owners: if user is not None and self.source_config.extract_owners:
# Keep track of how many user ids we were able to resolve # Keep track of how many user ids we were able to resolve
self.resolved_user_ids += 1 self.reporter.resolved_user_ids += 1
if user.email is None: if user.email is None:
self.email_ids_missing += 1 self.reporter.email_ids_missing += 1
return user return user
@ -1313,8 +1310,8 @@ class LookerDashboardSource(TestableSource, StatefulIngestionSourceBase):
if ( if (
self.source_config.extract_owners self.source_config.extract_owners
and self.resolved_user_ids > 0 and self.reporter.resolved_user_ids > 0
and self.email_ids_missing == self.resolved_user_ids and self.reporter.email_ids_missing == self.reporter.resolved_user_ids
): ):
# Looks like we tried to extract owners and could not find their email addresses. This is likely a permissions issue # Looks like we tried to extract owners and could not find their email addresses. This is likely a permissions issue
self.reporter.report_warning( self.reporter.report_warning(