yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-08-29 19:46:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ingestion/looker): column name missing in explore (#10892)

Browse Source
This commit is contained in:
sid-acryl 2024-07-12 13:17:53 +05:30 committed by GitHub
parent 89bda5bdd9
commit aa92a99130
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 13 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
metadata-ingestion/src/datahub/ingestion/source/looker/looker_common.py
View File

@ -347,26 +347,20 @@ class ExploreUpstreamViewField:
model_name: str, model_name: str,
upstream_views_file_path: Dict[str, Optional[str]], upstream_views_file_path: Dict[str, Optional[str]],
config: LookerCommonConfig, config: LookerCommonConfig,
remove_variant: bool = False,
) -> Optional[ColumnRef]: ) -> Optional[ColumnRef]:
assert self.field.name is not None assert self.field.name is not None
if len(self.field.name.split(".")) != 2: if len(self.field.name.split(".")) != 2:
return None # Inconsistent info received return None # Inconsistent info received
assert self.explore.name view_name: Optional[str] = self.explore.name
if self.field.original_view is not None:
view_name: Optional[str] = ( view_name = self.field.original_view
self.explore.name
if self.field.original_view is not None
else self.field.original_view
)
field_name = self.field.name.split(".")[1] field_name = self.field.name.split(".")[1]
if ( if remove_variant and self.field.field_group_variant is not None:
self.field.field_group_variant is not None
and self.field.field_group_variant.lower() in field_name.lower()
):
# remove variant at the end. +1 for "_" # remove variant at the end. +1 for "_"
field_name = field_name[ field_name = field_name[
: -(len(self.field.field_group_variant.lower()) + 1) : -(len(self.field.field_group_variant.lower()) + 1)
@ -381,7 +375,7 @@ class ExploreUpstreamViewField:
file_path: Optional[str] = ( file_path: Optional[str] = (
upstream_views_file_path.get(view_name) upstream_views_file_path.get(view_name)
if upstream_views_file_path.get(view_name) is not None if upstream_views_file_path.get(view_name)
else ViewFieldValue.NOT_AVAILABLE.value else ViewFieldValue.NOT_AVAILABLE.value
) )
@ -413,7 +407,7 @@ class ExploreUpstreamViewField:
) -> Optional[ColumnRef]: ) -> Optional[ColumnRef]:
assert self.field.name is not None assert self.field.name is not None
if self.field.dimension_group is None: # It is not part of Dimensional Group if self.field.dimension_group is None or self.field.field_group_variant is None:
return self._form_field_name( return self._form_field_name(
view_project_map, view_project_map,
explore_project_name, explore_project_name,
@ -422,15 +416,6 @@ class ExploreUpstreamViewField:
config, config,
) )
if self.field.field_group_variant is None:
return self._form_field_name(
view_project_map,
explore_project_name,
model_name,
upstream_views_file_path,
config,
) # Variant i.e. Month, Day, Year ... is not available
if self.field.type is None or not self.field.type.startswith("date_"): if self.field.type is None or not self.field.type.startswith("date_"):
return self._form_field_name( return self._form_field_name(
view_project_map, view_project_map,
@ -456,6 +441,7 @@ class ExploreUpstreamViewField:
model_name, model_name,
upstream_views_file_path, upstream_views_file_path,
config, config,
remove_variant=True,
) )
@ -999,12 +985,15 @@ class LookerExplore:
view_fields: List[ViewField] = [] view_fields: List[ViewField] = []
field_name_vs_raw_explore_field: Dict = {} field_name_vs_raw_explore_field: Dict = {}
if explore.fields is not None: if explore.fields is not None:
if explore.fields.dimensions is not None: if explore.fields.dimensions is not None:
for dim_field in explore.fields.dimensions: for dim_field in explore.fields.dimensions:
if dim_field.name is None: if dim_field.name is None:
continue continue
else: else:
field_name_vs_raw_explore_field[dim_field.name] = dim_field field_name_vs_raw_explore_field[dim_field.name] = dim_field
view_fields.append( view_fields.append(
@ -1045,6 +1034,7 @@ class LookerExplore:
if measure_field.name is None: if measure_field.name is None:
continue continue
else: else:
field_name_vs_raw_explore_field[ field_name_vs_raw_explore_field[
measure_field.name measure_field.name
] = measure_field ] = measure_field

2
metadata-ingestion/src/datahub/ingestion/source/looker/lookml_config.py
View File

@ -121,7 +121,7 @@ class LookMLSourceConfig(
AllowDenyPattern.allow_all(), AllowDenyPattern.allow_all(),
description="List of regex patterns for LookML views to include in the extraction.", description="List of regex patterns for LookML views to include in the extraction.",
) )
parse_table_names_from_sql: bool = Field(False, description="See note below.") parse_table_names_from_sql: bool = Field(True, description="See note below.")
sql_parser: str = Field( sql_parser: str = Field(
"datahub.utilities.sql_parser.DefaultSQLParser", description="See note below." "datahub.utilities.sql_parser.DefaultSQLParser", description="See note below."
) )