yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-11-08 06:53:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix: looker CLL errors (#21493)

Browse Source 
* Fix: looker CLL errors

* Addressed comments- added exception handling

* addressed comments

* linting
This commit is contained in:
Suman Maharana 2025-05-31 17:29:23 +05:30 committed by GitHub
parent b5f47e3309
commit 1c4500b119
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 55 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
ingestion/src/metadata/ingestion/source/dashboard/looker/metadata.py
View File

@ -649,13 +649,14 @@ class LookerSource(DashboardServiceSource):
Extract column level lineage from a LookML view. Extract column level lineage from a LookML view.
Returns a list of tuples containing (source_column, target_column) Returns a list of tuples containing (source_column, target_column)
""" """
logger.debug(f"Extracting column lineage for view: {view.name}")
column_lineage = [] column_lineage = []
try: try:
# Build a map: field_name → sql block # Build a map: field_name → sql block
field_sql_map = {} field_sql_map = {}
for field_type in ["dimensions", "measures", "dimension_groups"]: for field_type in ["dimensions", "measures", "dimension_groups"]:
for field in getattr(view, field_type, []): for field in getattr(view, field_type, []):
if hasattr(field, "sql"): if hasattr(field, "sql") and field.sql is not None:
field_sql_map[field.name] = field.sql field_sql_map[field.name] = field.sql
# Regex to extract ${TABLE}.col and ${field} # Regex to extract ${TABLE}.col and ${field}
@ -682,10 +683,17 @@ class LookerSource(DashboardServiceSource):
return source_cols return source_cols
# Build lineage for each field # Build lineage for each field
for field_name, _ in field_sql_map.items(): for field_name, sql in field_sql_map.items():
try:
if not sql: # Skip if sql is None or empty
continue
source_cols = resolve(field_name) source_cols = resolve(field_name)
for source_col in source_cols: for source_col in source_cols:
column_lineage.append((source_col, field_name)) column_lineage.append((source_col, field_name))
except Exception as err:
logger.warning(f"Error processing field {field_name}: {err}")
logger.debug(traceback.format_exc())
continue
return column_lineage return column_lineage
except Exception as e: except Exception as e:
@ -707,6 +715,9 @@ class LookerSource(DashboardServiceSource):
# TODO: column-level lineage parsing the explore columns with the format `view_name.col` # TODO: column-level lineage parsing the explore columns with the format `view_name.col`
# Now the context has the newly created view # Now the context has the newly created view
if explore_model: if explore_model:
logger.debug(
f"Building lineage request for view {self._view_data_model.name} to explore {explore_model.name}"
)
yield self._get_add_lineage_request( yield self._get_add_lineage_request(
from_entity=self._view_data_model, to_entity=explore_model from_entity=self._view_data_model, to_entity=explore_model
) )
@ -1066,6 +1077,8 @@ class LookerSource(DashboardServiceSource):
db_service_name: name of the service from the config db_service_name: name of the service from the config
to_entity: Dashboard Entity being used to_entity: Dashboard Entity being used
""" """
# pylint: disable=too-many-locals
logger.debug(f"Building lineage request for {source} to {to_entity.name}")
source_elements = fqn.split_table_name(table_name=source) source_elements = fqn.split_table_name(table_name=source)
@ -1086,7 +1099,24 @@ class LookerSource(DashboardServiceSource):
if column_lineage: if column_lineage:
processed_column_lineage = [] processed_column_lineage = []
for source_col, target_col in column_lineage: for column_tuple in column_lineage or []:
try:
if len(column_tuple) < 2:
logger.debug(
f"Skipping invalid column tuple: {column_tuple}"
)
continue
source_col = column_tuple[0]
target_col = column_tuple[-1]
if not source_col or not target_col:
logger.debug(
f"Skipping column tuple with empty values: source={source_col}, "
f"target={target_col}, to_entity={to_entity.name}"
)
continue
from_column = get_column_fqn( from_column = get_column_fqn(
table_entity=from_entity, column=str(target_col) table_entity=from_entity, column=str(target_col)
) )
@ -1096,8 +1126,17 @@ class LookerSource(DashboardServiceSource):
) )
if from_column and to_column: if from_column and to_column:
processed_column_lineage.append( processed_column_lineage.append(
ColumnLineage(fromColumns=[from_column], toColumn=to_column) ColumnLineage(
fromColumns=[from_column], toColumn=to_column
) )
)
except Exception as err:
logger.warning(
f"Error processing column lineage {column_tuple}: {err}"
)
logger.debug(traceback.format_exc())
continue
column_lineage = processed_column_lineage column_lineage = processed_column_lineage
if from_entity: if from_entity: