diff --git a/ingestion/src/metadata/ingestion/source/dashboard/looker/columns.py b/ingestion/src/metadata/ingestion/source/dashboard/looker/columns.py index e56f2fe341a..b74c47d6286 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/looker/columns.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/looker/columns.py @@ -11,12 +11,13 @@ """ Looker general utilities """ +from functools import singledispatch from typing import List, Sequence, Union, cast from looker_sdk.sdk.api40.models import LookmlModelExplore, LookmlModelExploreField from metadata.generated.schema.entity.data.table import Column, DataType -from metadata.ingestion.source.dashboard.looker.models import LookMlView +from metadata.ingestion.source.dashboard.looker.models import LookMlField, LookMlView # Some docs on types https://cloud.google.com/looker/docs/reference/param-dimension-filter-parameter-types LOOKER_TYPE_MAP = { @@ -94,13 +95,13 @@ def get_columns_from_model( Obtain the column (measures and dimensions) from the models """ columns = [] - all_fields = (model.fields.dimensions or []) + (model.fields.measures or []) + all_fields = get_model_fields(model) for field in cast(Sequence[LookmlModelExploreField], all_fields): type_ = LOOKER_TYPE_MAP.get(field.type, DataType.UNKNOWN) columns.append( Column( name=field.name, - displayName=getattr(field, "label_short", field.label), + displayName=getattr(field, "label_short", None) or field.label, dataType=type_, # We cannot get the inner type from the sdk of .lkml arrayDataType=DataType.UNKNOWN if type_ == DataType.ARRAY else None, @@ -110,3 +111,20 @@ def get_columns_from_model( ) return columns + + +@singledispatch +def get_model_fields( + model: Union[LookmlModelExplore, LookMlView] +) -> List[Union[LookmlModelExploreField, LookMlField]]: + raise NotImplementedError(f"Missing implementation for type {type(model)}") + + +@get_model_fields.register +def _(model: LookmlModelExplore) -> List[LookmlModelExploreField]: + return (model.fields.dimensions or []) + (model.fields.measures or []) + + +@get_model_fields.register +def _(model: LookMlView) -> List[LookMlField]: + return (model.dimensions or []) + (model.measures or []) diff --git a/ingestion/tests/unit/resources/lkml/recursive.explore.lkml b/ingestion/tests/unit/resources/lkml/recursive.explore.lkml new file mode 100644 index 00000000000..45ef887f522 --- /dev/null +++ b/ingestion/tests/unit/resources/lkml/recursive.explore.lkml @@ -0,0 +1,9 @@ +include: "views/recursive.view.lkml" + +view: recursive_explore { + + dimension: dim { + type: string + sql: ${TABLE}.name ;; + } +} diff --git a/ingestion/tests/unit/resources/lkml/views/recursive.view.lkml b/ingestion/tests/unit/resources/lkml/views/recursive.view.lkml new file mode 100644 index 00000000000..47eefc20dab --- /dev/null +++ b/ingestion/tests/unit/resources/lkml/views/recursive.view.lkml @@ -0,0 +1,9 @@ +include: "views/recursive_call.view.lkml" + +view: recursive { + + dimension: dim { + type: string + sql: ${TABLE}.name ;; + } +} diff --git a/ingestion/tests/unit/resources/lkml/views/recursive_call.view.lkml b/ingestion/tests/unit/resources/lkml/views/recursive_call.view.lkml new file mode 100644 index 00000000000..2e215d03ad8 --- /dev/null +++ b/ingestion/tests/unit/resources/lkml/views/recursive_call.view.lkml @@ -0,0 +1,9 @@ +include: "views/recursive.view.lkml" + +view: recursive_call { + + dimension: dim2 { + type: string + sql: ${TABLE}.name ;; + } +} diff --git a/ingestion/tests/unit/topology/dashboard/test_looker_lkml_parser.py b/ingestion/tests/unit/topology/dashboard/test_looker_lkml_parser.py index b0ddb5a7f2e..e60a9d0c88b 100644 --- a/ingestion/tests/unit/topology/dashboard/test_looker_lkml_parser.py +++ b/ingestion/tests/unit/topology/dashboard/test_looker_lkml_parser.py @@ -14,6 +14,14 @@ Test the lkml parser from pathlib import Path from unittest import TestCase +from looker_sdk.sdk.api40.models import ( + LookmlModelExplore, + LookmlModelExploreField, + LookmlModelExploreFieldset, +) + +from metadata.generated.schema.entity.data.table import Column, ColumnName, DataType +from metadata.ingestion.source.dashboard.looker.columns import get_columns_from_model from metadata.ingestion.source.dashboard.looker.links import get_path_from_link from metadata.ingestion.source.dashboard.looker.parser import ( Includes, @@ -138,6 +146,24 @@ class TestLkmlParser(TestCase): }, ) + def test_recursive_explore(self): + """ + We should stop the execution + """ + reader = LocalReader(BASE_PATH) + parser = LkmlParser(reader) + + view = parser.find_view( + view_name=ViewName("recursive_call"), + path=Includes("recursive.explore.lkml"), + ) + self.assertIsNotNone(view) + + view = parser.find_view( + view_name=ViewName("recursive"), path=Includes("recursive.explore.lkml") + ) + self.assertIsNotNone(view) + def test_get_path_from_link(self): """ Validate utility @@ -166,3 +192,90 @@ class TestLkmlParser(TestCase): parser = LkmlParser(reader) self.assertIn("cats.view.lkml", parser._expand(path)[0]) + + def test_explore_col_parser(self): + """ + We can parse a looker explore + """ + + explore = LookmlModelExplore( + name="test-explore", + fields=LookmlModelExploreFieldset( + dimensions=[ + LookmlModelExploreField( + name="dim1", + label="Dim 1 Label", + type="yesno", + description=None, + ), + LookmlModelExploreField( + name="dim2", + label_short="Dim 2 Label Short", + type="list", + description="something", + ), + ], + measures=[ + LookmlModelExploreField( + name="measure1", + type="duration_day", + ) + ], + ), + ) + + cols = get_columns_from_model(explore) + expected_cols = [ + Column( + name=ColumnName(__root__="dim1"), + displayName="Dim 1 Label", + dataType=DataType.BOOLEAN, + dataTypeDisplay="yesno", + description=None, + ), + Column( + name=ColumnName(__root__="dim2"), + displayName="Dim 2 Label Short", + dataType=DataType.ARRAY, + arrayDataType=DataType.UNKNOWN, + dataTypeDisplay="list", + description="something", + ), + Column( + name=ColumnName(__root__="measure1"), + displayName=None, + dataType=DataType.STRING, + dataTypeDisplay="duration_day", + description=None, + ), + ] + + self.assertEquals(cols, expected_cols) + + def test_view_col_parser(self): + """ + Test we can parse a view + """ + + reader = LocalReader(BASE_PATH) + parser = LkmlParser(reader) + + view = parser.find_view( + view_name=ViewName("cats"), path=Includes("kittens.explore.lkml") + ) + + cols = get_columns_from_model(view) + expected_cols = [ + Column( + name=ColumnName(__root__="name"), + dataType=DataType.STRING, + dataTypeDisplay="string", + ), + Column( + name=ColumnName(__root__="age"), + dataType=DataType.NUMBER, + dataTypeDisplay="int", + ), + ] + + self.assertEquals(cols, expected_cols)