Fix looker view parser (#12254)

ingestion/src/metadata/ingestion/source/dashboard/looker/columns.py

@@ -11,12 +11,13 @@
 """
 Looker general utilities
 """
+from functools import singledispatch
 from typing import List, Sequence, Union, cast
 
 from looker_sdk.sdk.api40.models import LookmlModelExplore, LookmlModelExploreField
 
 from metadata.generated.schema.entity.data.table import Column, DataType
-from metadata.ingestion.source.dashboard.looker.models import LookMlView
+from metadata.ingestion.source.dashboard.looker.models import LookMlField, LookMlView
 
 # Some docs on types https://cloud.google.com/looker/docs/reference/param-dimension-filter-parameter-types
 LOOKER_TYPE_MAP = {
@@ -94,13 +95,13 @@ def get_columns_from_model(
     Obtain the column (measures and dimensions) from the models
     """
     columns = []
-    all_fields = (model.fields.dimensions or []) + (model.fields.measures or [])
+    all_fields = get_model_fields(model)
     for field in cast(Sequence[LookmlModelExploreField], all_fields):
         type_ = LOOKER_TYPE_MAP.get(field.type, DataType.UNKNOWN)
         columns.append(
             Column(
                 name=field.name,
-                displayName=getattr(field, "label_short", field.label),
+                displayName=getattr(field, "label_short", None) or field.label,
                 dataType=type_,
                 # We cannot get the inner type from the sdk of .lkml
                 arrayDataType=DataType.UNKNOWN if type_ == DataType.ARRAY else None,
@@ -110,3 +111,20 @@ def get_columns_from_model(
         )
 
     return columns
+
+
+@singledispatch
+def get_model_fields(
+    model: Union[LookmlModelExplore, LookMlView]
+) -> List[Union[LookmlModelExploreField, LookMlField]]:
+    raise NotImplementedError(f"Missing implementation for type {type(model)}")
+
+
+@get_model_fields.register
+def _(model: LookmlModelExplore) -> List[LookmlModelExploreField]:
+    return (model.fields.dimensions or []) + (model.fields.measures or [])
+
+
+@get_model_fields.register
+def _(model: LookMlView) -> List[LookMlField]:
+    return (model.dimensions or []) + (model.measures or [])

ingestion/tests/unit/resources/lkml/recursive.explore.lkml

@@ -0,0 +1,9 @@
+include: "views/recursive.view.lkml"
+
+view: recursive_explore {
+
+  dimension: dim {
+    type: string
+    sql: ${TABLE}.name ;;
+  }
+}

ingestion/tests/unit/resources/lkml/views/recursive.view.lkml

@@ -0,0 +1,9 @@
+include: "views/recursive_call.view.lkml"
+
+view: recursive {
+
+  dimension: dim {
+    type: string
+    sql: ${TABLE}.name ;;
+  }
+}

ingestion/tests/unit/resources/lkml/views/recursive_call.view.lkml

@@ -0,0 +1,9 @@
+include: "views/recursive.view.lkml"
+
+view: recursive_call {
+
+  dimension: dim2 {
+    type: string
+    sql: ${TABLE}.name ;;
+  }
+}

ingestion/tests/unit/topology/dashboard/test_looker_lkml_parser.py

@@ -14,6 +14,14 @@ Test the lkml parser
 from pathlib import Path
 from unittest import TestCase
 
+from looker_sdk.sdk.api40.models import (
+    LookmlModelExplore,
+    LookmlModelExploreField,
+    LookmlModelExploreFieldset,
+)
+
+from metadata.generated.schema.entity.data.table import Column, ColumnName, DataType
+from metadata.ingestion.source.dashboard.looker.columns import get_columns_from_model
 from metadata.ingestion.source.dashboard.looker.links import get_path_from_link
 from metadata.ingestion.source.dashboard.looker.parser import (
     Includes,
@@ -138,6 +146,24 @@ class TestLkmlParser(TestCase):
             },
         )
 
+    def test_recursive_explore(self):
+        """
+        We should stop the execution
+        """
+        reader = LocalReader(BASE_PATH)
+        parser = LkmlParser(reader)
+
+        view = parser.find_view(
+            view_name=ViewName("recursive_call"),
+            path=Includes("recursive.explore.lkml"),
+        )
+        self.assertIsNotNone(view)
+
+        view = parser.find_view(
+            view_name=ViewName("recursive"), path=Includes("recursive.explore.lkml")
+        )
+        self.assertIsNotNone(view)
+
     def test_get_path_from_link(self):
         """
         Validate utility
@@ -166,3 +192,90 @@ class TestLkmlParser(TestCase):
         parser = LkmlParser(reader)
 
         self.assertIn("cats.view.lkml", parser._expand(path)[0])
+
+    def test_explore_col_parser(self):
+        """
+        We can parse a looker explore
+        """
+
+        explore = LookmlModelExplore(
+            name="test-explore",
+            fields=LookmlModelExploreFieldset(
+                dimensions=[
+                    LookmlModelExploreField(
+                        name="dim1",
+                        label="Dim 1 Label",
+                        type="yesno",
+                        description=None,
+                    ),
+                    LookmlModelExploreField(
+                        name="dim2",
+                        label_short="Dim 2 Label Short",
+                        type="list",
+                        description="something",
+                    ),
+                ],
+                measures=[
+                    LookmlModelExploreField(
+                        name="measure1",
+                        type="duration_day",
+                    )
+                ],
+            ),
+        )
+
+        cols = get_columns_from_model(explore)
+        expected_cols = [
+            Column(
+                name=ColumnName(__root__="dim1"),
+                displayName="Dim 1 Label",
+                dataType=DataType.BOOLEAN,
+                dataTypeDisplay="yesno",
+                description=None,
+            ),
+            Column(
+                name=ColumnName(__root__="dim2"),
+                displayName="Dim 2 Label Short",
+                dataType=DataType.ARRAY,
+                arrayDataType=DataType.UNKNOWN,
+                dataTypeDisplay="list",
+                description="something",
+            ),
+            Column(
+                name=ColumnName(__root__="measure1"),
+                displayName=None,
+                dataType=DataType.STRING,
+                dataTypeDisplay="duration_day",
+                description=None,
+            ),
+        ]
+
+        self.assertEquals(cols, expected_cols)
+
+    def test_view_col_parser(self):
+        """
+        Test we can parse a view
+        """
+
+        reader = LocalReader(BASE_PATH)
+        parser = LkmlParser(reader)
+
+        view = parser.find_view(
+            view_name=ViewName("cats"), path=Includes("kittens.explore.lkml")
+        )
+
+        cols = get_columns_from_model(view)
+        expected_cols = [
+            Column(
+                name=ColumnName(__root__="name"),
+                dataType=DataType.STRING,
+                dataTypeDisplay="string",
+            ),
+            Column(
+                name=ColumnName(__root__="age"),
+                dataType=DataType.NUMBER,
+                dataTypeDisplay="int",
+            ),
+        ]
+
+        self.assertEquals(cols, expected_cols)