Fix 17911: Looker parsing improvements for liquid templating and view/model aliasing (#17912)

* Looker parsing improvements for liquid templating and view/model aliasing

* add python-liquid dependency to looker plugin requirements

* move to static method with 'openmetadata' context and add rendering tests

* remove backtick stripping

---------

Co-authored-by: Imri Paran <imri.paran@gmail.com>

ingestion/setup.py

@@ -258,6 +258,7 @@ plugins: Dict[str, Set[str]] = {
         VERSIONS["lkml"],
         "gitpython~=3.1.34",
         VERSIONS["giturlparse"],
+        "python-liquid",
     },
     "mlflow": {"mlflow-skinny>=2.3.0"},
     "mongo": {VERSIONS["mongo"], VERSIONS["pandas"], VERSIONS["numpy"]},

ingestion/src/metadata/ingestion/source/dashboard/looker/metadata.py

@@ -28,6 +28,7 @@ from typing import Dict, Iterable, List, Optional, Sequence, Set, Type, Union, c
 
 import giturlparse
 import lkml
+from liquid import Template
 from looker_sdk.sdk.api40.methods import Looker40SDK
 from looker_sdk.sdk.api40.models import Dashboard as LookerDashboard
 from looker_sdk.sdk.api40.models import (
@@ -450,11 +451,11 @@ class LookerSource(DashboardServiceSource):
                                 view.name, "Data model (View) filtered out."
                             )
                             continue
-
+                        view_name = view.from_ if view.from_ else view.name
                         yield from self._process_view(
-                            view_name=ViewName(view.name), explore=model
+                            view_name=ViewName(view_name), explore=model
                         )
-                    if len(model.joins) == 0 and model.sql_table_name:
+                    if model.view_name:
                         yield from self._process_view(
                             view_name=ViewName(model.view_name), explore=model
                         )
@@ -570,7 +571,8 @@ class LookerSource(DashboardServiceSource):
             db_service_names = self.get_db_service_names()
 
             if view.sql_table_name:
-                source_table_name = self._clean_table_name(view.sql_table_name)
+                sql_table_name = self._render_table_name(view.sql_table_name)
+                source_table_name = self._clean_table_name(sql_table_name)
 
                 # View to the source is only there if we are informing the dbServiceNames
                 for db_service_name in db_service_names or []:
@@ -726,6 +728,33 @@ class LookerSource(DashboardServiceSource):
 
         return table_name.lower().split(" as ")[0].strip()
 
+    @staticmethod
+    def _render_table_name(table_name: str) -> str:
+        """
+        sql_table_names might contain Liquid templates
+        when defining an explore. e.g,:
+        sql_table_name:
+            {% if openmetadata %}
+                event
+            {% elsif event.created_week._in_query %}
+                event_by_week
+            {% else %}
+                event
+            {% endif %} ;;
+        we should render the template and give the option
+        to render a specific value during metadata ingestion
+        using the "openmetadata" context argument
+        :param table_name: table name with possible templating
+        :return: rendered table name
+        """
+        try:
+            context = {"openmetadata": True}
+            template = Template(table_name)
+            sql_table_name = template.render(context)
+        except Exception:
+            sql_table_name = table_name
+        return sql_table_name
+
     @staticmethod
     def get_dashboard_sources(dashboard_details: LookerDashboard) -> Set[str]:
         """

ingestion/tests/unit/topology/dashboard/test_looker.py

@@ -300,6 +300,53 @@ class LookerUnitTest(TestCase):
 
         self.assertEqual(self.looker._clean_table_name("TABLE AS ALIAS"), "table")
 
+    def test_render_table_name(self):
+        """
+        Check that table is rendered correctly if "openmetadata" or default condition apply, or no templating is present
+        """
+        tagged_table_name_template = """
+        {%- if openmetadata -%}
+        `BQ-project.dataset.sample_data`
+        {%- elsif prod -%}
+        `BQ-project.dataset.sample_data`
+        {%- elsif dev -%}
+        `BQ-project.{{_user_attributes['dbt_dev_schema']}}.sample_data`
+        {%- endif -%}
+        """
+        default_table_name_template = """
+        {%- if prod -%}
+        `BQ-project.dataset.sample_data`
+        {%- elsif dev -%}
+        `BQ-project.{{_user_attributes['dbt_dev_schema']}}.sample_data`
+        {%- else -%}
+        `BQ-project.dataset.sample_data`
+        {%- endif -%}
+        """
+        untagged_table_name_template = """
+        {%- if prod -%}
+        `BQ-project.dataset.sample_data`
+        {%- elsif dev -%}
+        `BQ-project.{{_user_attributes['dbt_dev_schema']}}.sample_data`
+        {%- endif -%}
+        """
+        table_name_plain = "`BQ-project.dataset.sample_data`"
+        self.assertEqual(
+            self.looker._render_table_name(tagged_table_name_template),
+            "`BQ-project.dataset.sample_data`",
+        )
+        self.assertEqual(
+            self.looker._render_table_name(default_table_name_template),
+            "`BQ-project.dataset.sample_data`",
+        )
+        self.assertNotEqual(
+            self.looker._render_table_name(untagged_table_name_template),
+            "`BQ-project.dataset.sample_data`",
+        )
+        self.assertEqual(
+            self.looker._render_table_name(table_name_plain),
+            "`BQ-project.dataset.sample_data`",
+        )
+
     def test_get_dashboard_sources(self):
         """
         Check how we are building the sources