Fixed filterpattern and lineage (#8802)

ingestion/src/metadata/ingestion/source/dashboard/redash.py

@@ -19,8 +19,9 @@ from metadata.generated.schema.api.data.createChart import CreateChartRequest
 from metadata.generated.schema.api.data.createDashboard import CreateDashboardRequest
 from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
 from metadata.generated.schema.entity.data.dashboard import (
-    Dashboard as Lineage_Dashboard,
+    Dashboard as LineageDashboard,
 )
+from metadata.generated.schema.entity.data.table import Table
 from metadata.generated.schema.entity.services.connections.dashboard.redashConnection import (
     RedashConnection,
 )
@@ -32,12 +33,10 @@ from metadata.generated.schema.metadataIngestion.workflow import (
 )
 from metadata.generated.schema.type.entityReference import EntityReference
 from metadata.ingestion.api.source import InvalidSourceException
-from metadata.ingestion.lineage.sql_lineage import (
+from metadata.ingestion.lineage.sql_lineage import clean_raw_query
-    clean_raw_query,
-    search_table_entities,
-)
 from metadata.ingestion.source.dashboard.dashboard_service import DashboardServiceSource
 from metadata.utils import fqn
+from metadata.utils.filters import filter_by_chart
 from metadata.utils.helpers import get_standard_chart_type
 from metadata.utils.logger import ingestion_logger
 
@@ -80,7 +79,7 @@ class RedashSource(DashboardServiceSource):
         """
         Get Dashboard Name
         """
-        return dashboard_details["id"]
+        return dashboard_details["name"]
 
     def get_dashboard_details(self, dashboard: dict) -> dict:
         """
@@ -121,34 +120,39 @@ class RedashSource(DashboardServiceSource):
         In redash we do not get table, database_schema or database name but we do get query
         the lineage is being generated based on the query
         """
+
+        to_fqn = fqn.build(
+            self.metadata,
+            entity_type=LineageDashboard,
+            service_name=self.config.serviceName,
+            dashboard_name=str(dashboard_details.get("id")),
+        )
+        to_entity = self.metadata.get_by_name(
+            entity=LineageDashboard,
+            fqn=to_fqn,
+        )
         for widgets in dashboard_details.get("widgets", []):
             visualization = widgets.get("visualization")
-            if not visualization.get("query"):
+            if not visualization:
                 continue
             if visualization.get("query", {}).get("query"):
                 parser = LineageRunner(clean_raw_query(visualization["query"]["query"]))
             for table in parser.source_tables:
                 table_name = str(table)
-                database_schema = None
+                database_schema_table = fqn.split_table_name(table_name)
-                if "." in table:
+                from_fqn = fqn.build(
-                    database_schema, table = fqn.split(table_name)[-2:]
-                table_entities = search_table_entities(
-                    metadata=self.metadata,
-                    database=None,
-                    service_name=db_service_name,
-                    database_schema=database_schema,
-                    table=table_name,
-                )
-                for from_entity in table_entities:
-                    to_entity = self.metadata.get_by_name(
-                        entity=Lineage_Dashboard,
-                        fqn=fqn.build(
                     self.metadata,
-                            Lineage_Dashboard,
+                    entity_type=Table,
-                            service_name=self.config.serviceName,
+                    service_name=db_service_name,
-                            dashboard_name=str(dashboard_details.get("id")),
+                    schema_name=database_schema_table.get("database_schema"),
-                        ),
+                    table_name=database_schema_table.get("table"),
+                    database_name=database_schema_table.get("database"),
                 )
+                from_entity = self.metadata.get_by_name(
+                    entity=Table,
+                    fqn=from_fqn,
+                )
+                if from_entity and to_entity:
                     yield self._get_add_lineage_request(
                         to_entity=to_entity, from_entity=from_entity
                     )
@@ -161,9 +165,17 @@ class RedashSource(DashboardServiceSource):
         """
         for widgets in dashboard_details.get("widgets", []):
             visualization = widgets.get("visualization")
+            chart_display_name = str(
+                visualization["query"]["name"] if visualization else widgets["id"]
+            )
+            if filter_by_chart(
+                self.source_config.chartFilterPattern, chart_display_name
+            ):
+                self.status.filter(chart_display_name, "Chart Pattern not allowed")
+                continue
             yield CreateChartRequest(
                 name=widgets["id"],
-                displayName=visualization["query"]["name"]
+                displayName=chart_display_name
                 if visualization and visualization["query"]
                 else "",
                 chartType=get_standard_chart_type(