Fix: Add Redash E2E test (#11091)

.github/workflows/py-cli-e2e-tests.yml

@@ -21,7 +21,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        e2e-test: ['python', 'mysql', 'bigquery', 'snowflake', 'dbt_redshift', 'mssql', 'vertica', 'tableau']
+        e2e-test: ['bigquery', 'dbt_redshift', 'mssql', 'mysql', 'redash', 'snowflake', 'tableau', 'vertica', 'python']
     environment: test
 
     steps:
@@ -108,6 +108,9 @@ jobs:
         E2E_TABLEAU_PASSWORD: ${{ secrets.E2E_TABLEAU_PASSWORD }}
         E2E_TABLEAU_HOST_PORT: ${{ secrets.E2E_TABLEAU_HOST_PORT }}
         E2E_TABLEAU_SITE: ${{ secrets.E2E_TABLEAU_SITE }}
+        E2E_REDASH_HOST_PORT: ${{ secrets.E2E_REDASH_HOST_PORT }}
+        E2E_REDASH_USERNAME: ${{ secrets.E2E_REDASH_USERNAME }}
+        E2E_REDASH_API_KEY: ${{ secrets.E2E_REDASH_API_KEY }}
       run: |
         source env/bin/activate
         export SITE_CUSTOMIZE_PATH=$(python -c "import site; import os; from pathlib import Path; print(os.path.relpath(site.getsitepackages()[0], str(Path.cwd())))")/sitecustomize.py

ingestion/src/metadata/ingestion/source/dashboard/redash/metadata.py

@@ -286,6 +286,7 @@ class RedashSource(DashboardServiceSource):
                     chartUrl=self.get_dashboard_url(dashboard_details),
                     description=visualization["description"] if visualization else "",
                 )
+                self.status.scanned(f"Chart: {chart_display_name}")
             except Exception as exc:
                 logger.debug(traceback.format_exc())
                 logger.warning(

ingestion/tests/cli_e2e/dashboard/redash/redash.yaml

@@ -0,0 +1,24 @@
+source:
+  type: redash
+  serviceName: local_redash
+  serviceConnection:
+    config:
+      type: Redash
+      hostPort: $E2E_REDASH_HOST_PORT
+      apiKey: $E2E_REDASH_API_KEY
+      username: $E2E_REDASH_USERNAME
+  sourceConfig:
+    config:
+      type: DashboardMetadata
+      dashboardFilterPattern: {}
+      chartFilterPattern: {}
+sink:
+  type: metadata-rest
+  config: {}
+workflowConfig:
+  loggerLevel: DEBUG
+  openMetadataServerConfig:
+    hostPort: http://localhost:8585/api
+    authProvider: openmetadata
+    securityConfig:
+      jwtToken: "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg"

ingestion/tests/cli_e2e/test_cli_redash.py

@@ -0,0 +1,64 @@
+#  Copyright 2022 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""
+Test Redash connector with CLI
+"""
+from typing import List
+
+from .common.test_cli_dashboard import CliCommonDashboard
+
+
+class RedashCliTest(CliCommonDashboard.TestSuite):
+    @staticmethod
+    def get_connector_name() -> str:
+        return "redash"
+
+    def get_includes_dashboards(self) -> List[str]:
+        return [".*Mil.*"]
+
+    def get_excludes_dashboards(self) -> List[str]:
+        return ["Test"]
+
+    def get_includes_charts(self) -> List[str]:
+        return ["4"]
+
+    def get_excludes_charts(self) -> List[str]:
+        return [".*Query.*"]
+
+    # Redash do not ingest datamodels
+    def get_includes_datamodels(self) -> List[str]:
+        return []
+
+    # Redash do not ingest datamodels
+    def get_excludes_datamodels(self) -> List[str]:
+        return []
+
+    def expected_entities(self) -> int:
+        return 12
+
+    def expected_lineage(self) -> int:
+        return 0
+
+    def expected_tags(self) -> int:
+        return 4
+
+    def expected_not_included_entities(self) -> int:
+        return 12
+
+    def expected_not_included_sink_entities(self) -> int:
+        return 13
+
+    def expected_filtered_mix(self) -> int:
+        return 6
+
+    def expected_filtered_sink_mix(self) -> int:
+        return 8