diff --git a/.github/workflows/py-cli-e2e-tests.yml b/.github/workflows/py-cli-e2e-tests.yml index ad227d978ee..759ecf466d4 100644 --- a/.github/workflows/py-cli-e2e-tests.yml +++ b/.github/workflows/py-cli-e2e-tests.yml @@ -21,7 +21,7 @@ jobs: strategy: fail-fast: false matrix: - e2e-test: ['python', 'mysql', 'bigquery', 'snowflake', 'dbt_redshift', 'mssql', 'vertica', 'tableau'] + e2e-test: ['bigquery', 'dbt_redshift', 'mssql', 'mysql', 'redash', 'snowflake', 'tableau', 'vertica', 'python'] environment: test steps: @@ -108,6 +108,9 @@ jobs: E2E_TABLEAU_PASSWORD: ${{ secrets.E2E_TABLEAU_PASSWORD }} E2E_TABLEAU_HOST_PORT: ${{ secrets.E2E_TABLEAU_HOST_PORT }} E2E_TABLEAU_SITE: ${{ secrets.E2E_TABLEAU_SITE }} + E2E_REDASH_HOST_PORT: ${{ secrets.E2E_REDASH_HOST_PORT }} + E2E_REDASH_USERNAME: ${{ secrets.E2E_REDASH_USERNAME }} + E2E_REDASH_API_KEY: ${{ secrets.E2E_REDASH_API_KEY }} run: | source env/bin/activate export SITE_CUSTOMIZE_PATH=$(python -c "import site; import os; from pathlib import Path; print(os.path.relpath(site.getsitepackages()[0], str(Path.cwd())))")/sitecustomize.py diff --git a/ingestion/src/metadata/ingestion/source/dashboard/redash/metadata.py b/ingestion/src/metadata/ingestion/source/dashboard/redash/metadata.py index c34cda12b42..60ed440b945 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/redash/metadata.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/redash/metadata.py @@ -286,6 +286,7 @@ class RedashSource(DashboardServiceSource): chartUrl=self.get_dashboard_url(dashboard_details), description=visualization["description"] if visualization else "", ) + self.status.scanned(f"Chart: {chart_display_name}") except Exception as exc: logger.debug(traceback.format_exc()) logger.warning( diff --git a/ingestion/tests/cli_e2e/dashboard/redash/redash.yaml b/ingestion/tests/cli_e2e/dashboard/redash/redash.yaml new file mode 100644 index 00000000000..2aa14014c55 --- /dev/null +++ b/ingestion/tests/cli_e2e/dashboard/redash/redash.yaml @@ -0,0 +1,24 @@ +source: + type: redash + serviceName: local_redash + serviceConnection: + config: + type: Redash + hostPort: $E2E_REDASH_HOST_PORT + apiKey: $E2E_REDASH_API_KEY + username: $E2E_REDASH_USERNAME + sourceConfig: + config: + type: DashboardMetadata + dashboardFilterPattern: {} + chartFilterPattern: {} +sink: + type: metadata-rest + config: {} +workflowConfig: + loggerLevel: DEBUG + openMetadataServerConfig: + hostPort: http://localhost:8585/api + authProvider: openmetadata + securityConfig: + jwtToken: "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg" \ No newline at end of file diff --git a/ingestion/tests/cli_e2e/test_cli_redash.py b/ingestion/tests/cli_e2e/test_cli_redash.py new file mode 100644 index 00000000000..dfd0e6ba936 --- /dev/null +++ b/ingestion/tests/cli_e2e/test_cli_redash.py @@ -0,0 +1,64 @@ +# Copyright 2022 Collate +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Test Redash connector with CLI +""" +from typing import List + +from .common.test_cli_dashboard import CliCommonDashboard + + +class RedashCliTest(CliCommonDashboard.TestSuite): + @staticmethod + def get_connector_name() -> str: + return "redash" + + def get_includes_dashboards(self) -> List[str]: + return [".*Mil.*"] + + def get_excludes_dashboards(self) -> List[str]: + return ["Test"] + + def get_includes_charts(self) -> List[str]: + return ["4"] + + def get_excludes_charts(self) -> List[str]: + return [".*Query.*"] + + # Redash do not ingest datamodels + def get_includes_datamodels(self) -> List[str]: + return [] + + # Redash do not ingest datamodels + def get_excludes_datamodels(self) -> List[str]: + return [] + + def expected_entities(self) -> int: + return 12 + + def expected_lineage(self) -> int: + return 0 + + def expected_tags(self) -> int: + return 4 + + def expected_not_included_entities(self) -> int: + return 12 + + def expected_not_included_sink_entities(self) -> int: + return 13 + + def expected_filtered_mix(self) -> int: + return 6 + + def expected_filtered_sink_mix(self) -> int: + return 8