diff --git a/.github/workflows/py-cli-e2e-tests.yml b/.github/workflows/py-cli-e2e-tests.yml index 8a504946170..350fe3d560c 100644 --- a/.github/workflows/py-cli-e2e-tests.yml +++ b/.github/workflows/py-cli-e2e-tests.yml @@ -21,7 +21,7 @@ jobs: strategy: fail-fast: false matrix: - e2e-test: ['bigquery', 'dbt_redshift', 'mssql', 'mysql', 'redash', 'snowflake', 'tableau', 'powerbi', 'vertica', 'python'] + e2e-test: ['bigquery', 'dbt_redshift', 'metabase', 'mssql', 'mysql', 'redash', 'snowflake', 'tableau', 'powerbi', 'vertica', 'python'] environment: test steps: @@ -114,6 +114,9 @@ jobs: E2E_POWERBI_CLIENT_SECRET: ${{ secrets.E2E_POWERBI_CLIENT_SECRET }} E2E_POWERBI_CLIENT_ID: ${{ secrets.E2E_POWERBI_CLIENT_ID }} E2E_POWERBI_TENANT_ID: ${{ secrets.E2E_POWERBI_TENANT_ID }} + E2E_METABASE_HOST_PORT: ${{ secrets.TEST_METABASE_HOST_PORT }} + E2E_METABASE_PASSWORD: ${{ secrets.TEST_METABASE_PASSWORD }} + E2E_METABASE_USERNAME: ${{ secrets.TEST_METABASE_USERNAME }} run: | source env/bin/activate export SITE_CUSTOMIZE_PATH=$(python -c "import site; import os; from pathlib import Path; print(os.path.relpath(site.getsitepackages()[0], str(Path.cwd())))")/sitecustomize.py diff --git a/ingestion/src/metadata/ingestion/source/dashboard/metabase/client.py b/ingestion/src/metadata/ingestion/source/dashboard/metabase/client.py index b519a21162f..c3e655c42b0 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/metabase/client.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/metabase/client.py @@ -59,6 +59,11 @@ class MetabaseClient: timeout=DEFAULT_TIMEOUT, ) return self.resp.json()["id"] + + except KeyError as exe: + msg = "Failed to fetch metabase session, please validate credentials" + raise SourceConnectionException(msg) from exe + except Exception as exc: msg = f"Unknown error in connection: {exc}." raise SourceConnectionException(msg) from exc diff --git a/ingestion/tests/cli_e2e/dashboard/metabase/metabase.yaml b/ingestion/tests/cli_e2e/dashboard/metabase/metabase.yaml new file mode 100644 index 00000000000..f392875b444 --- /dev/null +++ b/ingestion/tests/cli_e2e/dashboard/metabase/metabase.yaml @@ -0,0 +1,26 @@ +source: + type: metabase + serviceName: local_metabase + serviceConnection: + config: + type: Metabase + username: $E2E_METABASE_USERNAME + password: $E2E_METABASE_PASSWORD + hostPort: $E2E_METABASE_HOST_PORT + sourceConfig: + config: + type: DashboardMetadata + dashboardFilterPattern: {} + chartFilterPattern: {} + dbServiceNames: + - local_redshift +sink: + type: metadata-rest + config: {} +workflowConfig: + loggerLevel: DEBUG + openMetadataServerConfig: + hostPort: http://localhost:8585/api + authProvider: openmetadata + securityConfig: + "jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg" diff --git a/ingestion/tests/cli_e2e/dashboard/metabase/redshift.yaml b/ingestion/tests/cli_e2e/dashboard/metabase/redshift.yaml new file mode 100644 index 00000000000..c65207f96e8 --- /dev/null +++ b/ingestion/tests/cli_e2e/dashboard/metabase/redshift.yaml @@ -0,0 +1,25 @@ +source: + type: redshift + serviceName: local_redshift + serviceConnection: + config: + hostPort: $E2E_REDSHIFT_HOST_PORT + username: $E2E_REDSHIFT_USERNAME + password: $E2E_REDSHIFT_PASSWORD + database: $E2E_REDSHIFT_DATABASE + type: Redshift + sourceConfig: + config: + schemaFilterPattern: + includes: + - dbt_jaffle +sink: + type: metadata-rest + config: {} +workflowConfig: + loggerLevel: DEBUG + openMetadataServerConfig: + hostPort: http://localhost:8585/api + authProvider: openmetadata + securityConfig: + "jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg" diff --git a/ingestion/tests/cli_e2e/test_cli_metabase.py b/ingestion/tests/cli_e2e/test_cli_metabase.py new file mode 100644 index 00000000000..56cba0e9b25 --- /dev/null +++ b/ingestion/tests/cli_e2e/test_cli_metabase.py @@ -0,0 +1,77 @@ +# Copyright 2022 Collate +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Test Metabase connector with CLI +""" +from pathlib import Path +from typing import List + +from .base.test_cli import PATH_TO_RESOURCES +from .common.test_cli_dashboard import CliCommonDashboard + + +class MetabaseCliTest(CliCommonDashboard.TestSuite): + + # in case we want to do something before running the tests + def prepare(self) -> None: + redshift_file_path = str( + Path( + PATH_TO_RESOURCES + + f"/dashboard/{self.get_connector_name()}/redshift.yaml" + ) + ) + self.run_command(test_file_path=redshift_file_path) + + @staticmethod + def get_connector_name() -> str: + return "metabase" + + def get_includes_dashboards(self) -> List[str]: + return [".*jaffle_shop.*"] + + def get_excludes_dashboards(self) -> List[str]: + return [".*Delete.*"] + + def get_includes_charts(self) -> List[str]: + return [".*Query.*"] + + def get_excludes_charts(self) -> List[str]: + return [".*Question.*"] + + # Metabase do not ingest datamodels + def get_includes_datamodels(self) -> List[str]: + return [] + + # Metabase do not ingest datamodels + def get_excludes_datamodels(self) -> List[str]: + return [] + + def expected_entities(self) -> int: + return 6 + + def expected_lineage(self) -> int: + return 5 + + def expected_tags(self) -> int: + return 0 + + def expected_not_included_entities(self) -> int: + return 6 + + def expected_not_included_sink_entities(self) -> int: + return 11 + + def expected_filtered_mix(self) -> int: + return 2 + + def expected_filtered_sink_mix(self) -> int: + return 5 diff --git a/openmetadata-docs-v1/content/v0.13.2/connectors/dashboard/metabase/airflow.md b/openmetadata-docs-v1/content/v0.13.2/connectors/dashboard/metabase/airflow.md index 0291bec7881..7ea883ff269 100644 --- a/openmetadata-docs-v1/content/v0.13.2/connectors/dashboard/metabase/airflow.md +++ b/openmetadata-docs-v1/content/v0.13.2/connectors/dashboard/metabase/airflow.md @@ -54,19 +54,19 @@ The workflow is modeled around the following {% codeInfo srNumber=1 %} -**username**: Specify the User to connect to Metabase. It should have enough privileges to read all the metadata. +**username**: Username to connect to Metabase, for ex. `user@organization.com`. This user should have access to relevant dashboards and charts in Metabase to fetch the metadata. {% /codeInfo %} {% codeInfo srNumber=2 %} -**password**: Password for Metabase. +**password**: Password of the user account to connect with Metabase. {% /codeInfo %} {% codeInfo srNumber=3 %} -**hostPort**: URL to the Metabase instance. +**hostPort**: The hostPort parameter specifies the host and port of the Metabase instance. This should be specified as a string in the format `http://hostname:port` or `https://hostname:port`. For example, you might set the hostPort parameter to `https://org.metabase.com:3000`. {% /codeInfo %} diff --git a/openmetadata-docs-v1/content/v0.13.2/connectors/dashboard/metabase/cli.md b/openmetadata-docs-v1/content/v0.13.2/connectors/dashboard/metabase/cli.md index 35eaadc5db2..690ada5dfaf 100644 --- a/openmetadata-docs-v1/content/v0.13.2/connectors/dashboard/metabase/cli.md +++ b/openmetadata-docs-v1/content/v0.13.2/connectors/dashboard/metabase/cli.md @@ -58,19 +58,19 @@ This is a sample config for Metabase: {% codeInfo srNumber=1 %} -**username**: Specify the User to connect to Metabase. It should have enough privileges to read all the metadata. +**username**: Username to connect to Metabase, for ex. `user@organization.com`. This user should have access to relevant dashboards and charts in Metabase to fetch the metadata. {% /codeInfo %} {% codeInfo srNumber=2 %} -**password**: Password for Metabase. +**password**: Password of the user account to connect with Metabase. {% /codeInfo %} {% codeInfo srNumber=3 %} -**hostPort**: URL to the Metabase instance. +**hostPort**: The hostPort parameter specifies the host and port of the Metabase instance. This should be specified as a string in the format `http://hostname:port` or `https://hostname:port`. For example, you might set the hostPort parameter to `https://org.metabase.com:3000`. {% /codeInfo %} diff --git a/openmetadata-docs-v1/content/v0.13.2/connectors/dashboard/metabase/index.md b/openmetadata-docs-v1/content/v0.13.2/connectors/dashboard/metabase/index.md index 9c9535ddc4a..97cf7281496 100644 --- a/openmetadata-docs-v1/content/v0.13.2/connectors/dashboard/metabase/index.md +++ b/openmetadata-docs-v1/content/v0.13.2/connectors/dashboard/metabase/index.md @@ -162,9 +162,9 @@ desired. #### Connection Options -- **Host and Port**: URL to the Metabase instance. -- **Username**: Specify the User to connect to Metabase. It should have enough privileges to read all the metadata. -- **Password**: Password for Metabase. +- **Host and Port**: The hostPort parameter specifies the host and port of the Metabase instance. This should be specified as a string in the format `http://hostname:port` or `https://hostname:port`. For example, you might set the hostPort parameter to `https://org.metabase.com:3000`. +- **Username**: Username to connect to Metabase, for ex. `user@organization.com`. This user should have access to relevant dashboards and charts in Metabase to fetch the metadata. +- **Password**: Password of the user account to connect with Metabase. {% /extraContent %} diff --git a/openmetadata-docs-v1/content/v1.0.0/connectors/dashboard/metabase/airflow.md b/openmetadata-docs-v1/content/v1.0.0/connectors/dashboard/metabase/airflow.md index 0291bec7881..7ea883ff269 100644 --- a/openmetadata-docs-v1/content/v1.0.0/connectors/dashboard/metabase/airflow.md +++ b/openmetadata-docs-v1/content/v1.0.0/connectors/dashboard/metabase/airflow.md @@ -54,19 +54,19 @@ The workflow is modeled around the following {% codeInfo srNumber=1 %} -**username**: Specify the User to connect to Metabase. It should have enough privileges to read all the metadata. +**username**: Username to connect to Metabase, for ex. `user@organization.com`. This user should have access to relevant dashboards and charts in Metabase to fetch the metadata. {% /codeInfo %} {% codeInfo srNumber=2 %} -**password**: Password for Metabase. +**password**: Password of the user account to connect with Metabase. {% /codeInfo %} {% codeInfo srNumber=3 %} -**hostPort**: URL to the Metabase instance. +**hostPort**: The hostPort parameter specifies the host and port of the Metabase instance. This should be specified as a string in the format `http://hostname:port` or `https://hostname:port`. For example, you might set the hostPort parameter to `https://org.metabase.com:3000`. {% /codeInfo %} diff --git a/openmetadata-docs-v1/content/v1.0.0/connectors/dashboard/metabase/cli.md b/openmetadata-docs-v1/content/v1.0.0/connectors/dashboard/metabase/cli.md index 35eaadc5db2..690ada5dfaf 100644 --- a/openmetadata-docs-v1/content/v1.0.0/connectors/dashboard/metabase/cli.md +++ b/openmetadata-docs-v1/content/v1.0.0/connectors/dashboard/metabase/cli.md @@ -58,19 +58,19 @@ This is a sample config for Metabase: {% codeInfo srNumber=1 %} -**username**: Specify the User to connect to Metabase. It should have enough privileges to read all the metadata. +**username**: Username to connect to Metabase, for ex. `user@organization.com`. This user should have access to relevant dashboards and charts in Metabase to fetch the metadata. {% /codeInfo %} {% codeInfo srNumber=2 %} -**password**: Password for Metabase. +**password**: Password of the user account to connect with Metabase. {% /codeInfo %} {% codeInfo srNumber=3 %} -**hostPort**: URL to the Metabase instance. +**hostPort**: The hostPort parameter specifies the host and port of the Metabase instance. This should be specified as a string in the format `http://hostname:port` or `https://hostname:port`. For example, you might set the hostPort parameter to `https://org.metabase.com:3000`. {% /codeInfo %} diff --git a/openmetadata-docs-v1/content/v1.0.0/connectors/dashboard/metabase/index.md b/openmetadata-docs-v1/content/v1.0.0/connectors/dashboard/metabase/index.md index 4dde9057500..acd4e435d22 100644 --- a/openmetadata-docs-v1/content/v1.0.0/connectors/dashboard/metabase/index.md +++ b/openmetadata-docs-v1/content/v1.0.0/connectors/dashboard/metabase/index.md @@ -162,9 +162,9 @@ desired. #### Connection Options -- **Host and Port**: URL to the Metabase instance. -- **Username**: Specify the User to connect to Metabase. It should have enough privileges to read all the metadata. -- **Password**: Password for Metabase. +- **Host and Port**: The hostPort parameter specifies the host and port of the Metabase instance. This should be specified as a string in the format `http://hostname:port` or `https://hostname:port`. For example, you might set the hostPort parameter to `https://org.metabase.com:3000`. +- **Username**: Username to connect to Metabase, for ex. `user@organization.com`. This user should have access to relevant dashboards and charts in Metabase to fetch the metadata. +- **Password**: Password of the user account to connect with Metabase. {% /extraContent %} diff --git a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Dashboard/Metabase.md b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Dashboard/Metabase.md index 962ae4e1936..7e95bbb9897 100644 --- a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Dashboard/Metabase.md +++ b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Dashboard/Metabase.md @@ -3,29 +3,29 @@ In this section, we provide guides and references to use the Metabase connector. # Requirements - -You can find further information on the Kafka connector in the [docs](https://docs.open-metadata.org/connectors/dashboard/metabase). + +To run the Ingestion via the UI you'll need to use the OpenMetadata Ingestion Container, which comes shipped with custom Airflow plugins to handle the workflow deployment. + +You can find further information on the Metabase connector in the [docs](https://docs.open-metadata.org/connectors/dashboard/metabase). + ## Connection Details $$section ### Username $(id="username") -Username to connect to Metabase. This user should have privileges to read all the metadata in Metabase. - +Username to connect to Metabase, for ex. `user@organization.com`. This user should have access to relevant dashboards and charts in Metabase to fetch the metadata. $$ $$section ### Password $(id="password") -Password to connect to Metabase. - +Password of the user account to connect with Metabase. $$ $$section ### Host Port $(id="hostPort") -Host and Port of the Metabase instance. - +The hostPort parameter specifies the host and port of the Metabase instance. This should be specified as a string in the format `http://hostname:port` or `https://hostname:port`. For example, you might set the hostPort parameter to `https://org.metabase.com:3000`. $$