Metabase E2E Test & docs (#11126)

.github/workflows/py-cli-e2e-tests.yml

@@ -21,7 +21,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        e2e-test: ['bigquery', 'dbt_redshift', 'mssql', 'mysql', 'redash', 'snowflake', 'tableau', 'powerbi', 'vertica', 'python']
+        e2e-test: ['bigquery', 'dbt_redshift', 'metabase', 'mssql', 'mysql', 'redash', 'snowflake', 'tableau', 'powerbi', 'vertica', 'python']
     environment: test
 
     steps:
@@ -114,6 +114,9 @@ jobs:
         E2E_POWERBI_CLIENT_SECRET: ${{ secrets.E2E_POWERBI_CLIENT_SECRET }}
         E2E_POWERBI_CLIENT_ID: ${{ secrets.E2E_POWERBI_CLIENT_ID }}
         E2E_POWERBI_TENANT_ID: ${{ secrets.E2E_POWERBI_TENANT_ID }}
+        E2E_METABASE_HOST_PORT: ${{ secrets.TEST_METABASE_HOST_PORT }}
+        E2E_METABASE_PASSWORD: ${{ secrets.TEST_METABASE_PASSWORD }}
+        E2E_METABASE_USERNAME: ${{ secrets.TEST_METABASE_USERNAME }}
       run: |
         source env/bin/activate
         export SITE_CUSTOMIZE_PATH=$(python -c "import site; import os; from pathlib import Path; print(os.path.relpath(site.getsitepackages()[0], str(Path.cwd())))")/sitecustomize.py

ingestion/src/metadata/ingestion/source/dashboard/metabase/client.py

@@ -59,6 +59,11 @@ class MetabaseClient:
                 timeout=DEFAULT_TIMEOUT,
             )
             return self.resp.json()["id"]
+
+        except KeyError as exe:
+            msg = "Failed to fetch metabase session, please validate credentials"
+            raise SourceConnectionException(msg) from exe
+
         except Exception as exc:
             msg = f"Unknown error in connection: {exc}."
             raise SourceConnectionException(msg) from exc

ingestion/tests/cli_e2e/dashboard/metabase/metabase.yaml

@@ -0,0 +1,26 @@
+source:
+  type: metabase
+  serviceName: local_metabase
+  serviceConnection:
+    config:
+      type: Metabase
+      username: $E2E_METABASE_USERNAME
+      password: $E2E_METABASE_PASSWORD
+      hostPort: $E2E_METABASE_HOST_PORT
+  sourceConfig:
+    config:
+      type: DashboardMetadata
+      dashboardFilterPattern: {}
+      chartFilterPattern: {}
+      dbServiceNames:
+      - local_redshift
+sink:
+  type: metadata-rest
+  config: {}
+workflowConfig:
+  loggerLevel: DEBUG
+  openMetadataServerConfig:
+    hostPort: http://localhost:8585/api
+    authProvider: openmetadata
+    securityConfig:
+      "jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg"

ingestion/tests/cli_e2e/dashboard/metabase/redshift.yaml

@@ -0,0 +1,25 @@
+source:
+  type: redshift
+  serviceName: local_redshift
+  serviceConnection:
+    config:
+      hostPort: $E2E_REDSHIFT_HOST_PORT
+      username: $E2E_REDSHIFT_USERNAME
+      password: $E2E_REDSHIFT_PASSWORD
+      database: $E2E_REDSHIFT_DATABASE
+      type: Redshift
+  sourceConfig:
+    config:
+      schemaFilterPattern:
+        includes:
+        - dbt_jaffle
+sink:
+  type: metadata-rest
+  config: {}
+workflowConfig:
+  loggerLevel: DEBUG
+  openMetadataServerConfig:
+    hostPort: http://localhost:8585/api
+    authProvider: openmetadata
+    securityConfig:
+      "jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg"

ingestion/tests/cli_e2e/test_cli_metabase.py

@@ -0,0 +1,77 @@
+#  Copyright 2022 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""
+Test Metabase connector with CLI
+"""
+from pathlib import Path
+from typing import List
+
+from .base.test_cli import PATH_TO_RESOURCES
+from .common.test_cli_dashboard import CliCommonDashboard
+
+
+class MetabaseCliTest(CliCommonDashboard.TestSuite):
+
+    # in case we want to do something before running the tests
+    def prepare(self) -> None:
+        redshift_file_path = str(
+            Path(
+                PATH_TO_RESOURCES
+                + f"/dashboard/{self.get_connector_name()}/redshift.yaml"
+            )
+        )
+        self.run_command(test_file_path=redshift_file_path)
+
+    @staticmethod
+    def get_connector_name() -> str:
+        return "metabase"
+
+    def get_includes_dashboards(self) -> List[str]:
+        return [".*jaffle_shop.*"]
+
+    def get_excludes_dashboards(self) -> List[str]:
+        return [".*Delete.*"]
+
+    def get_includes_charts(self) -> List[str]:
+        return [".*Query.*"]
+
+    def get_excludes_charts(self) -> List[str]:
+        return [".*Question.*"]
+
+    # Metabase do not ingest datamodels
+    def get_includes_datamodels(self) -> List[str]:
+        return []
+
+    # Metabase do not ingest datamodels
+    def get_excludes_datamodels(self) -> List[str]:
+        return []
+
+    def expected_entities(self) -> int:
+        return 6
+
+    def expected_lineage(self) -> int:
+        return 5
+
+    def expected_tags(self) -> int:
+        return 0
+
+    def expected_not_included_entities(self) -> int:
+        return 6
+
+    def expected_not_included_sink_entities(self) -> int:
+        return 11
+
+    def expected_filtered_mix(self) -> int:
+        return 2
+
+    def expected_filtered_sink_mix(self) -> int:
+        return 5

openmetadata-docs-v1/content/v0.13.2/connectors/dashboard/metabase/airflow.md

@@ -54,19 +54,19 @@ The workflow is modeled around the following
 
 {% codeInfo srNumber=1 %}
 
-**username**: Specify the User to connect to Metabase. It should have enough privileges to read all the metadata.
+**username**: Username to connect to Metabase, for ex. `user@organization.com`. This user should have access to relevant dashboards and charts in Metabase to fetch the metadata.
 
 {% /codeInfo %}
 
 {% codeInfo srNumber=2 %}
 
-**password**: Password for Metabase.
+**password**: Password of the user account to connect with Metabase.
 
 {% /codeInfo %}
 
 {% codeInfo srNumber=3 %}
 
-**hostPort**: URL to the Metabase instance.
+**hostPort**: The hostPort parameter specifies the host and port of the Metabase instance. This should be specified as a string in the format `http://hostname:port` or `https://hostname:port`. For example, you might set the hostPort parameter to `https://org.metabase.com:3000`.
 
 {% /codeInfo %}
 

openmetadata-docs-v1/content/v0.13.2/connectors/dashboard/metabase/cli.md

@@ -58,19 +58,19 @@ This is a sample config for Metabase:
 
 {% codeInfo srNumber=1 %}
 
-**username**: Specify the User to connect to Metabase. It should have enough privileges to read all the metadata.
+**username**: Username to connect to Metabase, for ex. `user@organization.com`. This user should have access to relevant dashboards and charts in Metabase to fetch the metadata.
 
 {% /codeInfo %}
 
 {% codeInfo srNumber=2 %}
 
-**password**: Password for Metabase.
+**password**: Password of the user account to connect with Metabase.
 
 {% /codeInfo %}
 
 {% codeInfo srNumber=3 %}
 
-**hostPort**: URL to the Metabase instance.
+**hostPort**: The hostPort parameter specifies the host and port of the Metabase instance. This should be specified as a string in the format `http://hostname:port` or `https://hostname:port`. For example, you might set the hostPort parameter to `https://org.metabase.com:3000`.
 
 {% /codeInfo %}
 

openmetadata-docs-v1/content/v0.13.2/connectors/dashboard/metabase/index.md

@@ -162,9 +162,9 @@ desired.
 #### Connection Options
 
 
-- **Host and Port**: URL to the Metabase instance.
-- **Username**: Specify the User to connect to Metabase. It should have enough privileges to read all the metadata.
-- **Password**: Password for Metabase.
+- **Host and Port**: The hostPort parameter specifies the host and port of the Metabase instance. This should be specified as a string in the format `http://hostname:port` or `https://hostname:port`. For example, you might set the hostPort parameter to `https://org.metabase.com:3000`.
+- **Username**: Username to connect to Metabase, for ex. `user@organization.com`. This user should have access to relevant dashboards and charts in Metabase to fetch the metadata.
+- **Password**: Password of the user account to connect with Metabase.
 
 {% /extraContent %}
 

openmetadata-docs-v1/content/v1.0.0/connectors/dashboard/metabase/airflow.md

@@ -54,19 +54,19 @@ The workflow is modeled around the following
 
 {% codeInfo srNumber=1 %}
 
-**username**: Specify the User to connect to Metabase. It should have enough privileges to read all the metadata.
+**username**: Username to connect to Metabase, for ex. `user@organization.com`. This user should have access to relevant dashboards and charts in Metabase to fetch the metadata.
 
 {% /codeInfo %}
 
 {% codeInfo srNumber=2 %}
 
-**password**: Password for Metabase.
+**password**: Password of the user account to connect with Metabase.
 
 {% /codeInfo %}
 
 {% codeInfo srNumber=3 %}
 
-**hostPort**: URL to the Metabase instance.
+**hostPort**: The hostPort parameter specifies the host and port of the Metabase instance. This should be specified as a string in the format `http://hostname:port` or `https://hostname:port`. For example, you might set the hostPort parameter to `https://org.metabase.com:3000`.
 
 {% /codeInfo %}
 

openmetadata-docs-v1/content/v1.0.0/connectors/dashboard/metabase/cli.md

@@ -58,19 +58,19 @@ This is a sample config for Metabase:
 
 {% codeInfo srNumber=1 %}
 
-**username**: Specify the User to connect to Metabase. It should have enough privileges to read all the metadata.
+**username**: Username to connect to Metabase, for ex. `user@organization.com`. This user should have access to relevant dashboards and charts in Metabase to fetch the metadata.
 
 {% /codeInfo %}
 
 {% codeInfo srNumber=2 %}
 
-**password**: Password for Metabase.
+**password**: Password of the user account to connect with Metabase.
 
 {% /codeInfo %}
 
 {% codeInfo srNumber=3 %}
 
-**hostPort**: URL to the Metabase instance.
+**hostPort**: The hostPort parameter specifies the host and port of the Metabase instance. This should be specified as a string in the format `http://hostname:port` or `https://hostname:port`. For example, you might set the hostPort parameter to `https://org.metabase.com:3000`.
 
 {% /codeInfo %}
 

openmetadata-docs-v1/content/v1.0.0/connectors/dashboard/metabase/index.md

@@ -162,9 +162,9 @@ desired.
 #### Connection Options
 
 
-- **Host and Port**: URL to the Metabase instance.
-- **Username**: Specify the User to connect to Metabase. It should have enough privileges to read all the metadata.
-- **Password**: Password for Metabase.
+- **Host and Port**: The hostPort parameter specifies the host and port of the Metabase instance. This should be specified as a string in the format `http://hostname:port` or `https://hostname:port`. For example, you might set the hostPort parameter to `https://org.metabase.com:3000`.
+- **Username**: Username to connect to Metabase, for ex. `user@organization.com`. This user should have access to relevant dashboards and charts in Metabase to fetch the metadata.
+- **Password**: Password of the user account to connect with Metabase.
 
 {% /extraContent %}
 

openmetadata-ui/src/main/resources/ui/public/locales/en-US/Dashboard/Metabase.md

@@ -3,29 +3,29 @@
 In this section, we provide guides and references to use the Metabase connector.
 
 # Requirements
-<!-- to be updated -->
-You can find further information on the Kafka connector in the [docs](https://docs.open-metadata.org/connectors/dashboard/metabase).
+
+To run the Ingestion via the UI you'll need to use the OpenMetadata Ingestion Container, which comes shipped with custom Airflow plugins to handle the workflow deployment.
+
+You can find further information on the Metabase connector in the [docs](https://docs.open-metadata.org/connectors/dashboard/metabase).
+
 
 ## Connection Details
 
 $$section
 ### Username $(id="username")
 
-Username to connect to Metabase. This user should have privileges to read all the metadata in Metabase.
-<!-- username to be updated -->
+Username to connect to Metabase, for ex. `user@organization.com`. This user should have access to relevant dashboards and charts in Metabase to fetch the metadata.
 $$
 
 $$section
 ### Password $(id="password")
 
-Password to connect to Metabase.
-<!-- password to be updated -->
+Password of the user account to connect with Metabase.
 $$
 
 $$section
 ### Host Port $(id="hostPort")
 
-Host and Port of the Metabase instance.
-<!-- hostPort to be updated -->
+The hostPort parameter specifies the host and port of the Metabase instance. This should be specified as a string in the format `http://hostname:port` or `https://hostname:port`. For example, you might set the hostPort parameter to `https://org.metabase.com:3000`.
 $$