E2E for Bigquery (#9007)

.github/workflows/py-cli-e2e-tests.yml

@@ -21,7 +21,7 @@ jobs:
     strategy:
       matrix:
         py-version: ['3.9']
-        e2e-test: ['mysql']
+        e2e-test: ['mysql', 'bigquery']
 
     steps:
     - name: Checkout
@@ -66,6 +66,7 @@ jobs:
       continue-on-error: true
       env:
         E2E_TEST: ${{ matrix.e2e-test }}
+        E2E_BQ_PROJECT_ID: ${{ secrets.E2E_BQ_PROJECT_ID }}
       run: |
         source env/bin/activate
         python -m pytest -c ingestion/setup.cfg ingestion/tests/cli_e2e/test_cli_$E2E_TEST.py

ingestion/src/metadata/ingestion/source/database/bigquery.py

@@ -274,7 +274,7 @@ class BigquerySource(CommonDbSourceService):
             except NotImplementedError:
                 logger.warning("View definition not implemented")
                 view_definition = ""
-            return view_definition
+            return f"CREATE VIEW {schema_name}.{table_name} AS {view_definition}"
         return None
 
     def get_table_partition_details(

ingestion/src/metadata/orm_profiler/metrics/datalake_metrics_computation_registry.py

@@ -100,8 +100,8 @@ def get_static_metrics(
                     )
                 )
         row_dict = {}
-        for index, table_metric in enumerate(metrics):
+        for index, column_metric in enumerate(metrics):
-            row_dict[table_metric.name()] = row[index]
+            row_dict[column_metric.name()] = row[index]
         return row_dict
     except Exception as exc:
         logger.debug(
@@ -142,7 +142,7 @@ def get_window_metrics(*args, **kwargs):
 
 
 compute_metrics_registry = enum_register()
-compute_metrics_registry.add("Table")(get_table_metrics)
 compute_metrics_registry.add("Static")(get_static_metrics)
+compute_metrics_registry.add("Table")(get_table_metrics)
 compute_metrics_registry.add("Query")(get_query_metrics)
 compute_metrics_registry.add("Window")(get_window_metrics)

ingestion/src/metadata/orm_profiler/metrics/static/count.py

@@ -47,7 +47,7 @@ class Count(StaticMetric):
             return len(data_frame[self.col.name])
         except Exception as err:
             logger.debug(
-                f"Don't know how to process type {self.col.datatype} when computing MEAN"
+                f"Don't know how to process type {self.col.datatype} when computing Count"
             )
             logger.error(err)
             return 0

ingestion/tests/cli_e2e/common_e2e_sqa_mixins.py

@@ -0,0 +1,30 @@
+#  Copyright 2022 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""
+Add Common E2E Sqlalchemy Mixins
+"""
+
+
+class SQACommonMethods:
+    def create_table_and_view(self) -> None:
+        with self.engine.connect() as connection:
+            connection.execute(self.create_table_query)
+            for insert_query in self.insert_data_queries:
+                connection.execute(insert_query)
+            connection.execute(self.create_view_query)
+            connection.close()
+
+    def delete_table_and_view(self) -> None:
+        with self.engine.connect() as connection:
+            connection.execute(self.drop_view_query)
+            connection.execute(self.drop_table_query)
+            connection.close()

ingestion/tests/cli_e2e/database/bigquery/bigquery.yaml

@@ -0,0 +1,28 @@
+---
+source:
+  type: bigquery
+  serviceName: local_bigquery
+  serviceConnection:
+    config:
+      type: BigQuery
+      taxonomyProjectID: [$CYPRESS_BQ_PROJECT_ID]
+      credentials:
+        gcsConfig:
+          type: service_account
+          projectId: $E2E_BQ_PROJECT_ID
+          privateKeyId: $CYPRESS_BQ_PRIVATE_KEY_ID
+          privateKey: $CYPRESS_BQ_PRIVATE_KEY
+          clientEmail: $CYPRESS_BQ_CLIENT_EMAIL
+          clientId: $CYPRESS_BQ_CLIENT_ID
+  sourceConfig:
+    config:
+      type: DatabaseMetadata
+sink:
+  type: metadata-rest
+  config: {}
+workflowConfig:
+  openMetadataServerConfig:
+    hostPort: http://localhost:8585/api
+    authProvider: openmetadata
+    securityConfig:
+      jwtToken: eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg

ingestion/tests/cli_e2e/test_cli_bigquery.py

@@ -0,0 +1,100 @@
+#  Copyright 2022 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""
+Test Bigquery connector with CLI
+"""
+from typing import List
+
+from .common_e2e_sqa_mixins import SQACommonMethods
+from .test_cli_db_base_common import CliCommonDB
+
+
+class BigqueryCliTest(CliCommonDB.TestSuite, SQACommonMethods):
+
+    create_table_query: str = """
+        CREATE TABLE `open-metadata-beta.exclude_me`.orders (
+            id int,
+            order_name string
+        )
+    """
+
+    create_view_query: str = """
+       CREATE VIEW `open-metadata-beta.exclude_me.view_orders` AS
+                     SELECT orders.id as id, orders.order_name as order_name
+                       FROM `open-metadata-beta`.exclude_me.orders;
+    """
+
+    insert_data_queries: List[str] = [
+        "INSERT INTO `open-metadata-beta.exclude_me`.orders (id, order_name) VALUES (1,'XBOX');",
+        "INSERT INTO `open-metadata-beta.exclude_me`.orders (id, order_name) VALUES (2,'PS');",
+    ]
+
+    drop_table_query: str = """
+        DROP TABLE IF EXISTS `open-metadata-beta.exclude_me`.orders;
+    """
+
+    drop_view_query: str = """
+        DROP VIEW  IF EXISTS `open-metadata-beta.exclude_me`.view_orders;
+    """
+
+    def create_table_and_view(self) -> None:
+        SQACommonMethods.create_table_and_view(self)
+
+    def delete_table_and_view(self) -> None:
+        SQACommonMethods.delete_table_and_view(self)
+
+    @staticmethod
+    def get_connector_name() -> str:
+        return "bigquery"
+
+    @staticmethod
+    def expected_tables() -> int:
+        return 2
+
+    def inserted_rows_count(self) -> int:
+        return len(self.insert_data_queries)
+
+    @staticmethod
+    def fqn_created_table() -> str:
+        return "local_bigquery.open-metadata-beta.exclude_me.orders"
+
+    @staticmethod
+    def get_includes_schemas() -> List[str]:
+        return ["testschema"]
+
+    @staticmethod
+    def get_includes_tables() -> List[str]:
+        return ["testtable"]
+
+    @staticmethod
+    def get_excludes_tables() -> List[str]:
+        return ["exclude_table"]
+
+    @staticmethod
+    def expected_filtered_schema_includes() -> int:
+        return 1
+
+    @staticmethod
+    def expected_filtered_schema_excludes() -> int:
+        return 1
+
+    @staticmethod
+    def expected_filtered_table_includes() -> int:
+        return 1
+
+    @staticmethod
+    def expected_filtered_table_excludes() -> int:
+        return 1
+
+    @staticmethod
+    def expected_filtered_mix() -> int:
+        return 1

ingestion/tests/cli_e2e/test_cli_mysql.py

@@ -14,10 +14,11 @@ Test MySql connector with CLI
 """
 from typing import List
 
+from .common_e2e_sqa_mixins import SQACommonMethods
 from .test_cli_db_base_common import CliCommonDB
 
 
-class MysqlCliTest(CliCommonDB.TestSuite):
+class MysqlCliTest(CliCommonDB.TestSuite, SQACommonMethods):
 
     create_table_query: str = """
         CREATE TABLE persons (
@@ -50,18 +51,10 @@ class MysqlCliTest(CliCommonDB.TestSuite):
         return "mysql"
 
     def create_table_and_view(self) -> None:
-        with self.engine.connect() as connection:
+        SQACommonMethods.create_table_and_view(self)
-            connection.execute(self.create_table_query)
-            for insert_query in self.insert_data_queries:
-                connection.execute(insert_query)
-            connection.execute(self.create_view_query)
-            connection.close()
 
     def delete_table_and_view(self) -> None:
-        with self.engine.connect() as connection:
+        SQACommonMethods.delete_table_and_view(self)
-            connection.execute(self.drop_view_query)
-            connection.execute(self.drop_table_query)
-            connection.close()
 
     @staticmethod
     def expected_tables() -> int: