diff --git a/ingestion/tests/cli_e2e/database/bigquery_multiple_project/bigquery_multiple_project.yaml b/ingestion/tests/cli_e2e/database/bigquery_multiple_project/bigquery_multiple_project.yaml index 46daf6dc150..ab961ce816f 100644 --- a/ingestion/tests/cli_e2e/database/bigquery_multiple_project/bigquery_multiple_project.yaml +++ b/ingestion/tests/cli_e2e/database/bigquery_multiple_project/bigquery_multiple_project.yaml @@ -21,6 +21,10 @@ source: sourceConfig: config: type: DatabaseMetadata + schemaFilterPattern: + includes: + - do_not_touch + - exclude_me sink: type: metadata-rest config: {} diff --git a/ingestion/tests/cli_e2e/test_cli_bigquery_multiple_project.py b/ingestion/tests/cli_e2e/test_cli_bigquery_multiple_project.py index fb82a42b14a..04765b2c5ad 100644 --- a/ingestion/tests/cli_e2e/test_cli_bigquery_multiple_project.py +++ b/ingestion/tests/cli_e2e/test_cli_bigquery_multiple_project.py @@ -14,6 +14,8 @@ Test Bigquery connector with CLI """ from typing import List +from metadata.ingestion.api.status import Status + from .base.e2e_types import E2EType from .common.test_cli_db import CliCommonDB from .common_e2e_sqa_mixins import SQACommonMethods @@ -102,15 +104,15 @@ class BigqueryCliTest(CliCommonDB.TestSuite, SQACommonMethods): @staticmethod def expected_filtered_table_includes() -> int: - return 9 - - @staticmethod - def expected_filtered_table_excludes() -> int: return 10 + @staticmethod + def expected_filtered_table_excludes() -> int: + return 11 + @staticmethod def expected_filtered_mix() -> int: - return 9 + return 10 @staticmethod def delete_queries() -> List[str]: @@ -128,6 +130,17 @@ class BigqueryCliTest(CliCommonDB.TestSuite, SQACommonMethods): """, ] + def assert_for_vanilla_ingestion( + self, source_status: Status, sink_status: Status + ) -> None: + self.assertTrue(len(source_status.failures) == 0) + self.assertTrue(len(source_status.warnings) == 0) + self.assertTrue(len(source_status.filtered) >= 9) + self.assertTrue(len(source_status.records) >= self.expected_tables()) + self.assertTrue(len(sink_status.failures) == 0) + self.assertTrue(len(sink_status.warnings) == 0) + self.assertTrue(len(sink_status.records) > self.expected_tables()) + def test_create_table_with_profiler(self) -> None: # delete table in case it exists self.delete_table_and_view() diff --git a/ingestion/tests/cli_e2e/test_cli_dbt_redshift.py b/ingestion/tests/cli_e2e/test_cli_dbt_redshift.py index 2676fd0fbfd..1f7c9e83d23 100644 --- a/ingestion/tests/cli_e2e/test_cli_dbt_redshift.py +++ b/ingestion/tests/cli_e2e/test_cli_dbt_redshift.py @@ -67,7 +67,7 @@ class DbtCliTest(CliDBTBase.TestSuite): ) -> None: self.assertTrue(len(source_status.failures) == 0) self.assertTrue(len(source_status.warnings) == 0) - self.assertTrue(len(source_status.filtered) == 9) + self.assertTrue(len(source_status.filtered) >= 10) self.assertTrue(len(source_status.records) >= self.expected_tables()) self.assertTrue(len(sink_status.failures) == 0) self.assertTrue(len(sink_status.warnings) == 0) diff --git a/ingestion/tests/cli_e2e/test_cli_mysql.py b/ingestion/tests/cli_e2e/test_cli_mysql.py index 18d2866eae7..e4424252dec 100644 --- a/ingestion/tests/cli_e2e/test_cli_mysql.py +++ b/ingestion/tests/cli_e2e/test_cli_mysql.py @@ -117,7 +117,7 @@ class MysqlCliTest(CliCommonDB.TestSuite, SQACommonMethods): @staticmethod def expected_filtered_table_includes() -> int: - return 63 + return 64 @staticmethod def expected_filtered_table_excludes() -> int: @@ -125,4 +125,4 @@ class MysqlCliTest(CliCommonDB.TestSuite, SQACommonMethods): @staticmethod def expected_filtered_mix() -> int: - return 63 + return 64