diff --git a/ingestion/src/metadata/ingestion/source/database/athena/client.py b/ingestion/src/metadata/ingestion/source/database/athena/client.py index 48764b987c6..b741266ece3 100644 --- a/ingestion/src/metadata/ingestion/source/database/athena/client.py +++ b/ingestion/src/metadata/ingestion/source/database/athena/client.py @@ -49,7 +49,7 @@ class AthenaLakeFormationClient: return lf_tags.LFTagOnDatabase except Exception as exc: logger.debug(traceback.format_exc()) - logger.error( + logger.warning( f"Unable to get lf tags for database resource [{name}] due to: {exc}" ) return None @@ -76,7 +76,7 @@ class AthenaLakeFormationClient: return LFTags(**response) except Exception as exc: logger.debug(traceback.format_exc()) - logger.error( + logger.warning( f"Unable to get lf tags for table resource [{table_name}] due to: {exc}" ) return None diff --git a/ingestion/tests/cli_e2e/test_cli_athena.py b/ingestion/tests/cli_e2e/test_cli_athena.py index 037854e95f3..99a2f35f049 100644 --- a/ingestion/tests/cli_e2e/test_cli_athena.py +++ b/ingestion/tests/cli_e2e/test_cli_athena.py @@ -17,6 +17,7 @@ from typing import List import pytest +from metadata.ingestion.api.status import Status from metadata.workflow.metadata import MetadataWorkflow from .base.e2e_types import E2EType @@ -67,7 +68,7 @@ class AthenaCliTest(CliCommonDB.TestSuite): @staticmethod def fqn_created_table() -> str: - return "e2e_athena.database_name.testdatalake_db.customers" + return "e2e_athena.database_name.e2e_db.customers" @staticmethod def fqn_deleted_table() -> None: @@ -75,7 +76,7 @@ class AthenaCliTest(CliCommonDB.TestSuite): @staticmethod def get_includes_schemas() -> List[str]: - return ["testdatalake_db"] + return ["e2e_db"] @staticmethod def get_includes_tables() -> List[str]: @@ -87,7 +88,7 @@ class AthenaCliTest(CliCommonDB.TestSuite): @staticmethod def expected_filtered_schema_includes() -> int: - return 4 + return 5 @staticmethod def expected_filtered_schema_excludes() -> int: @@ -95,15 +96,15 @@ class AthenaCliTest(CliCommonDB.TestSuite): @staticmethod def expected_filtered_table_includes() -> int: - return 29 + return 7 @staticmethod def expected_filtered_table_excludes() -> int: - return 2 + return 7 @staticmethod def expected_filtered_mix() -> int: - return 6 + return 7 def retrieve_lineage(self, entity_fqn: str) -> dict: pass @@ -128,3 +129,20 @@ class AthenaCliTest(CliCommonDB.TestSuite): result = self.run_command("profile") sink_status, source_status = self.retrieve_statuses(result) self.assert_for_table_with_profiler(source_status, sink_status) + + def assert_for_vanilla_ingestion( + self, source_status: Status, sink_status: Status + ) -> None: + self.assertTrue(len(source_status.failures) == 0) + self.assertTrue(len(source_status.warnings) == 0) + self.assertTrue(len(source_status.filtered) == 5) + self.assertTrue( + (len(source_status.records) + len(source_status.updated_records)) + >= self.expected_tables() + ) + self.assertTrue(len(sink_status.failures) == 0) + self.assertTrue(len(sink_status.warnings) == 0) + self.assertTrue( + (len(sink_status.records) + len(sink_status.updated_records)) + >= self.expected_tables() + )