yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-10-15 10:48:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixed athena e2e db (#14774)

Browse Source
This commit is contained in:
Onkar Ravgan 2024-01-19 11:51:38 +05:30 committed by GitHub
parent 9c4ec6c760
commit f10b5a3617
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 26 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
ingestion/src/metadata/ingestion/source/database/athena/client.py
View File

@ -49,7 +49,7 @@ class AthenaLakeFormationClient:
return lf_tags.LFTagOnDatabase
except Exception as exc:
logger.debug(traceback.format_exc())
logger.error(
logger.warning(
f"Unable to get lf tags for database resource [{name}] due to: {exc}"
)
return None
@ -76,7 +76,7 @@ class AthenaLakeFormationClient:
return LFTags(**response)
except Exception as exc:
logger.debug(traceback.format_exc())
logger.error(
logger.warning(
f"Unable to get lf tags for table resource [{table_name}] due to: {exc}"
)
return None

30
ingestion/tests/cli_e2e/test_cli_athena.py
View File

@ -17,6 +17,7 @@ from typing import List
import pytest
from metadata.ingestion.api.status import Status
from metadata.workflow.metadata import MetadataWorkflow
from .base.e2e_types import E2EType
@ -67,7 +68,7 @@ class AthenaCliTest(CliCommonDB.TestSuite):
@staticmethod
def fqn_created_table() -> str:
return "e2e_athena.database_name.testdatalake_db.customers"
return "e2e_athena.database_name.e2e_db.customers"
@staticmethod
def fqn_deleted_table() -> None:
@ -75,7 +76,7 @@ class AthenaCliTest(CliCommonDB.TestSuite):
@staticmethod
def get_includes_schemas() -> List[str]:
return ["testdatalake_db"]
return ["e2e_db"]
@staticmethod
def get_includes_tables() -> List[str]:
@ -87,7 +88,7 @@ class AthenaCliTest(CliCommonDB.TestSuite):
@staticmethod
def expected_filtered_schema_includes() -> int:
return 4
return 5
@staticmethod
def expected_filtered_schema_excludes() -> int:
@ -95,15 +96,15 @@ class AthenaCliTest(CliCommonDB.TestSuite):
@staticmethod
def expected_filtered_table_includes() -> int:
return 29
return 7
@staticmethod
def expected_filtered_table_excludes() -> int:
return 2
return 7
@staticmethod
def expected_filtered_mix() -> int:
return 6
return 7
def retrieve_lineage(self, entity_fqn: str) -> dict:
pass
@ -128,3 +129,20 @@ class AthenaCliTest(CliCommonDB.TestSuite):
result = self.run_command("profile")
sink_status, source_status = self.retrieve_statuses(result)
self.assert_for_table_with_profiler(source_status, sink_status)
def assert_for_vanilla_ingestion(
self, source_status: Status, sink_status: Status
) -> None:
self.assertTrue(len(source_status.failures) == 0)
self.assertTrue(len(source_status.warnings) == 0)
self.assertTrue(len(source_status.filtered) == 5)
self.assertTrue(
(len(source_status.records) + len(source_status.updated_records))
>= self.expected_tables()
)
self.assertTrue(len(sink_status.failures) == 0)
self.assertTrue(len(sink_status.warnings) == 0)
self.assertTrue(
(len(sink_status.records) + len(sink_status.updated_records))
>= self.expected_tables()
)