yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-10-31 02:29:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

MINOR: athena usage & lineage minor fix (#16156)

Browse Source 
* MINOR: athena usage & lineage minor fix

* py format
This commit is contained in:
Mayur Singal 2024-05-07 22:29:39 +05:30 committed by GitHub
parent 5cae2137d3
commit cf9b1a55ec
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 8 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
ingestion/src/metadata/ingestion/source/database/athena/client.py
View File

@ -74,7 +74,7 @@ class AthenaLakeFormationClient:
return LFTags(**response) return LFTags(**response)
except Exception as exc: except Exception as exc:
logger.debug(traceback.format_exc()) logger.debug(traceback.format_exc())
logger.warning( logger.debug(
f"Unable to get lf tags for table resource [{table_name}] due to: {exc}" f"Unable to get lf tags for table resource [{table_name}] due to: {exc}"
) )
return LFTags() return LFTags()

2
ingestion/src/metadata/ingestion/source/database/athena/lineage.py
View File

@ -15,6 +15,7 @@ from typing import Iterable, Optional
from metadata.generated.schema.type.tableQuery import TableQuery from metadata.generated.schema.type.tableQuery import TableQuery
from metadata.ingestion.source.database.athena.query_parser import ( from metadata.ingestion.source.database.athena.query_parser import (
QUERY_SUCCESS_STATUS,
AthenaQueryParserSource, AthenaQueryParserSource,
) )
from metadata.ingestion.source.database.lineage_source import LineageSource from metadata.ingestion.source.database.lineage_source import LineageSource
@ -37,6 +38,7 @@ class AthenaLineageSource(AthenaQueryParserSource, LineageSource):
if ( if (
query.Status.SubmissionDateTime.date() >= self.start.date() query.Status.SubmissionDateTime.date() >= self.start.date()
and self.is_not_dbt_or_om_query(query.Query) and self.is_not_dbt_or_om_query(query.Query)
and query.Status.State.upper() == QUERY_SUCCESS_STATUS
): ):
yield TableQuery( yield TableQuery(
query=query.Query, query=query.Query,

2
ingestion/src/metadata/ingestion/source/database/athena/models.py
View File

@ -23,7 +23,7 @@ class QueryExecutionIdsResponse(BaseModel):
class Status(BaseModel): class Status(BaseModel):
State: Optional[str] State: Optional[str] = "FAILED" # Default value
SubmissionDateTime: Optional[datetime] SubmissionDateTime: Optional[datetime]

2
ingestion/src/metadata/ingestion/source/database/athena/query_parser.py
View File

@ -41,6 +41,8 @@ ATHENA_QUERY_PAGINATOR_LIMIT = 50
ATHENA_ENABLED_WORK_GROUP_STATE = "ENABLED" ATHENA_ENABLED_WORK_GROUP_STATE = "ENABLED"
QUERY_SUCCESS_STATUS = "SUCCEEDED"
class AthenaQueryParserSource(QueryParserSource, ABC): class AthenaQueryParserSource(QueryParserSource, ABC):
""" """

2
ingestion/src/metadata/ingestion/source/database/athena/usage.py
View File

@ -15,6 +15,7 @@ from typing import Iterable
from metadata.generated.schema.type.tableQuery import TableQueries, TableQuery from metadata.generated.schema.type.tableQuery import TableQueries, TableQuery
from metadata.ingestion.source.database.athena.query_parser import ( from metadata.ingestion.source.database.athena.query_parser import (
QUERY_SUCCESS_STATUS,
AthenaQueryParserSource, AthenaQueryParserSource,
) )
from metadata.ingestion.source.database.usage_source import UsageSource from metadata.ingestion.source.database.usage_source import UsageSource
@ -56,6 +57,7 @@ class AthenaUsageSource(AthenaQueryParserSource, UsageSource):
for query in query_list.QueryExecutions for query in query_list.QueryExecutions
if query.Status if query.Status
and query.Query and query.Query
and query.Status.State.upper() == QUERY_SUCCESS_STATUS
and query.Status.SubmissionDateTime.date() >= self.start.date() and query.Status.SubmissionDateTime.date() >= self.start.date()
and self.is_not_dbt_or_om_query(query.Query) and self.is_not_dbt_or_om_query(query.Query)
] ]