yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2026-01-06 04:26:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixed athena import and lineage/usage validation (#12663)

Browse Source 
* Athena fixes

* fixed to lower version
This commit is contained in:
Onkar Ravgan 2023-07-31 16:07:48 +05:30 committed by GitHub
parent dacd13f6d0
commit 795498c54a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 7 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
ingestion/setup.py
View File

@ -120,7 +120,7 @@ base_requirements = {
plugins: Dict[str, Set[str]] = {
"airflow": {VERSIONS["airflow"]}, # Same as ingestion container. For development.
"amundsen": {VERSIONS["neo4j"]},
"athena": {"PyAthena[SQLAlchemy]"},
"athena": {"pyathena==2.25.2"},
"atlas": {},
"azuresql": {VERSIONS["pyodbc"]},
"azure-sso": {VERSIONS["msal"]},

11
ingestion/src/metadata/ingestion/source/database/athena/query_parser.py
View File

@ -69,11 +69,12 @@ class AthenaQueryParserSource(QueryParserSource, ABC):
paginator_response = paginator.paginate()
for response in paginator_response:
response_obj = QueryExecutionIdsResponse(**response)
query_details_response = self.client.batch_get_query_execution(
QueryExecutionIds=response_obj.QueryExecutionIds
)
query_details_list = AthenaQueryExecutionList(**query_details_response)
yield query_details_list
if response_obj.QueryExecutionIds:
query_details_response = self.client.batch_get_query_execution(
QueryExecutionIds=response_obj.QueryExecutionIds
)
query_details_list = AthenaQueryExecutionList(**query_details_response)
yield query_details_list
query_limit -= 1
if not query_limit:
break