yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-07-15 21:19:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

MINOR: databricks describe table query update (#17396)

Browse Source 
* databricks describe table query update

* remove extra args

---------

Co-authored-by: Ayush Shah <ayush@getcollate.io>
This commit is contained in:
harshsoni2024 2024-08-12 17:35:04 +05:30 committed by GitHub
parent 6a31c579f6
commit c84b77859e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 11 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
ingestion/src/metadata/ingestion/source/database/databricks/metadata.py
View File

@ -295,7 +295,8 @@ def get_table_names(
else: else:
table_name = row[0] table_name = row[0]
if schema: if schema:
table_type = get_table_type(connection, schema, table_name) database = kw.get("db_name")
table_type = get_table_type(connection, database, schema, table_name)
if not table_type or table_type == "FOREIGN": if not table_type or table_type == "FOREIGN":
# skip the table if it's foreign table / error in fetching table_type # skip the table if it's foreign table / error in fetching table_type
logger.debug( logger.debug(
@ -310,20 +311,23 @@ def get_table_names(
return [table for table in tables if table not in views] return [table for table in tables if table not in views]
def get_table_type(connection, schema, table): def get_table_type(connection, database, schema, table):
"""get table type (regular/foreign)""" """get table type (regular/foreign)"""
try: try:
if database:
query = DATABRICKS_GET_TABLE_COMMENTS.format( query = DATABRICKS_GET_TABLE_COMMENTS.format(
schema_name=schema, table_name=table database_name=database, schema_name=schema, table_name=table
) )
else:
query = f"DESCRIBE TABLE EXTENDED {schema}.{table}"
rows = connection.execute(query) rows = connection.execute(query)
for row in rows: for row in rows:
row_dict = dict(row) row_dict = dict(row)
if row_dict.get("col_name") == "Type": if row_dict.get("col_name") == "Type":
# get type of table # get type of table
return row_dict.get("data_type") return row_dict.get("data_type")
except Exception: except DatabaseError as err:
pass logger.error(f"Failed to fetch table type for table {table} due to: {err}")
return return