yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-07-15 04:59:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

MINOR: databricks describe table query update (#17396)

Browse Source 
* databricks describe table query update

* remove extra args

---------

Co-authored-by: Ayush Shah <ayush@getcollate.io>
This commit is contained in:
harshsoni2024 2024-08-12 17:35:04 +05:30 committed by GitHub
parent 6a31c579f6
commit c84b77859e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 11 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
ingestion/src/metadata/ingestion/source/database/databricks/metadata.py
View File

@ -295,7 +295,8 @@ def get_table_names(
else:
table_name = row[0]
if schema:
table_type = get_table_type(connection, schema, table_name)
database = kw.get("db_name")
table_type = get_table_type(connection, database, schema, table_name)
if not table_type or table_type == "FOREIGN":
# skip the table if it's foreign table / error in fetching table_type
logger.debug(
@ -310,20 +311,23 @@ def get_table_names(
return [table for table in tables if table not in views]
def get_table_type(connection, schema, table):
def get_table_type(connection, database, schema, table):
"""get table type (regular/foreign)"""
try:
if database:
query = DATABRICKS_GET_TABLE_COMMENTS.format(
schema_name=schema, table_name=table
database_name=database, schema_name=schema, table_name=table
)
else:
query = f"DESCRIBE TABLE EXTENDED {schema}.{table}"
rows = connection.execute(query)
for row in rows:
row_dict = dict(row)
if row_dict.get("col_name") == "Type":
# get type of table
return row_dict.get("data_type")
except Exception:
pass
except DatabaseError as err:
logger.error(f"Failed to fetch table type for table {table} due to: {err}")
return