yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-11-09 07:23:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

lower model references to match between dbt and database (spark) (#3695)

Browse Source 
* lower model references to match between dbt and database (spark)

Add lower on model_fqn and get_table_fqn to increase matching between dbt and database references on Spark.

* Update sql_source.py

move lower out of general function and apply only on node matching

* Update sql_source.py

Co-authored-by: Ayush Shah <ayush@getcollate.io>
This commit is contained in:
Trøyel 2022-03-28 18:25:18 +02:00 committed by GitHub
parent 144e0b3cf3
commit 07ae09bed1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
ingestion/src/metadata/ingestion/source/sql_source.py
View File

@ -418,7 +418,7 @@ class SQLSource(Source[OMetaDatabaseAndTable]):
columns=columns, columns=columns,
upstream=upstream_nodes, upstream=upstream_nodes,
) )
model_fqdn = f"{schema}.{model_name}" model_fqdn = f"{schema}.{model_name}".lower()
except Exception as err: except Exception as err:
logger.debug(traceback.print_exc()) logger.debug(traceback.print_exc())
logger.error(err) logger.error(err)
@ -433,7 +433,7 @@ class SQLSource(Source[OMetaDatabaseAndTable]):
table = table.replace(".", "_DOT_") table = table.replace(".", "_DOT_")
table_fqn = self.get_table_fqn( table_fqn = self.get_table_fqn(
self.config.service_name, database, table self.config.service_name, database, table
) ).lower()
upstream_nodes.append(table_fqn) upstream_nodes.append(table_fqn)
except Exception as err: # pylint: disable=broad-except except Exception as err: # pylint: disable=broad-except
logger.error( logger.error(
@ -443,7 +443,7 @@ class SQLSource(Source[OMetaDatabaseAndTable]):
return upstream_nodes return upstream_nodes
def _get_data_model(self, schema, table_name): def _get_data_model(self, schema, table_name):
table_fqn = f"{schema}.{table_name}" table_fqn = f"{schema}.{table_name}".lower()
if table_fqn in self.data_models: if table_fqn in self.data_models:
model = self.data_models[table_fqn] model = self.data_models[table_fqn]
return model return model