From 07ae09bed147b42864bf8c43b6adacfe7fc1eb52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tr=C3=B8yel?= Date: Mon, 28 Mar 2022 18:25:18 +0200 Subject: [PATCH] lower model references to match between dbt and database (spark) (#3695) * lower model references to match between dbt and database (spark) Add lower on model_fqn and get_table_fqn to increase matching between dbt and database references on Spark. * Update sql_source.py move lower out of general function and apply only on node matching * Update sql_source.py Co-authored-by: Ayush Shah --- ingestion/src/metadata/ingestion/source/sql_source.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ingestion/src/metadata/ingestion/source/sql_source.py b/ingestion/src/metadata/ingestion/source/sql_source.py index 084267b470a..133172fa8d4 100644 --- a/ingestion/src/metadata/ingestion/source/sql_source.py +++ b/ingestion/src/metadata/ingestion/source/sql_source.py @@ -418,7 +418,7 @@ class SQLSource(Source[OMetaDatabaseAndTable]): columns=columns, upstream=upstream_nodes, ) - model_fqdn = f"{schema}.{model_name}" + model_fqdn = f"{schema}.{model_name}".lower() except Exception as err: logger.debug(traceback.print_exc()) logger.error(err) @@ -433,7 +433,7 @@ class SQLSource(Source[OMetaDatabaseAndTable]): table = table.replace(".", "_DOT_") table_fqn = self.get_table_fqn( self.config.service_name, database, table - ) + ).lower() upstream_nodes.append(table_fqn) except Exception as err: # pylint: disable=broad-except logger.error( @@ -443,7 +443,7 @@ class SQLSource(Source[OMetaDatabaseAndTable]): return upstream_nodes def _get_data_model(self, schema, table_name): - table_fqn = f"{schema}.{table_name}" + table_fqn = f"{schema}.{table_name}".lower() if table_fqn in self.data_models: model = self.data_models[table_fqn] return model