From 0c23b68d40f08339c1ecbec01889724bcdfc49c0 Mon Sep 17 00:00:00 2001 From: Tom Vijlbrief Date: Mon, 1 Nov 2021 15:22:40 +0100 Subject: [PATCH] Handle . in column names (#989) * Handle . in column names * Handle . for profiling --- ingestion/src/metadata/ingestion/sink/metadata_rest.py | 5 +++++ ingestion/src/metadata/ingestion/source/sql_source.py | 7 +++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/ingestion/src/metadata/ingestion/sink/metadata_rest.py b/ingestion/src/metadata/ingestion/sink/metadata_rest.py index 197eeb218b0..6491791052b 100644 --- a/ingestion/src/metadata/ingestion/sink/metadata_rest.py +++ b/ingestion/src/metadata/ingestion/sink/metadata_rest.py @@ -160,6 +160,11 @@ class MetadataRestSink(Sink): table_id=created_table.id, sample_data=table_and_db.table.sampleData ) if table_and_db.table.tableProfile is not None: + for tp in table_and_db.table.tableProfile: + for pd in tp: + if pd[0] == "columnProfile": + for col in pd[1]: + col.name = col.name.replace(".", "_DOT_") self.client.ingest_table_profile_data( table_id=created_table.id, table_profile=table_and_db.table.tableProfile, diff --git a/ingestion/src/metadata/ingestion/source/sql_source.py b/ingestion/src/metadata/ingestion/source/sql_source.py index 62546e33e56..cd29fe033b0 100644 --- a/ingestion/src/metadata/ingestion/source/sql_source.py +++ b/ingestion/src/metadata/ingestion/source/sql_source.py @@ -177,7 +177,9 @@ class SQLSource(Source): query = self.config.query.format(schema, table) logger.info(query) results = self.connection.execute(query) - cols = list(results.keys()) + cols = [] + for col in results.keys(): + cols.append(col.replace(".", "_DOT_")) rows = [] for r in results: row = list(r) @@ -347,7 +349,8 @@ class SQLSource(Source): try: for column in columns: if "." in column["name"]: - continue + logger.info(f"Found '.' in {column['name']}") + column["name"] = column["name"].replace(".", "_DOT_") children = None data_type_display = None col_data_length = None