From cdca199ec89043d4443627b95e65594d3ff402dc Mon Sep 17 00:00:00 2001 From: Suman Maharana Date: Wed, 5 Jun 2024 11:11:04 +0530 Subject: [PATCH] Fix #16370 - Added view/DDL definition rule (#16486) * added rule includeDDL * code optimize * review changes --- .../source/database/bigquery/metadata.py | 2 +- .../source/database/clickhouse/metadata.py | 15 +++++++++------ .../source/database/common_db_source.py | 15 ++++++++++----- .../source/database/oracle/metadata.py | 19 +++++++++++-------- 4 files changed, 31 insertions(+), 20 deletions(-) diff --git a/ingestion/src/metadata/ingestion/source/database/bigquery/metadata.py b/ingestion/src/metadata/ingestion/source/database/bigquery/metadata.py index 4ed1a29e753..e77f42a4fa6 100644 --- a/ingestion/src/metadata/ingestion/source/database/bigquery/metadata.py +++ b/ingestion/src/metadata/ingestion/source/database/bigquery/metadata.py @@ -630,7 +630,7 @@ class BigquerySource( self.connection, table_name, schema_name ) schema_definition = ( - str(schema_definition.strip()) + str(schema_definition).strip() if schema_definition is not None else None ) diff --git a/ingestion/src/metadata/ingestion/source/database/clickhouse/metadata.py b/ingestion/src/metadata/ingestion/source/database/clickhouse/metadata.py index e33739d4f4b..1c6c999e161 100644 --- a/ingestion/src/metadata/ingestion/source/database/clickhouse/metadata.py +++ b/ingestion/src/metadata/ingestion/source/database/clickhouse/metadata.py @@ -156,16 +156,19 @@ class ClickhouseSource(CommonDbSourceService): if table_type in {TableType.View, TableType.MaterializedView}: definition_fn = inspector.get_view_definition schema_definition = definition_fn(table_name, schema_name) - else: - schema_definition = inspector.get_table_ddl( - self.connection, table_name, schema_name + return ( + str(schema_definition).strip() + if schema_definition is not None + else None ) - schema_definition = ( - str(schema_definition.strip()) + schema_definition = inspector.get_table_ddl( + self.connection, table_name, schema_name + ) + return ( + str(schema_definition).strip() if schema_definition is not None else None ) - return schema_definition except NotImplementedError: logger.warning("Schema definition not implemented") diff --git a/ingestion/src/metadata/ingestion/source/database/common_db_source.py b/ingestion/src/metadata/ingestion/source/database/common_db_source.py index b8d7910c91f..5cc677169ec 100644 --- a/ingestion/src/metadata/ingestion/source/database/common_db_source.py +++ b/ingestion/src/metadata/ingestion/source/database/common_db_source.py @@ -458,12 +458,17 @@ class CommonDbSourceService( inspector=self.inspector, ) - schema_definition = self.get_schema_definition( - table_type=table_type, - table_name=table_name, - schema_name=schema_name, - inspector=self.inspector, + schema_definition = ( + self.get_schema_definition( + table_type=table_type, + table_name=table_name, + schema_name=schema_name, + inspector=self.inspector, + ) + if self.source_config.includeDDL + else None ) + table_constraints = self.update_table_constraints( table_constraints, foreign_columns ) diff --git a/ingestion/src/metadata/ingestion/source/database/oracle/metadata.py b/ingestion/src/metadata/ingestion/source/database/oracle/metadata.py index 2857b9af088..045aa51073f 100644 --- a/ingestion/src/metadata/ingestion/source/database/oracle/metadata.py +++ b/ingestion/src/metadata/ingestion/source/database/oracle/metadata.py @@ -154,20 +154,23 @@ class OracleSource(StoredProcedureMixin, CommonDbSourceService): schema_definition = inspector.get_table_ddl( self.connection, table_name, schema_name ) + return ( + str(schema_definition).strip() + if schema_definition is not None + else None + ) - else: - definition_fn = inspector.get_view_definition - if table_type == TableType.MaterializedView: - definition_fn = inspector.get_mview_definition + definition_fn = inspector.get_view_definition + if table_type == TableType.MaterializedView: + definition_fn = inspector.get_mview_definition - schema_definition = definition_fn(table_name, schema_name) + schema_definition = definition_fn(table_name, schema_name) - schema_definition = ( - str(schema_definition.strip()) + return ( + str(schema_definition).strip() if schema_definition is not None else None ) - return schema_definition except NotImplementedError: logger.warning("Schema definition not implemented")