diff --git a/ingestion/src/metadata/ingestion/source/postgres.py b/ingestion/src/metadata/ingestion/source/postgres.py index 977e0f53785..bd2ed44c0d3 100644 --- a/ingestion/src/metadata/ingestion/source/postgres.py +++ b/ingestion/src/metadata/ingestion/source/postgres.py @@ -52,6 +52,21 @@ class PostgresSource(SQLSource): def get_status(self) -> SourceStatus: return self.status + def _is_partition(self, table_name: str, schema_name: str) -> bool: + cur = self.pgconn.cursor() + cur.execute( + """ + SELECT relispartition as is_partition + FROM pg_catalog.pg_class c + JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace + WHERE c.relname = %s + AND n.nspname = %s + """, + (table_name, schema_name), + ) + is_partition = cur.fetchone()[0] + return is_partition + def type_of_column_name(self, sa_type, table_name: str, column_name: str): cur = self.pgconn.cursor() schema_table = table_name.split(".") diff --git a/ingestion/src/metadata/ingestion/source/sql_source.py b/ingestion/src/metadata/ingestion/source/sql_source.py index f0b24931b11..a9488f5cca5 100644 --- a/ingestion/src/metadata/ingestion/source/sql_source.py +++ b/ingestion/src/metadata/ingestion/source/sql_source.py @@ -203,6 +203,12 @@ class SQLSource(Source[OMetaDatabaseAndTable]): "Table pattern not allowed", ) continue + if self._is_partition(table_name, schema): + self.status.filter( + f"{self.config.get_service_name()}.{table_name}", + "Table is partition", + ) + continue description = _get_table_description(schema, table_name, inspector) fqn = f"{self.config.service_name}.{schema}.{table_name}" self.database_source_state.add(fqn) @@ -317,6 +323,9 @@ class SQLSource(Source[OMetaDatabaseAndTable]): if table.fullyQualifiedName not in self.database_source_state: yield DeleteTable(table=table) + def _is_partition(self, table_name: str, schema: str) -> bool: + return False + def _parse_data_model(self): """ Get all the DBT information and feed it to the Table Entity diff --git a/ingestion/src/metadata/utils/column_type_parser.py b/ingestion/src/metadata/utils/column_type_parser.py index b6c0dbb2784..2286c6bb48d 100644 --- a/ingestion/src/metadata/utils/column_type_parser.py +++ b/ingestion/src/metadata/utils/column_type_parser.py @@ -132,6 +132,7 @@ class ColumnTypeParser: "TEXT": "TEXT", "TIME": "TIME", "TIMESTAMP WITHOUT TIME ZONE": "TIMESTAMP", + "TIMESTAMP WITH TIME ZONE": "TIMESTAMP", "TIMESTAMP": "TIMESTAMP", "TIMESTAMPTZ": "TIMESTAMP", "TIMESTAMP_NTZ": "TIMESTAMP", @@ -144,6 +145,8 @@ class ColumnTypeParser: "VARBINARY": "VARBINARY", "VARCHAR": "VARCHAR", "VARIANT": "JSON", + "JSON": "JSON", + "JSONB": "JSON", "XML": "BINARY", "XMLTYPE": "BINARY", "UUID": "UUID",