From c64f3fef34ada4f3ac4e8c35f3c4055d15342574 Mon Sep 17 00:00:00 2001 From: Mayur Singal <39544459+ulixius9@users.noreply.github.com> Date: Thu, 10 Feb 2022 20:52:29 +0530 Subject: [PATCH] Fix #2678: Fixed vertica import errors and invalid dataTypeDisplay (#2709) Co-authored-by: Mayur SIngal --- .../resources/json/schema/entity/data/table.json | 3 ++- .../src/metadata/ingestion/source/sql_source.py | 8 +++++--- ingestion/src/metadata/ingestion/source/vertica.py | 13 +++++++++++-- ingestion/src/metadata/utils/column_type_parser.py | 1 + 4 files changed, 19 insertions(+), 6 deletions(-) diff --git a/catalog-rest-service/src/main/resources/json/schema/entity/data/table.json b/catalog-rest-service/src/main/resources/json/schema/entity/data/table.json index 7b557a51e56..21403669edd 100644 --- a/catalog-rest-service/src/main/resources/json/schema/entity/data/table.json +++ b/catalog-rest-service/src/main/resources/json/schema/entity/data/table.json @@ -68,7 +68,8 @@ "SET", "GEOGRAPHY", "ENUM", - "JSON" + "JSON", + "UUID" ] }, "constraint": { diff --git a/ingestion/src/metadata/ingestion/source/sql_source.py b/ingestion/src/metadata/ingestion/source/sql_source.py index 4b2654d0b63..31dfe61120e 100644 --- a/ingestion/src/metadata/ingestion/source/sql_source.py +++ b/ingestion/src/metadata/ingestion/source/sql_source.py @@ -522,13 +522,15 @@ class SQLSource(Source[OMetaDatabaseAndTable]): repr(column["type"]), column["name"] ) ) - col_data_length = ( - 1 if col_data_length is None else col_data_length - ) dataTypeDisplay = ( f"{data_type_display}" if data_type_display else "{}({})".format(col_type, col_data_length) + if col_data_length + else col_type + ) + col_data_length = ( + 1 if col_data_length is None else col_data_length ) om_column = Column( name=column["name"], diff --git a/ingestion/src/metadata/ingestion/source/vertica.py b/ingestion/src/metadata/ingestion/source/vertica.py index 8ea25f11a82..3e01ffd4d6d 100644 --- a/ingestion/src/metadata/ingestion/source/vertica.py +++ b/ingestion/src/metadata/ingestion/source/vertica.py @@ -12,8 +12,10 @@ import re from textwrap import dedent -from sqlalchemy import exc, sql +from sqlalchemy import exc, sql, util from sqlalchemy.engine import reflection +from sqlalchemy.sql import sqltypes +from sqlalchemy.sql.sqltypes import VARCHAR, String from sqlalchemy_vertica.base import VerticaDialect from metadata.ingestion.ometa.openmetadata_rest import MetadataServerConfig @@ -26,6 +28,13 @@ from metadata.utils.sql_queries import ( ) +class UUID(String): + + """The SQL UUID type.""" + + __visit_name__ = "UUID" + + @reflection.cache def get_columns(self, connection, table_name, schema=None, **kw): if schema is not None: @@ -129,7 +138,7 @@ def _get_column_info( args = () elif charlen: args = (int(charlen),) - + self.ischema_names["UUID"] = UUID if attype.upper() in self.ischema_names: coltype = self.ischema_names[attype.upper()] else: diff --git a/ingestion/src/metadata/utils/column_type_parser.py b/ingestion/src/metadata/utils/column_type_parser.py index a5e99b37ce8..15dbee6ae9e 100644 --- a/ingestion/src/metadata/utils/column_type_parser.py +++ b/ingestion/src/metadata/utils/column_type_parser.py @@ -132,6 +132,7 @@ class ColumnTypeParser: "VARIANT": "JSON", "XML": "BINARY", "XMLTYPE": "BINARY", + "UUID": "UUID", } _COMPLEX_TYPE = re.compile("^(struct|map|array|uniontype)")