diff --git a/ingestion/src/metadata/ingestion/source/database/column_type_parser.py b/ingestion/src/metadata/ingestion/source/database/column_type_parser.py index 7ab833f36cf..d72e57aa613 100644 --- a/ingestion/src/metadata/ingestion/source/database/column_type_parser.py +++ b/ingestion/src/metadata/ingestion/source/database/column_type_parser.py @@ -216,6 +216,7 @@ class ColumnTypeParser: "MACADDR8": "MACADDR", "CIDR": "CIDR", "INET": "INET", + "TSRANGE": "DATETIMERANGE", # ORACLE "BINARY_DOUBLE": "DOUBLE", "BINARY_FLOAT": "FLOAT", diff --git a/ingestion/src/metadata/ingestion/source/sqa_types.py b/ingestion/src/metadata/ingestion/source/sqa_types.py index 50aa36f18da..3095cc40669 100644 --- a/ingestion/src/metadata/ingestion/source/sqa_types.py +++ b/ingestion/src/metadata/ingestion/source/sqa_types.py @@ -28,6 +28,12 @@ class SQAStruct(types.String): """ +class SQADateTimeRange(types.String): + """ + Custom DateTimeRange type definition + """ + + class SQAUnion(types.String): """ Custom Struct type definition diff --git a/ingestion/src/metadata/profiler/orm/converter/common.py b/ingestion/src/metadata/profiler/orm/converter/common.py index 0fb2ba00a2f..f3e45890f1d 100644 --- a/ingestion/src/metadata/profiler/orm/converter/common.py +++ b/ingestion/src/metadata/profiler/orm/converter/common.py @@ -67,6 +67,7 @@ class CommonMapTypes: DataType.IMAGE: CustomTypes.IMAGE.value, DataType.IPV4: CustomTypes.IP.value, DataType.IPV6: CustomTypes.IP.value, + DataType.DATETIMERANGE: CustomTypes.SQADATETIMERANGE.value, } def map_types(self, col: Column, table_service_type): diff --git a/ingestion/src/metadata/profiler/orm/registry.py b/ingestion/src/metadata/profiler/orm/registry.py index e2e4d1be68f..f0dcbbba957 100644 --- a/ingestion/src/metadata/profiler/orm/registry.py +++ b/ingestion/src/metadata/profiler/orm/registry.py @@ -21,6 +21,7 @@ from metadata.generated.schema.entity.data.table import DataType from metadata.ingestion.source import sqa_types from metadata.profiler.orm.types.bytea_to_string import ByteaToHex from metadata.profiler.orm.types.custom_array import CustomArray +from metadata.profiler.orm.types.custom_datetimerange import CustomDateTimeRange from metadata.profiler.orm.types.custom_hex_byte_string import HexByteString from metadata.profiler.orm.types.custom_image import CustomImage from metadata.profiler.orm.types.custom_ip import CustomIP @@ -38,6 +39,7 @@ class CustomTypes(TypeRegistry): TIMESTAMP = CustomTimestamp IMAGE = CustomImage IP = CustomIP + SQADATETIMERANGE = CustomDateTimeRange class Dialects(Enum): diff --git a/ingestion/src/metadata/profiler/orm/types/custom_datetimerange.py b/ingestion/src/metadata/profiler/orm/types/custom_datetimerange.py new file mode 100644 index 00000000000..ed753721488 --- /dev/null +++ b/ingestion/src/metadata/profiler/orm/types/custom_datetimerange.py @@ -0,0 +1,46 @@ +# Copyright 2021 Collate +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=abstract-method + +""" +Expand sqlalchemy types to map them to OpenMetadata DataType +""" +from sqlalchemy.sql.sqltypes import String, TypeDecorator + +from metadata.utils.logger import profiler_logger + +logger = profiler_logger() + + +class CustomDateTimeRange(TypeDecorator): + """ + Convert CustomDateTimeRange + """ + + impl = String + cache_ok = True + + @property + def python_type(self): + return str + + def process_result_value(self, value, dialect): + """This is executed during result retrieval + Needs to be done, as DateTimeRange returns DateTimeRange object which is not json serializable + Args: + value: database record + dialect: database dialect + Returns: + python conversion DateTimeRange for sample data + """ + + return f"{[value.lower, value.upper]}" diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/data/table.json b/openmetadata-spec/src/main/resources/json/schema/entity/data/table.json index e31178bb58a..f54e92a5b8a 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/data/table.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/data/table.json @@ -150,7 +150,8 @@ "NTEXT", "IMAGE", "IPV4", - "IPV6" + "IPV6", + "DATETIMERANGE" ] }, "constraint": {