From 71927cc30b9968d2b16d24e9557d6e24b8716d54 Mon Sep 17 00:00:00 2001 From: Mayur Singal <39544459+ulixius9@users.noreply.github.com> Date: Mon, 24 Mar 2025 10:46:18 +0530 Subject: [PATCH] MINOR: Fix mariadb profiling with Time datatype (#20376) --- .../orm/converter/converter_registry.py | 2 + .../orm/converter/mariadb/converter.py | 28 ++++++++++ .../src/metadata/profiler/orm/registry.py | 2 + .../profiler/orm/types/custom_time.py | 54 +++++++++++++++++++ 4 files changed, 86 insertions(+) create mode 100644 ingestion/src/metadata/profiler/orm/converter/mariadb/converter.py create mode 100644 ingestion/src/metadata/profiler/orm/types/custom_time.py diff --git a/ingestion/src/metadata/profiler/orm/converter/converter_registry.py b/ingestion/src/metadata/profiler/orm/converter/converter_registry.py index 26ad4e80cef..3b95ff4dd1c 100644 --- a/ingestion/src/metadata/profiler/orm/converter/converter_registry.py +++ b/ingestion/src/metadata/profiler/orm/converter/converter_registry.py @@ -20,6 +20,7 @@ from metadata.generated.schema.entity.services.databaseService import ( from metadata.profiler.orm.converter.azuresql.converter import AzureSqlMapTypes from metadata.profiler.orm.converter.bigquery.converter import BigqueryMapTypes from metadata.profiler.orm.converter.common import CommonMapTypes +from metadata.profiler.orm.converter.mariadb.converter import MariaDBMapTypes from metadata.profiler.orm.converter.mssql.converter import MssqlMapTypes from metadata.profiler.orm.converter.redshift.converter import RedshiftMapTypes from metadata.profiler.orm.converter.snowflake.converter import SnowflakeMapTypes @@ -30,3 +31,4 @@ converter_registry[DatabaseServiceType.Snowflake] = SnowflakeMapTypes converter_registry[DatabaseServiceType.Redshift] = RedshiftMapTypes converter_registry[DatabaseServiceType.Mssql] = MssqlMapTypes converter_registry[DatabaseServiceType.AzureSQL] = AzureSqlMapTypes +converter_registry[DatabaseServiceType.MariaDB] = MariaDBMapTypes diff --git a/ingestion/src/metadata/profiler/orm/converter/mariadb/converter.py b/ingestion/src/metadata/profiler/orm/converter/mariadb/converter.py new file mode 100644 index 00000000000..17a344157a6 --- /dev/null +++ b/ingestion/src/metadata/profiler/orm/converter/mariadb/converter.py @@ -0,0 +1,28 @@ +# Copyright 2025 Collate +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Converter logic to transform an OpenMetadata Table Entity for MariaDB +to an SQLAlchemy ORM class. +""" + +from metadata.generated.schema.entity.data.table import DataType +from metadata.profiler.orm.converter.common import CommonMapTypes +from metadata.profiler.orm.registry import CustomTypes + + +class MariaDBMapTypes(CommonMapTypes): + def __init__(self) -> None: + self._TYPE_MAP.update( + { + DataType.TIME: CustomTypes.TIME.value, + } + ) diff --git a/ingestion/src/metadata/profiler/orm/registry.py b/ingestion/src/metadata/profiler/orm/registry.py index 7020450db9c..319d6ff0a5b 100644 --- a/ingestion/src/metadata/profiler/orm/registry.py +++ b/ingestion/src/metadata/profiler/orm/registry.py @@ -28,6 +28,7 @@ from metadata.profiler.orm.types.custom_datetimerange import CustomDateTimeRange from metadata.profiler.orm.types.custom_hex_byte_string import HexByteString from metadata.profiler.orm.types.custom_image import CustomImage from metadata.profiler.orm.types.custom_ip import CustomIP +from metadata.profiler.orm.types.custom_time import CustomTime from metadata.profiler.orm.types.custom_timestamp import CustomTimestamp from metadata.profiler.orm.types.undetermined_type import UndeterminedType from metadata.profiler.orm.types.uuid import UUIDString @@ -44,6 +45,7 @@ class CustomTypes(TypeRegistry): IP = CustomIP SQADATETIMERANGE = CustomDateTimeRange UNDETERMINED = UndeterminedType + TIME = CustomTime class PythonDialects(Enum): diff --git a/ingestion/src/metadata/profiler/orm/types/custom_time.py b/ingestion/src/metadata/profiler/orm/types/custom_time.py new file mode 100644 index 00000000000..070ba0c1af6 --- /dev/null +++ b/ingestion/src/metadata/profiler/orm/types/custom_time.py @@ -0,0 +1,54 @@ +# Copyright 2025 Collate +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=abstract-method + +""" +Expand sqlalchemy types to map them to OpenMetadata DataType +""" +import datetime + +from sqlalchemy.sql.sqltypes import TIME, TypeDecorator + +from metadata.utils.logger import profiler_logger + +logger = profiler_logger() + + +class CustomTime(TypeDecorator): + """ + Convert int time to timedelta object + """ + + __visit_name__ = "TIME" + impl = TIME + cache_ok = True + + def result_processor(self, dialect, coltype): + time = datetime.time + + def process(value): + # convert from a timedelta value + if value is not None: + if isinstance(value, int): + value = datetime.timedelta(seconds=value) + microseconds = value.microseconds + seconds = value.seconds + minutes = seconds // 60 + return time( + minutes // 60, + minutes % 60, + seconds - minutes * 60, + microsecond=microseconds, + ) + return None + + return process