From d30619a81fe3651deb8eac13e1c156738af81dd9 Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Wed, 20 Apr 2022 09:14:14 +0200 Subject: [PATCH] Fix #3972 - Add dialects to LenFn and not use FQN in converter (#4262) Fix #3972 - Add dialects to LenFn and not use FQN in converter (#4262) --- ingestion/src/metadata/orm_profiler/orm/converter.py | 8 +++++--- .../src/metadata/orm_profiler/orm/functions/length.py | 4 +++- ingestion/src/metadata/orm_profiler/orm/registry.py | 2 +- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/ingestion/src/metadata/orm_profiler/orm/converter.py b/ingestion/src/metadata/orm_profiler/orm/converter.py index 267f2b2881f..1842eae958a 100644 --- a/ingestion/src/metadata/orm_profiler/orm/converter.py +++ b/ingestion/src/metadata/orm_profiler/orm/converter.py @@ -95,7 +95,6 @@ def ometa_to_orm( We are building the class dynamically using `type` and passing SQLAlchemy `Base` class as the bases tuple for inheritance. - TODO: Remove the dialect once we solve the hierarchy service.db.schema.table. Check #3529 """ cols = { @@ -103,14 +102,17 @@ def ometa_to_orm( for idx, col in enumerate(table.columns) } + schema_name = get_schema_name(schema) + orm_name = f"{schema_name}_{table.name}".replace(".", "_") + # Type takes positional arguments in the form of (name, bases, dict) orm = type( - table.fullyQualifiedName.__root__.replace(".", "_"), # Output class name + orm_name, # Output class name (Base,), # SQLAlchemy declarative base { "__tablename__": str(table.name.__root__), "__table_args__": { - "schema": get_schema_name(schema), + "schema": schema_name, "extend_existing": True, # Recreates the table ORM object if it already exists. Useful for testing }, **cols, diff --git a/ingestion/src/metadata/orm_profiler/orm/functions/length.py b/ingestion/src/metadata/orm_profiler/orm/functions/length.py index 0cc16d2add0..46a104b5991 100644 --- a/ingestion/src/metadata/orm_profiler/orm/functions/length.py +++ b/ingestion/src/metadata/orm_profiler/orm/functions/length.py @@ -33,8 +33,10 @@ def _(element, compiler, **kw): @compiles(LenFn, Dialects.SQLite) @compiles(LenFn, Dialects.Vertica) -@compiles(LenFn, Dialects.Hive) # For some reason hive's dialect is in bytes... +@compiles(LenFn, Dialects.Hive) @compiles(LenFn, Dialects.Postgres) @compiles(LenFn, Dialects.Databricks) +@compiles(LenFn, Dialects.MySQL) +@compiles(LenFn, Dialects.MariaDB) def _(element, compiler, **kw): return "LENGTH(%s)" % compiler.process(element.clauses, **kw) diff --git a/ingestion/src/metadata/orm_profiler/orm/registry.py b/ingestion/src/metadata/orm_profiler/orm/registry.py index 9561c9319a0..1f4362a1a78 100644 --- a/ingestion/src/metadata/orm_profiler/orm/registry.py +++ b/ingestion/src/metadata/orm_profiler/orm/registry.py @@ -34,7 +34,7 @@ class Dialects(Enum): and profiling data. """ - Hive = b"hive" + Hive = b"hive" # Hive requires bytes Postgres = "postgresql" BigQuery = "bigquery" MySQL = "mysql"