yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-10-18 12:18:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixes #20956: Teradata profiler (#21292)

Browse Source 
* add teradata functions

* fix teradata schema

* reformat code

* change random approach for teradata

---------

Co-authored-by: Teddy <teddy.crepineau@gmail.com>
This commit is contained in:
gpby 2025-05-21 10:12:15 +03:00 committed by GitHub
parent 7cf9fb3820
commit 342eaee092
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 16 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
ingestion/src/metadata/profiler/orm/functions/length.py
View File

@ -52,6 +52,7 @@ def _(element, compiler, **kw):
@compiles(LenFn, Dialects.Hana) @compiles(LenFn, Dialects.Hana)
@compiles(LenFn, Dialects.Druid) @compiles(LenFn, Dialects.Druid)
@compiles(LenFn, Dialects.Doris) @compiles(LenFn, Dialects.Doris)
@compiles(LenFn, Dialects.Teradata)
def _(element, compiler, **kw): def _(element, compiler, **kw):
return "LENGTH(%s)" % compiler.process(element.clauses, **kw) return "LENGTH(%s)" % compiler.process(element.clauses, **kw)

9
ingestion/src/metadata/profiler/orm/functions/md5.py
View File

@ -40,3 +40,12 @@ def _(element, compiler, **kw):
@compiles(MD5, PythonDialects.BigQuery.value) @compiles(MD5, PythonDialects.BigQuery.value)
def _(element, compiler, **kw): def _(element, compiler, **kw):
return f"TO_HEX(MD5(CAST({compiler.process(element.clauses, **kw)} AS STRING)))" return f"TO_HEX(MD5(CAST({compiler.process(element.clauses, **kw)} AS STRING)))"
@compiles(MD5, PythonDialects.Teradata.value)
def _(element, compiler, **kw):
# There is no MD5 in Teradata or any other hashes
# But we can use UDF function hash_md5 published by Teradata Community
return (
f"HASH_MD5(CAST({compiler.process(element.clauses, **kw)} AS VARCHAR(32000)))"
)

1
ingestion/src/metadata/profiler/orm/functions/modulo.py
View File

@ -61,6 +61,7 @@ def _(element, compiler, **kw):
@compiles(ModuloFn, Dialects.Vertica) @compiles(ModuloFn, Dialects.Vertica)
@compiles(ModuloFn, Dialects.Hana) @compiles(ModuloFn, Dialects.Hana)
@compiles(ModuloFn, Dialects.Cockroach) @compiles(ModuloFn, Dialects.Cockroach)
@compiles(ModuloFn, Dialects.Teradata)
def _(element, compiler, **kw): def _(element, compiler, **kw):
"""Modulo function for specific dialect""" """Modulo function for specific dialect"""
value, base = validate_and_compile(element, compiler, **kw) value, base = validate_and_compile(element, compiler, **kw)

4
ingestion/src/metadata/profiler/orm/functions/random_num.py
View File

@ -98,10 +98,14 @@ def _(*_, **__):
@compiles(RandomNumFn, Dialects.Snowflake) @compiles(RandomNumFn, Dialects.Snowflake)
@compiles(RandomNumFn, Dialects.Teradata)
def _(*_, **__): def _(*_, **__):
"""We use FROM <table> SAMPLE BERNOULLI (n) for sampling """We use FROM <table> SAMPLE BERNOULLI (n) for sampling
in snowflake. We'll return 0 to make sure we get all the rows in snowflake. We'll return 0 to make sure we get all the rows
from the already sampled results when executing row::MOD(0, 100) < profile_sample. from the already sampled results when executing row::MOD(0, 100) < profile_sample.
Teradata RANDOM(0,100) function can't be used inside ORDER BY clause. That's why
use the same trick.
""" """
return "0" return "0"

1
ingestion/src/metadata/profiler/orm/registry.py
View File

@ -86,6 +86,7 @@ class PythonDialects(Enum):
SingleStore = "singlestore" SingleStore = "singlestore"
SQLite = "sqlite" SQLite = "sqlite"
Snowflake = "snowflake" Snowflake = "snowflake"
Teradata = "teradatasql"
Trino = "trino" Trino = "trino"
Vertica = "vertica" Vertica = "vertica"