yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-08-22 16:08:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixes #16697: Modify the Query to avoid Numeric Data Overflow (#16920)

Browse Source
This commit is contained in:
Ayush Shah 2024-07-12 17:23:53 +05:30 committed by GitHub
parent de18d7df62
commit eeda6d24ae
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 27 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
ingestion/src/metadata/profiler/metrics/static/mean.py
View File

@ -52,6 +52,15 @@ def _(element, compiler, **kw):
return f"if(isNaN(avg({proc})), null, avg({proc}))" return f"if(isNaN(avg({proc})), null, avg({proc}))"
@compiles(avg, Dialects.Redshift)
def _(element, compiler, **kw):
"""
Cast to decimal to get around potential integer overflow error
"""
proc = compiler.process(element.clauses, **kw)
return f"avg(CAST({proc} AS DECIMAL(38,0)))"
@compiles(avg, Dialects.MSSQL) @compiles(avg, Dialects.MSSQL)
def _(element, compiler, **kw): def _(element, compiler, **kw):
""" """

7
ingestion/src/metadata/profiler/orm/functions/sum.py
View File

@ -32,6 +32,13 @@ def _(element, compiler, **kw):
return f"SUM(CAST({proc} AS BIGINT))" return f"SUM(CAST({proc} AS BIGINT))"
@compiles(SumFn, Dialects.Redshift)
def _(element, compiler, **kw):
"""Cast to Decimal to address overflow error from summing 32-bit int in most database dialects"""
proc = compiler.process(element.clauses, **kw)
return f"SUM(CAST({proc} AS Decimal(38,0)))"
@compiles(SumFn, Dialects.Athena) @compiles(SumFn, Dialects.Athena)
@compiles(SumFn, Dialects.Trino) @compiles(SumFn, Dialects.Trino)
@compiles(SumFn, Dialects.Presto) @compiles(SumFn, Dialects.Presto)

21
ingestion/tests/cli_e2e/test_cli_redshift.py
View File

@ -26,7 +26,8 @@ class RedshiftCliTest(CliCommonDB.TestSuite, SQACommonMethods):
CREATE TABLE IF NOT EXISTS e2e_cli_tests.dbt_jaffle.persons ( CREATE TABLE IF NOT EXISTS e2e_cli_tests.dbt_jaffle.persons (
person_id int, person_id int,
full_name varchar(255), full_name varchar(255),
birthdate date birthdate date,
bigint_col bigint
) )
""" """
@ -38,13 +39,13 @@ class RedshiftCliTest(CliCommonDB.TestSuite, SQACommonMethods):
insert_data_queries: List[str] = [ insert_data_queries: List[str] = [
""" """
INSERT INTO e2e_cli_tests.dbt_jaffle.persons (person_id, full_name, birthdate) VALUES INSERT INTO e2e_cli_tests.dbt_jaffle.persons (person_id, full_name, birthdate, bigint_col) VALUES
(1,'Peter Parker', '2004-08-10'), (1,'Peter Parker', '2004-08-10', 9223372036854775807),
(2,'Bruce Banner', '1988-12-18'), (2,'Bruce Banner', '1988-12-18', 9223372036854775807),
(3,'Steve Rogers', '1988-07-04'), (3,'Steve Rogers', '1988-07-04', 9223372036854775807),
(4,'Natasha Romanoff', '1997-12-03'), (4,'Natasha Romanoff', '1997-12-03', 9223372036854775807),
(5,'Wanda Maximoff', '1998-02-10'), (5,'Wanda Maximoff', '1998-02-10', 9223372036854775807),
(6,'Diana Prince', '1976-03-17'); (6,'Diana Prince', '1976-03-17', 9000000000000000007);
""" """
] ]
@ -193,13 +194,13 @@ class RedshiftCliTest(CliCommonDB.TestSuite, SQACommonMethods):
"interQuartileRange": 467.7975, "interQuartileRange": 467.7975,
"max": 856.41, "max": 856.41,
"maxLength": None, "maxLength": None,
"mean": -160.16, "mean": -159.0,
"median": -288.81, "median": -288.81,
"min": -999.63, "min": -999.63,
"minLength": None, "minLength": None,
"missingCount": None, "missingCount": None,
"missingPercentage": None, "missingPercentage": None,
"nonParametricSkew": 0.24351799263849705, "nonParametricSkew": 0.24571372424720792,
"nullCount": 0.0, "nullCount": 0.0,
"nullProportion": 0.0, "nullProportion": 0.0,
"stddev": 528.297718809555, "stddev": 528.297718809555,