yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-08-19 14:37:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixes #17085: Update Oracle count & unique count function to handle HexByteString/Blob types (#17596)

Browse Source
This commit is contained in:
Ayush Shah 2024-09-06 12:42:41 +05:30 committed by GitHub
parent 8191202850
commit b2f21fa070
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 28 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
ingestion/src/metadata/profiler/orm/functions/count.py
View File

@ -18,6 +18,7 @@ from sqlalchemy.sql.sqltypes import NVARCHAR, TEXT
from metadata.profiler.metrics.core import CACHE from metadata.profiler.metrics.core import CACHE
from metadata.profiler.orm.registry import Dialects from metadata.profiler.orm.registry import Dialects
from metadata.profiler.orm.types.custom_hex_byte_string import HexByteString
# Keep SQA docs style defining custom constructs # Keep SQA docs style defining custom constructs
# pylint: disable=consider-using-f-string,duplicate-code # pylint: disable=consider-using-f-string,duplicate-code
@ -36,6 +37,14 @@ def _(element, compiler, **kw):
return compiler.process(element.clauses, **kw) return compiler.process(element.clauses, **kw)
@compiles(CountFn, Dialects.Oracle)
def _(element, compiler, **kw):
col_type = element.clauses.clauses[0].type
if isinstance(col_type, HexByteString):
return f"DBMS_LOB.GETLENGTH({compiler.process(element.clauses, **kw)})"
return compiler.process(element.clauses, **kw)
@compiles(CountFn, Dialects.MSSQL) @compiles(CountFn, Dialects.MSSQL)
def _(element, compiler, **kw): def _(element, compiler, **kw):
col_type = element.clauses.clauses[0].type col_type = element.clauses.clauses[0].type

6
ingestion/src/metadata/profiler/orm/functions/unique_count.py
View File

@ -53,5 +53,11 @@ def _unique_count_query_mssql(col, session, sample):
) )
def _unique_count_query_oracle(col, session, sample):
count_fn = CountFn(col)
return _unique_count_query(count_fn, session, sample)
_unique_count_query_mapper = defaultdict(lambda: _unique_count_query) _unique_count_query_mapper = defaultdict(lambda: _unique_count_query)
_unique_count_query_mapper[Dialects.MSSQL] = _unique_count_query_mssql _unique_count_query_mapper[Dialects.MSSQL] = _unique_count_query_mssql
_unique_count_query_mapper[Dialects.Oracle] = _unique_count_query_oracle

26
ingestion/tests/cli_e2e/test_cli_oracle.py
View File

@ -15,8 +15,6 @@ Oracle E2E tests
from typing import List from typing import List
import pytest
from metadata.ingestion.api.status import Status from metadata.ingestion.api.status import Status
from .base.e2e_types import E2EType from .base.e2e_types import E2EType
@ -48,14 +46,19 @@ class OracleCliTest(CliCommonDB.TestSuite, SQACommonMethods):
insert_data_queries: List[str] = [ insert_data_queries: List[str] = [
""" """
INSERT INTO admin.admin_emp (empno, ename, ssn, job, mgr, sal, comm, comments, status) WITH names AS ( INSERT INTO admin.admin_emp (empno, ename, ssn, job, mgr, sal, comm, comments, status, photo) WITH names AS (
SELECT 1, 'John Doe', 12356789, 'Manager', 121, 5200.0, 5000.0, 'Amazing', 'Active' FROM dual UNION ALL SELECT 1, 'John Doe', 12356789, 'Manager', 121, 5200.0, 5000.0, 'Amazing', 'Active', EMPTY_BLOB() FROM dual UNION ALL
SELECT 2, 'Jane Doe', 123467189, 'Clerk', 131, 503.0, 5000.0, 'Wow', 'Active' FROM dual UNION ALL SELECT 2, 'Jane Doe', 123467189, 'Clerk', 131, 503.0, 5000.0, 'Wow', 'Active', EMPTY_BLOB() FROM dual UNION ALL
SELECT 3, 'Jon Doe', 123562789, 'Assistant', 141, 5000.0, 5000.0, 'Nice', 'Active' FROM dual UNION ALL SELECT 3, 'Jon Doe', 123562789, 'Assistant', 141, 5000.0, 5000.0, 'Nice', 'Active', EMPTY_BLOB() FROM dual
SELECT 4, 'Jon Doe', 13456789, 'Manager', 151, 5050.0, 5000.0, 'Excellent', 'Active' FROM dual
) )
SELECT * from names SELECT * from names
""" """,
"""
INSERT INTO admin.admin_emp (empno, ename, ssn, job, mgr, sal, comm, comments, status, photo) WITH names AS (
SELECT 4, 'Jon Doe', 13456789, 'Manager', 151, 5050.0, 5000.0, 'Excellent', 'Active', UTL_RAW.CAST_TO_RAW('your_binary_data') FROM dual
)
SELECT * from names
""",
] ]
drop_table_query: str = """ drop_table_query: str = """
@ -98,11 +101,11 @@ SELECT * from names
@staticmethod @staticmethod
def fqn_created_table() -> str: def fqn_created_table() -> str:
return "e2e_oracle.default.admin.admin_emp" return "e2e_oracle.default.admin.ADMIN_EMP"
@staticmethod @staticmethod
def _fqn_deleted_table() -> str: def _fqn_deleted_table() -> str:
return "e2e_oracle.default.admin.admin_emp" return "e2e_oracle.default.admin.ADMIN_EMP"
@staticmethod @staticmethod
def get_includes_schemas() -> List[str]: def get_includes_schemas() -> List[str]:
@ -136,9 +139,6 @@ SELECT * from names
def expected_filtered_mix() -> int: def expected_filtered_mix() -> int:
return 43 return 43
@pytest.mark.xfail(
reason="Issue Raised: https://github.com/open-metadata/OpenMetadata/issues/17085"
)
def test_create_table_with_profiler(self) -> None: def test_create_table_with_profiler(self) -> None:
# delete table in case it exists # delete table in case it exists
self.delete_table_and_view() self.delete_table_and_view()