yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-11-01 03:09:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ingest/dbt-athena): dbt-athena types mapping for complex types (#8264)

Browse Source 
Co-authored-by: Harshal Sheth <hsheth2@gmail.com>
This commit is contained in:
Serhii Dimchenko 2023-06-22 11:40:27 +02:00 committed by GitHub
parent 9c65715c5e
commit 5b9fd977eb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 13 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
metadata-ingestion/src/datahub/ingestion/source/sql/sql_types.py
View File

@ -237,10 +237,17 @@ def resolve_trino_modified_type(type_string: str) -> Any:
def resolve_athena_modified_type(type_string: str) -> Any:
# for cases like struct<...>, array<...>, map<...>
match_complex = re.match(r"([a-zA-Z]+)<.+>", type_string)
# for cases like timestamp(3), decimal(10,0)
match = re.match(r"([a-zA-Z]+)\(.+\)", type_string)
if match:
modified_type_base: str = match.group(1)
match_simple = re.match(r"([a-zA-Z]+)\(.+\)", type_string)
modified_type_base = ""
if match_complex:
modified_type_base = match_complex.group(1)
elif match_simple:
modified_type_base = match_simple.group(1)
if modified_type_base:
return ATHENA_SQL_TYPES_MAP[modified_type_base]
return ATHENA_SQL_TYPES_MAP[type_string]

6
metadata-ingestion/tests/integration/dbt/test_dbt.py
View File

@ -326,9 +326,9 @@ def test_resolve_trino_modified_type(data_type, expected_data_type):
("date", "date"),
("timestamp", "timestamp"),
("timestamp(3)", "timestamp"),
("struct(x bigint, y double)", "struct"),
("array(struct(x bigint, y double))", "array"),
("map(varchar, varchar)", "map"),
("struct<x timestamp(3), y timestamp>", "struct"),
("array<struct<x bigint, y double>>", "array"),
("map<varchar, varchar>", "map"),
],
)
def test_resolve_athena_modified_type(data_type, expected_data_type):