From cfe3c54d02e9cc93e117770e10ca5844fa7ffd15 Mon Sep 17 00:00:00 2001 From: Aseem Bansal Date: Sat, 8 Jan 2022 01:45:36 +0530 Subject: [PATCH] fix(ingest): dbt - fix warning due to struct type not being mapped (#3846) --- metadata-ingestion/src/datahub/ingestion/source/aws/glue.py | 5 +++-- metadata-ingestion/src/datahub/ingestion/source/dbt.py | 2 ++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py b/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py index d5f2415993..3bca5b059a 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py +++ b/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py @@ -24,6 +24,7 @@ from datahub.metadata.com.linkedin.pegasus2avro.schema import ( MySqlDDL, NullTypeClass, NumberTypeClass, + RecordType, SchemaField, SchemaFieldDataType, SchemaMetadata, @@ -661,7 +662,7 @@ def get_column_type( "set": ArrayTypeClass, "smallint": NumberTypeClass, "string": StringTypeClass, - "struct": MapTypeClass, + "struct": RecordType, "timestamp": TimeTypeClass, "tinyint": NumberTypeClass, "union": UnionTypeClass, @@ -672,7 +673,7 @@ def get_column_type( "array": ArrayTypeClass, "set": ArrayTypeClass, "map": MapTypeClass, - "struct": MapTypeClass, + "struct": RecordType, "varchar": StringTypeClass, "decimal": NumberTypeClass, } diff --git a/metadata-ingestion/src/datahub/ingestion/source/dbt.py b/metadata-ingestion/src/datahub/ingestion/source/dbt.py index 274db2932f..e52ef9ac79 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/dbt.py +++ b/metadata-ingestion/src/datahub/ingestion/source/dbt.py @@ -39,6 +39,7 @@ from datahub.metadata.com.linkedin.pegasus2avro.schema import ( MySqlDDL, NullTypeClass, NumberTypeClass, + RecordType, SchemaField, SchemaFieldDataType, SchemaMetadata, @@ -473,6 +474,7 @@ _field_type_mapping = { "timestamp without time zone": DateTypeClass, "integer": NumberTypeClass, "float8": NumberTypeClass, + "struct": RecordType, **POSTGRES_TYPES_MAP, **SNOWFLAKE_TYPES_MAP, **BIGQUERY_TYPES_MAP,