yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-11-09 16:03:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ingest): dbt - fix warning due to struct type not being mapped (#3846)

Browse Source
This commit is contained in:
Aseem Bansal 2022-01-08 01:45:36 +05:30 committed by GitHub
parent 94fc93fe7c
commit cfe3c54d02
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 5 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
metadata-ingestion/src/datahub/ingestion/source/aws/glue.py
View File

@ -24,6 +24,7 @@ from datahub.metadata.com.linkedin.pegasus2avro.schema import (
MySqlDDL, MySqlDDL,
NullTypeClass, NullTypeClass,
NumberTypeClass, NumberTypeClass,
RecordType,
SchemaField, SchemaField,
SchemaFieldDataType, SchemaFieldDataType,
SchemaMetadata, SchemaMetadata,
@ -661,7 +662,7 @@ def get_column_type(
"set": ArrayTypeClass, "set": ArrayTypeClass,
"smallint": NumberTypeClass, "smallint": NumberTypeClass,
"string": StringTypeClass, "string": StringTypeClass,
"struct": MapTypeClass, "struct": RecordType,
"timestamp": TimeTypeClass, "timestamp": TimeTypeClass,
"tinyint": NumberTypeClass, "tinyint": NumberTypeClass,
"union": UnionTypeClass, "union": UnionTypeClass,
@ -672,7 +673,7 @@ def get_column_type(
"array": ArrayTypeClass, "array": ArrayTypeClass,
"set": ArrayTypeClass, "set": ArrayTypeClass,
"map": MapTypeClass, "map": MapTypeClass,
"struct": MapTypeClass, "struct": RecordType,
"varchar": StringTypeClass, "varchar": StringTypeClass,
"decimal": NumberTypeClass, "decimal": NumberTypeClass,
} }

2
metadata-ingestion/src/datahub/ingestion/source/dbt.py
View File

@ -39,6 +39,7 @@ from datahub.metadata.com.linkedin.pegasus2avro.schema import (
MySqlDDL, MySqlDDL,
NullTypeClass, NullTypeClass,
NumberTypeClass, NumberTypeClass,
RecordType,
SchemaField, SchemaField,
SchemaFieldDataType, SchemaFieldDataType,
SchemaMetadata, SchemaMetadata,
@ -473,6 +474,7 @@ _field_type_mapping = {
"timestamp without time zone": DateTypeClass, "timestamp without time zone": DateTypeClass,
"integer": NumberTypeClass, "integer": NumberTypeClass,
"float8": NumberTypeClass, "float8": NumberTypeClass,
"struct": RecordType,
**POSTGRES_TYPES_MAP, **POSTGRES_TYPES_MAP,
**SNOWFLAKE_TYPES_MAP, **SNOWFLAKE_TYPES_MAP,
**BIGQUERY_TYPES_MAP, **BIGQUERY_TYPES_MAP,