yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-10-31 02:37:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ingestion): Support mapping from avro "boolean" and "map" types t… (#2364)

Browse Source 
Fixes: #2363

Co-authored-by: thomas.larsson <thomas.larsson@klarna.com>
This commit is contained in:
Thomas Larsson 2021-04-08 23:23:12 +02:00 committed by GitHub
parent 4215dcd53c
commit e02a17aecf
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 28 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
metadata-ingestion/src/datahub/ingestion/extractor/schema_util.py
View File

@ -9,6 +9,7 @@ from datahub.metadata.com.linkedin.pegasus2avro.schema import (
BytesTypeClass, BytesTypeClass,
EnumTypeClass, EnumTypeClass,
FixedTypeClass, FixedTypeClass,
MapTypeClass,
NullTypeClass, NullTypeClass,
NumberTypeClass, NumberTypeClass,
RecordTypeClass, RecordTypeClass,
@ -25,6 +26,7 @@ logger = logging.getLogger(__name__)
_field_type_mapping = { _field_type_mapping = {
"null": NullTypeClass, "null": NullTypeClass,
"bool": BooleanTypeClass, "bool": BooleanTypeClass,
"boolean": BooleanTypeClass,
"int": NumberTypeClass, "int": NumberTypeClass,
"long": NumberTypeClass, "long": NumberTypeClass,
"float": NumberTypeClass, "float": NumberTypeClass,
@ -32,6 +34,7 @@ _field_type_mapping = {
"bytes": BytesTypeClass, "bytes": BytesTypeClass,
"string": StringTypeClass, "string": StringTypeClass,
"record": RecordTypeClass, "record": RecordTypeClass,
"map": MapTypeClass,
"enum": EnumTypeClass, "enum": EnumTypeClass,
"array": ArrayTypeClass, "array": ArrayTypeClass,
"union": UnionTypeClass, "union": UnionTypeClass,

25
metadata-ingestion/tests/unit/test_schema_util.py
View File

@ -47,6 +47,23 @@ EXAMPLE_EVENT_OPTIONAL_FIELD_VIA_PRIMITIVE_TYPE = """
} }
""" """
SCHEMA_WITH_MAP_TYPE_FIELD = """
{
"type": "record",
"name": "some.event.name",
"namespace": "some.namespace",
"fields": [
{
"name": "some.field.name",
"type": {
"type": "map",
"values": "long"
}
}
]
}
"""
class SchemaUtilTest(unittest.TestCase): class SchemaUtilTest(unittest.TestCase):
def test_avro_schema_to_mce_fields_events_with_nullable_fields(self): def test_avro_schema_to_mce_fields_events_with_nullable_fields(self):
@ -61,3 +78,11 @@ class SchemaUtilTest(unittest.TestCase):
fields = avro_schema_to_mce_fields(event) fields = avro_schema_to_mce_fields(event)
self.assertEqual(1, len(fields)) self.assertEqual(1, len(fields))
self.assertTrue(fields[0].nullable) self.assertTrue(fields[0].nullable)
def test_avro_schema_to_mce_fields_sample_events_with_different_field_types(self):
EXAMPLES = [SCHEMA_WITH_MAP_TYPE_FIELD]
for schema in EXAMPLES:
fields = avro_schema_to_mce_fields(schema)
self.assertEqual(1, len(fields))