yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-10-27 17:02:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ingestion): Support mapping from avro "boolean" and "map" types t… (#2364)

Browse Source 
Fixes: #2363

Co-authored-by: thomas.larsson <thomas.larsson@klarna.com>
This commit is contained in:
Thomas Larsson 2021-04-08 23:23:12 +02:00 committed by GitHub
parent 4215dcd53c
commit e02a17aecf
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 28 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
metadata-ingestion/src/datahub/ingestion/extractor/schema_util.py
View File

@ -9,6 +9,7 @@ from datahub.metadata.com.linkedin.pegasus2avro.schema import (
BytesTypeClass,
EnumTypeClass,
FixedTypeClass,
MapTypeClass,
NullTypeClass,
NumberTypeClass,
RecordTypeClass,
@ -25,6 +26,7 @@ logger = logging.getLogger(__name__)
_field_type_mapping = {
"null": NullTypeClass,
"bool": BooleanTypeClass,
"boolean": BooleanTypeClass,
"int": NumberTypeClass,
"long": NumberTypeClass,
"float": NumberTypeClass,
@ -32,6 +34,7 @@ _field_type_mapping = {
"bytes": BytesTypeClass,
"string": StringTypeClass,
"record": RecordTypeClass,
"map": MapTypeClass,
"enum": EnumTypeClass,
"array": ArrayTypeClass,
"union": UnionTypeClass,

25
metadata-ingestion/tests/unit/test_schema_util.py
View File

@ -47,6 +47,23 @@ EXAMPLE_EVENT_OPTIONAL_FIELD_VIA_PRIMITIVE_TYPE = """
}
"""
SCHEMA_WITH_MAP_TYPE_FIELD = """
{
"type": "record",
"name": "some.event.name",
"namespace": "some.namespace",
"fields": [
{
"name": "some.field.name",
"type": {
"type": "map",
"values": "long"
}
}
]
}
"""
class SchemaUtilTest(unittest.TestCase):
def test_avro_schema_to_mce_fields_events_with_nullable_fields(self):
@ -61,3 +78,11 @@ class SchemaUtilTest(unittest.TestCase):
fields = avro_schema_to_mce_fields(event)
self.assertEqual(1, len(fields))
self.assertTrue(fields[0].nullable)
def test_avro_schema_to_mce_fields_sample_events_with_different_field_types(self):
EXAMPLES = [SCHEMA_WITH_MAP_TYPE_FIELD]
for schema in EXAMPLES:
fields = avro_schema_to_mce_fields(schema)
self.assertEqual(1, len(fields))