yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-11-15 19:03:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ingest): add missing partition keys in schema for glue sources (#3238)

Browse Source
This commit is contained in:
aseembansal-gogo 2021-09-16 10:09:49 +05:30 committed by GitHub
parent 28cbb2b4ef
commit 903348e1db
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 49 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
metadata-ingestion/src/datahub/ingestion/source/aws/glue.py
View File

@ -555,6 +555,23 @@ class GlueSource(Source):
nullable=True, nullable=True,
) )
fields.append(schema_field) fields.append(schema_field)
partition_keys = table["PartitionKeys"]
for partition_key in partition_keys:
schema_field = SchemaField(
fieldPath=partition_key["Name"],
nativeDataType=partition_key["Type"],
type=get_column_type(
glue_source,
partition_key["Type"],
table_name,
partition_key["Name"],
),
recursive=False,
nullable=False,
)
fields.append(schema_field)
return SchemaMetadata( return SchemaMetadata(
schemaName=table_name, schemaName=table_name,
version=0, version=0,

32
metadata-ingestion/tests/unit/glue/glue_mces_golden.json
View File

@ -192,6 +192,22 @@
"globalTags": null, "globalTags": null,
"glossaryTerms": null, "glossaryTerms": null,
"isPartOfKey": false "isPartOfKey": false
},
{
"fieldPath": "year",
"jsonPath": null,
"nullable": false,
"description": null,
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.StringType": {}
}
},
"nativeDataType": "string",
"recursive": false,
"globalTags": null,
"glossaryTerms": null,
"isPartOfKey": false
} }
], ],
"primaryKeys": null, "primaryKeys": null,
@ -458,6 +474,22 @@
"globalTags": null, "globalTags": null,
"glossaryTerms": null, "glossaryTerms": null,
"isPartOfKey": false "isPartOfKey": false
},
{
"fieldPath": "year",
"jsonPath": null,
"nullable": false,
"description": null,
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.StringType": {}
}
},
"nativeDataType": "string",
"recursive": false,
"globalTags": null,
"glossaryTerms": null,
"isPartOfKey": false
} }
], ],
"primaryKeys": null, "primaryKeys": null,