yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-12-12 18:47:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(ingest/hive): identify partition columns in hive tables (#12833)

Browse Source
This commit is contained in:
Deepak Garg 2025-03-14 00:07:52 +05:30 committed by GitHub
parent 0e62e8c77a
commit eb17f80e8a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 17 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
metadata-ingestion/src/datahub/ingestion/source/sql/hive.py
View File

@ -777,6 +777,7 @@ class HiveSource(TwoTierSQLAlchemySource):
column,
inspector,
pk_constraints,
partition_keys=partition_keys,
)
if self._COMPLEX_TYPE.match(fields[0].nativeDataType) and isinstance(
@ -849,3 +850,15 @@ class HiveSource(TwoTierSQLAlchemySource):
default_db=default_db,
default_schema=default_schema,
)
def get_partitions(
self, inspector: Inspector, schema: str, table: str
) -> Optional[List[str]]:
partition_columns: List[dict] = inspector.get_indexes(
table_name=table, schema=schema
)
for partition_column in partition_columns:
if partition_column.get("column_names"):
return partition_column.get("column_names")
return []

3
metadata-ingestion/tests/integration/hive/hive_mces_all_db_golden.json
View File

@ -1255,7 +1255,8 @@
},
"nativeDataType": "string",
"recursive": false,
"isPartOfKey": false
"isPartOfKey": false,
"isPartitioningKey": true
}
]
}

3
metadata-ingestion/tests/integration/hive/hive_mces_golden.json
View File

@ -1255,7 +1255,8 @@
},
"nativeDataType": "string",
"recursive": false,
"isPartOfKey": false
"isPartOfKey": false,
"isPartitioningKey": true
}
]
}