yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-11-02 03:39:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(ingest/presto-on-hive): enable partition key for presto-on-hive (#8380)

Browse Source
This commit is contained in:
zheyu001 2023-08-02 12:23:27 +08:00 committed by GitHub
parent 4f9fc671dc
commit bf47d65412
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 51 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/SchemaFieldMapper.java
View File

@ -34,6 +34,7 @@ public class SchemaFieldMapper {
result.setGlossaryTerms(GlossaryTermsMapper.map(input.getGlossaryTerms(), entityUrn));
}
result.setIsPartOfKey(input.isIsPartOfKey());
result.setIsPartitioningKey(input.isIsPartitioningKey());
return result;
}

5
datahub-graphql-core/src/main/resources/entity.graphql
View File

@ -2872,6 +2872,11 @@ type SchemaField {
Whether the field is part of a key schema
"""
isPartOfKey: Boolean
"""
Whether the field is part of a partitioning key schema
"""
isPartitioningKey: Boolean
}
"""

2
datahub-web-react/src/Mocks.tsx
View File

@ -549,6 +549,7 @@ export const dataset3WithSchema = {
type: SchemaFieldDataType.String,
nativeDataType: 'varchar(100)',
isPartOfKey: false,
isPartitioningKey: false,
jsonPath: null,
globalTags: null,
glossaryTerms: null,
@ -563,6 +564,7 @@ export const dataset3WithSchema = {
type: SchemaFieldDataType.String,
nativeDataType: 'boolean',
isPartOfKey: false,
isPartitioningKey: false,
jsonPath: null,
globalTags: null,
glossaryTerms: null,

2
datahub-web-react/src/app/entity/dataset/profile/schema/utils/schemaTitleRenderer.tsx
View File

@ -7,6 +7,7 @@ import { ExtendedSchemaFields } from './types';
import TypeLabel from '../../../../shared/tabs/Dataset/Schema/components/TypeLabel';
import { ForeignKeyConstraint, SchemaMetadata } from '../../../../../../types.generated';
import PrimaryKeyLabel from '../../../../shared/tabs/Dataset/Schema/components/PrimaryKeyLabel';
import PartitioningKeyLabel from '../../../../shared/tabs/Dataset/Schema/components/PartitioningKeyLabel';
import NullableLabel from '../../../../shared/tabs/Dataset/Schema/components/NullableLabel';
import ForeignKeyLabel from '../../../../shared/tabs/Dataset/Schema/components/ForeignKeyLabel';
@ -62,6 +63,7 @@ export default function useSchemaTitleRenderer(
</FieldPathText>
<TypeLabel type={record.type} nativeDataType={record.nativeDataType} />
{(schemaMetadata?.primaryKeys?.includes(fieldPath) || record.isPartOfKey) && <PrimaryKeyLabel />}
{record.isPartitioningKey && <PartitioningKeyLabel />}
{record.nullable && <NullableLabel />}
{schemaMetadata?.foreignKeys
?.filter(

21
datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/components/PartitioningKeyLabel.tsx Normal file
View File

@ -0,0 +1,21 @@
import React from 'react';
import { Badge } from 'antd';
import styled from 'styled-components';
import { blue } from '@ant-design/colors';
import { ANTD_GRAY } from '../../../../constants';
const PartitioningKeyBadge = styled(Badge)`
margin-left: 4px;
&&& .ant-badge-count {
background-color: ${ANTD_GRAY[1]};
color: ${blue[5]};
border: 1px solid ${blue[2]};
font-size: 12px;
font-weight: 400;
height: 22px;
}
`;
export default function PartitioningKeyLabel() {
return <PartitioningKeyBadge count="Partition Key" />;
}

1
datahub-web-react/src/graphql/fragments.graphql
View File

@ -678,6 +678,7 @@ fragment schemaFieldFields on SchemaField {
nativeDataType
recursive
isPartOfKey
isPartitioningKey
globalTags {
...globalTagsFields
}

1
datahub-web-react/src/graphql/versionedDataset.graphql
View File

@ -10,6 +10,7 @@ query getVersionedDataset($urn: String!, $versionStamp: String) {
nativeDataType
recursive
isPartOfKey
isPartitioningKey
}
lastObserved
}

14
metadata-ingestion/src/datahub/ingestion/source/sql/presto_on_hive.py
View File

@ -524,6 +524,8 @@ class PrestoOnHiveSource(SQLAlchemySource):
# add table schema fields
schema_fields = self.get_schema_fields(dataset_name, columns)
self._set_partition_key(columns, schema_fields)
schema_metadata = get_schema_metadata(
self.report,
dataset_name,
@ -888,6 +890,18 @@ class PrestoOnHiveSource(SQLAlchemySource):
default_nullable=True,
)
def _set_partition_key(self, columns, schema_fields):
if len(columns) > 0:
partition_key_names = set()
for column in columns:
if column["is_partition_col"]:
partition_key_names.add(column["col_name"])
for schema_field in schema_fields:
name = schema_field.fieldPath.split(".")[-1]
if name in partition_key_names:
schema_field.isPartitioningKey = True
class SQLAlchemyClient:
def __init__(self, config: SQLAlchemyConfig):

1
metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_1.json
View File

@ -1277,6 +1277,7 @@
"nativeDataType": "string",
"recursive": false,
"isPartOfKey": false,
"isPartitioningKey": true,
"jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}"
},
{

1
metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_2.json
View File

@ -1277,6 +1277,7 @@
"nativeDataType": "string",
"recursive": false,
"isPartOfKey": false,
"isPartitioningKey": true,
"jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}"
},
{

1
metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_3.json
View File

@ -1277,6 +1277,7 @@
"nativeDataType": "string",
"recursive": false,
"isPartOfKey": false,
"isPartitioningKey": true,
"jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}"
},
{

1
metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_4.json
View File

@ -1277,6 +1277,7 @@
"nativeDataType": "string",
"recursive": false,
"isPartOfKey": false,
"isPartitioningKey": true,
"jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}"
},
{