mirror of
https://github.com/open-metadata/OpenMetadata.git
synced 2025-08-18 22:18:23 +00:00
056e6368d0
* refactor!: change partition metadata structure for table entities * refactor!: updated json schema for TypeScript code gen * chore: migration of partition for table entities * style: python & java linting * updated ui side change for table partitioned key * miner fix * addressing comments * fixed ci error --------- Co-authored-by: Shailesh Parmar <shailesh.parmar.webdev@gmail.com>
100 lines
3.9 KiB
Python
100 lines
3.9 KiB
Python
# Copyright 2021 Collate
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
"""Partition utility"""
|
|
|
|
from typing import List, Optional
|
|
|
|
from metadata.generated.schema.entity.data.table import (
|
|
PartitionColumnDetails,
|
|
PartitionIntervalTypes,
|
|
PartitionIntervalUnit,
|
|
PartitionProfilerConfig,
|
|
Table,
|
|
)
|
|
from metadata.generated.schema.entity.services.databaseService import (
|
|
DatabaseServiceType,
|
|
)
|
|
|
|
|
|
def get_partition_details(entity: Table) -> Optional[PartitionProfilerConfig]:
|
|
"""Build PartitionProfilerConfig object from entity
|
|
|
|
Args:
|
|
entity (Table): entity table
|
|
Returns:
|
|
PartitionProfilerConfig
|
|
"""
|
|
if (
|
|
hasattr(entity, "tableProfilerConfig")
|
|
and hasattr(entity.tableProfilerConfig, "partitioning")
|
|
and entity.tableProfilerConfig.partitioning
|
|
):
|
|
return entity.tableProfilerConfig.partitioning
|
|
|
|
if (
|
|
hasattr(entity, "serviceType")
|
|
and entity.serviceType == DatabaseServiceType.BigQuery
|
|
):
|
|
if hasattr(entity, "tablePartition") and entity.tablePartition:
|
|
column_partitions: Optional[
|
|
List[PartitionColumnDetails]
|
|
] = entity.tablePartition.columns
|
|
if not column_partitions:
|
|
raise TypeError("table partition missing. Skipping table")
|
|
|
|
partiton = column_partitions[0]
|
|
|
|
if partiton.intervalType == PartitionIntervalTypes.TIME_UNIT:
|
|
return PartitionProfilerConfig(
|
|
enablePartitioning=True,
|
|
partitionColumnName=partiton.columnName,
|
|
partitionIntervalUnit=PartitionIntervalUnit.DAY
|
|
if partiton.interval != "HOUR"
|
|
else partiton.interval,
|
|
partitionInterval=1,
|
|
partitionIntervalType=partiton.intervalType.value,
|
|
partitionValues=None,
|
|
partitionIntegerRangeStart=None,
|
|
partitionIntegerRangeEnd=None,
|
|
)
|
|
if partiton.intervalType == PartitionIntervalTypes.INGESTION_TIME:
|
|
return PartitionProfilerConfig(
|
|
enablePartitioning=True,
|
|
partitionColumnName="_PARTITIONDATE"
|
|
if partiton.interval == "DAY"
|
|
else "_PARTITIONTIME",
|
|
partitionIntervalUnit=PartitionIntervalUnit.DAY
|
|
if partiton.interval != "HOUR"
|
|
else partiton.interval,
|
|
partitionInterval=1,
|
|
partitionIntervalType=partiton.intervalType.value,
|
|
partitionValues=None,
|
|
partitionIntegerRangeStart=None,
|
|
partitionIntegerRangeEnd=None,
|
|
)
|
|
if partiton.intervalType == PartitionIntervalTypes.INTEGER_RANGE:
|
|
return PartitionProfilerConfig(
|
|
enablePartitioning=True,
|
|
partitionColumnName=partiton.columnName,
|
|
partitionIntervalUnit=None,
|
|
partitionInterval=None,
|
|
partitionIntervalType=partiton.intervalType.value,
|
|
partitionValues=None,
|
|
partitionIntegerRangeStart=1,
|
|
partitionIntegerRangeEnd=10000,
|
|
)
|
|
raise TypeError(
|
|
f"Unsupported partition type {partiton.intervalType}. Skipping table"
|
|
)
|
|
|
|
return None
|