OpenMetadata/ingestion/src/metadata/utils/partition.py

#  Copyright 2021 Collate
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#  http://www.apache.org/licenses/LICENSE-2.0
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

"""Partition utility"""

from typing import List, Optional

from metadata.generated.schema.entity.data.table import (
    PartitionColumnDetails,
    PartitionIntervalTypes,
    PartitionIntervalUnit,
    PartitionProfilerConfig,
    Table,
)
from metadata.generated.schema.entity.services.databaseService import (
    DatabaseServiceType,
)


def get_partition_details(entity: Table) -> Optional[PartitionProfilerConfig]:
    """Build PartitionProfilerConfig object from entity

    Args:
        entity (Table): entity table
    Returns:
        PartitionProfilerConfig
    """
    if (
        hasattr(entity, "tableProfilerConfig")
        and hasattr(entity.tableProfilerConfig, "partitioning")
        and entity.tableProfilerConfig.partitioning
    ):
        return entity.tableProfilerConfig.partitioning

    if (
        hasattr(entity, "serviceType")
        and entity.serviceType == DatabaseServiceType.BigQuery
    ):
        if hasattr(entity, "tablePartition") and entity.tablePartition:
            column_partitions: Optional[
                List[PartitionColumnDetails]
            ] = entity.tablePartition.columns
            if not column_partitions:
                raise TypeError("table partition missing. Skipping table")

            partiton = column_partitions[0]

            if partiton.intervalType == PartitionIntervalTypes.TIME_UNIT:
                return PartitionProfilerConfig(
                    enablePartitioning=True,
                    partitionColumnName=partiton.columnName,
                    partitionIntervalUnit=PartitionIntervalUnit.DAY
                    if partiton.interval != "HOUR"
                    else partiton.interval,
                    partitionInterval=1,
                    partitionIntervalType=partiton.intervalType.value,
                    partitionValues=None,
                    partitionIntegerRangeStart=None,
                    partitionIntegerRangeEnd=None,
                )
            if partiton.intervalType == PartitionIntervalTypes.INGESTION_TIME:
                return PartitionProfilerConfig(
                    enablePartitioning=True,
                    partitionColumnName="_PARTITIONDATE"
                    if partiton.interval == "DAY"
                    else "_PARTITIONTIME",
                    partitionIntervalUnit=PartitionIntervalUnit.DAY
                    if partiton.interval != "HOUR"
                    else partiton.interval,
                    partitionInterval=1,
                    partitionIntervalType=partiton.intervalType.value,
                    partitionValues=None,
                    partitionIntegerRangeStart=None,
                    partitionIntegerRangeEnd=None,
                )
            if partiton.intervalType == PartitionIntervalTypes.INTEGER_RANGE:
                return PartitionProfilerConfig(
                    enablePartitioning=True,
                    partitionColumnName=partiton.columnName,
                    partitionIntervalUnit=None,
                    partitionInterval=None,
                    partitionIntervalType=partiton.intervalType.value,
                    partitionValues=None,
                    partitionIntegerRangeStart=1,
                    partitionIntegerRangeEnd=10000,
                )
            raise TypeError(
                f"Unsupported partition type {partiton.intervalType}. Skipping table"
            )

    return None