yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-11-08 23:43:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ingest/s3): wrong sorting in case of multi-partition key (#8536)

Browse Source
This commit is contained in:
Aseem Bansal 2023-08-02 09:54:33 +05:30 committed by GitHub
parent ea95e36c2b
commit bb33f015ca
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 9 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
metadata-ingestion/src/datahub/ingestion/source/s3/source.py
View File

@ -178,9 +178,9 @@ def partitioned_folder_comparator(folder1: str, folder2: str) -> int:
try: try:
# Stripping = from the folder names as it most probably partition name part like year=2021 # Stripping = from the folder names as it most probably partition name part like year=2021
if "=" in folder1 and "=" in folder2: if "=" in folder1 and "=" in folder2:
if folder1.split("=", 1)[0] == folder2.split("=", 1)[0]: if folder1.rsplit("=", 1)[0] == folder2.rsplit("=", 1)[0]:
folder1 = folder1.split("=", 1)[1] folder1 = folder1.rsplit("=", 1)[-1]
folder2 = folder2.split("=", 1)[1] folder2 = folder2.rsplit("=", 1)[-1]
num_folder1 = int(folder1) num_folder1 = int(folder1)
num_folder2 = int(folder2) num_folder2 = int(folder2)

6
metadata-ingestion/tests/unit/s3/test_s3_source.py
View File

@ -8,6 +8,12 @@ def test_partition_comparator_numeric_folder_name():
assert partitioned_folder_comparator(folder1, folder2) == -1 assert partitioned_folder_comparator(folder1, folder2) == -1
def test_partition_multi_level_key():
folder1 = "backup/metadata_aspect_v2/year=2023/month=01"
folder2 = "backup/metadata_aspect_v2/year=2023/month=2"
assert partitioned_folder_comparator(folder1, folder2) == -1
def test_partition_comparator_numeric_folder_name2(): def test_partition_comparator_numeric_folder_name2():
folder1 = "12" folder1 = "12"
folder2 = "3" folder2 = "3"