From 1f379a8697b75e5deddb0129ebf0859d4b7f715e Mon Sep 17 00:00:00 2001 From: Keshav Mohta <68001229+keshavmohta09@users.noreply.github.com> Date: Fri, 12 Sep 2025 12:31:20 +0530 Subject: [PATCH] fix: added depth in json and pass in metadata entry (#23332) --- .../ingestion/source/storage/storage_service.py | 1 + .../storage/manifestMetadataConfig.json | 15 ++++++++++++--- .../storage/manifestMetadataConfig.ts | 4 ++++ 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/ingestion/src/metadata/ingestion/source/storage/storage_service.py b/ingestion/src/metadata/ingestion/source/storage/storage_service.py index 054ac6bfb0a..01ec3ba2464 100644 --- a/ingestion/src/metadata/ingestion/source/storage/storage_service.py +++ b/ingestion/src/metadata/ingestion/source/storage/storage_service.py @@ -274,6 +274,7 @@ class StorageServiceSource(TopologyRunnerMixin, Source, ABC): isPartitioned=entry.isPartitioned, partitionColumns=entry.partitionColumns, separator=entry.separator, + depth=entry.depth, ) for entry in manifest.entries if entry.containerName == container_name diff --git a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/storage/manifestMetadataConfig.json b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/storage/manifestMetadataConfig.json index cb781a65343..cf79f8aba60 100644 --- a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/storage/manifestMetadataConfig.json +++ b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/storage/manifestMetadataConfig.json @@ -46,10 +46,17 @@ "$ref": "../../entity/data/table.json#/definitions/column" }, "default": null + }, + "depth": { + "title": "Depth", + "description": "Depth level in the storage path hierarchy where data files reside.", + "type": "integer", + "default": 0 } }, "required": [ - "containerName" ,"dataPath" + "containerName", + "dataPath" ] } }, @@ -63,6 +70,8 @@ "default": null } }, - "required": ["entries"], + "required": [ + "entries" + ], "additionalProperties": false -} +} \ No newline at end of file diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/storage/manifestMetadataConfig.ts b/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/storage/manifestMetadataConfig.ts index 9ec1418f5d0..382c2ebca2a 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/storage/manifestMetadataConfig.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/storage/manifestMetadataConfig.ts @@ -30,6 +30,10 @@ export interface ManifestMetadataEntry { * The path where the data resides in the container, excluding the bucket name */ dataPath: string; + /** + * Depth level in the storage path hierarchy where data files reside. + */ + depth?: number; /** * Flag indicating whether the container's data is partitioned */