From 5801e6c3800dc4a563e93b3a61d37d4d877bc138 Mon Sep 17 00:00:00 2001 From: Mayur Singal <39544459+ulixius9@users.noreply.github.com> Date: Mon, 21 Oct 2024 10:31:27 +0530 Subject: [PATCH] MINOR: Add location path to table entity (#18307) --- .../ingestion/source/database/common_db_source.py | 9 +++++++++ .../ingestion/source/database/databricks/metadata.py | 8 ++++++++ .../org/openmetadata/service/jdbi3/TableRepository.java | 1 + .../service/resources/databases/TableResource.java | 1 + .../openmetadata/service/search/indexes/TableIndex.java | 1 + .../resources/elasticsearch/en/table_index_mapping.json | 3 +++ .../resources/elasticsearch/jp/table_index_mapping.json | 3 +++ .../resources/elasticsearch/zh/table_index_mapping.json | 3 +++ .../main/resources/json/schema/api/data/createTable.json | 5 +++++ .../main/resources/json/schema/entity/data/table.json | 5 +++++ 10 files changed, 39 insertions(+) diff --git a/ingestion/src/metadata/ingestion/source/database/common_db_source.py b/ingestion/src/metadata/ingestion/source/database/common_db_source.py index 08cddc27835..024c59b9efd 100644 --- a/ingestion/src/metadata/ingestion/source/database/common_db_source.py +++ b/ingestion/src/metadata/ingestion/source/database/common_db_source.py @@ -478,6 +478,12 @@ class CommonDbSourceService( """Not Implemented""" yield from [] + def get_location_path(self, table_name: str, schema_name: str) -> Optional[str]: + """ + Method to fetch the location path of the table + by default there will be no location path + """ + @calculate_execution_time_generator() def yield_table( self, table_name_and_type: Tuple[str, TableType] @@ -553,6 +559,9 @@ class CommonDbSourceService( table_type=table_type, ), owners=self.get_owner_ref(table_name=table_name), + locationPath=self.get_location_path( + table_name=table_name, schema_name=schema_name + ), ) is_partitioned, partition_details = self.get_table_partition_details( diff --git a/ingestion/src/metadata/ingestion/source/database/databricks/metadata.py b/ingestion/src/metadata/ingestion/source/database/databricks/metadata.py index 6f142931c83..a0d748f1e08 100644 --- a/ingestion/src/metadata/ingestion/source/database/databricks/metadata.py +++ b/ingestion/src/metadata/ingestion/source/database/databricks/metadata.py @@ -705,6 +705,14 @@ class DatabricksSource(ExternalTableLineageMixin, CommonDbSourceService, MultiDB ) return description + def get_location_path(self, table_name: str, schema_name: str) -> Optional[str]: + """ + Method to fetch the location path of the table + """ + return self.external_location_map.get( + (self.context.get().database, schema_name, table_name) + ) + def _filter_owner_name(self, owner_name: str) -> str: """remove unnecessary keyword from name""" pattern = r"\(Unknown\)" diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java index 31a58599e32..f490c355a66 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java @@ -1114,6 +1114,7 @@ public class TableRepository extends EntityRepository { recordChange("sourceUrl", original.getSourceUrl(), updated.getSourceUrl()); recordChange("retentionPeriod", original.getRetentionPeriod(), updated.getRetentionPeriod()); recordChange("sourceHash", original.getSourceHash(), updated.getSourceHash()); + recordChange("locationPath", original.getLocationPath(), updated.getLocationPath()); } private void updateConstraints(Table origTable, Table updatedTable) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/TableResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/TableResource.java index 60b0e55aa3b..8c3a827261c 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/TableResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/TableResource.java @@ -1232,6 +1232,7 @@ public class TableResource extends EntityResource { .copy(new Table(), create, user) .withColumns(create.getColumns()) .withSourceUrl(create.getSourceUrl()) + .withLocationPath(create.getLocationPath()) .withTableConstraints(create.getTableConstraints()) .withTablePartition(create.getTablePartition()) .withTableType(create.getTableType()) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TableIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TableIndex.java index 3815f0b3f3b..3f38ce6189f 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TableIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TableIndex.java @@ -98,6 +98,7 @@ public record TableIndex(Table table) implements ColumnIndex { doc.put("schema_suggest", schemaSuggest); doc.put("database_suggest", databaseSuggest); doc.put("serviceType", table.getServiceType()); + doc.put("locationPath", table.getLocationPath()); doc.put("service", getEntityWithDisplayName(table.getService())); doc.put("database", getEntityWithDisplayName(table.getDatabase())); doc.put("lineage", SearchIndex.getLineageData(table.getEntityReference())); diff --git a/openmetadata-service/src/main/resources/elasticsearch/en/table_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/en/table_index_mapping.json index a2205a4bb36..490f6e27773 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/en/table_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/en/table_index_mapping.json @@ -499,6 +499,9 @@ } } }, + "locationPath": { + "type": "text" + }, "usageSummary": { "properties": { "dailyStats": { diff --git a/openmetadata-service/src/main/resources/elasticsearch/jp/table_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/jp/table_index_mapping.json index e79282e1597..73887f3253c 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/jp/table_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/jp/table_index_mapping.json @@ -477,6 +477,9 @@ } } }, + "locationPath": { + "type": "text" + }, "usageSummary": { "properties": { "dailyStats": { diff --git a/openmetadata-service/src/main/resources/elasticsearch/zh/table_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/zh/table_index_mapping.json index 24d23992917..bb98b74f580 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/zh/table_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/zh/table_index_mapping.json @@ -452,6 +452,9 @@ } } }, + "locationPath": { + "type": "text" + }, "usageSummary": { "properties": { "dailyStats": { diff --git a/openmetadata-spec/src/main/resources/json/schema/api/data/createTable.json b/openmetadata-spec/src/main/resources/json/schema/api/data/createTable.json index d59f6463a46..da1b2f30043 100644 --- a/openmetadata-spec/src/main/resources/json/schema/api/data/createTable.json +++ b/openmetadata-spec/src/main/resources/json/schema/api/data/createTable.json @@ -34,6 +34,11 @@ "dataModel": { "$ref": "../../entity/data/table.json#/definitions/dataModel" }, + "locationPath": { + "description": "Full storage path in case of external and managed tables.", + "type": "string", + "default": null + }, "tableConstraints": { "type": "array", "items": { diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/data/table.json b/openmetadata-spec/src/main/resources/json/schema/entity/data/table.json index 1f8a77c2e7a..46eba06fc22 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/data/table.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/data/table.json @@ -996,6 +996,11 @@ "description": "Reference to the Location that contains this table.", "$ref": "../../type/entityReference.json" }, + "locationPath": { + "description": "Full storage path in case of external and managed tables.", + "type": "string", + "default": null + }, "schemaDefinition": { "description": "DDL for Tables and Views", "$ref": "../../type/basic.json#/definitions/sqlQuery"