From 13f2d3e25e4643076d13b4e90d9d276611453e1c Mon Sep 17 00:00:00 2001 From: Onkar Ravgan Date: Wed, 4 Oct 2023 15:07:09 +0530 Subject: [PATCH] Added entity_ref instead of fqn in unused data assets aggregrate API (#13421) --- .../search/elasticsearch/ElasticSearchClient.java | 8 +++++--- .../elasticsearch/EsUnusedAssetsAggregator.java | 15 ++++++++------- .../search/opensearch/OpenSearchClient.java | 8 +++++--- .../opensearch/OsUnusedAssetsAggregator.java | 15 ++++++++------- .../aggregatedCostAnalysisReportData.json | 5 ++--- .../reportDataType/rawCostAnalysisReportData.json | 2 +- .../schema/dataInsight/type/unusedAssets.json | 6 +++--- 7 files changed, 32 insertions(+), 27 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java index 94f832eca18..75dd7ff9e4b 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java @@ -297,8 +297,9 @@ public class ElasticSearchClient implements SearchClient { searchSourceBuilder = buildDomainsSearch(request.getQuery(), request.getFrom(), request.getSize()); break; case "raw_cost_analysis_report_data_index": + case "aggregated_cost_analysis_report_data_index": searchSourceBuilder = - buildRawCostAnalysisReportDataSearch(request.getQuery(), request.getFrom(), request.getSize()); + buildCostAnalysisReportDataSearch(request.getQuery(), request.getFrom(), request.getSize()); break; default: searchSourceBuilder = buildAggregateSearchBuilder(request.getQuery(), request.getFrom(), request.getSize()); @@ -335,7 +336,8 @@ public class ElasticSearchClient implements SearchClient { if (request.getIndex().equalsIgnoreCase("domain_search_index") || request.getIndex().equalsIgnoreCase("data_products_search_index") || request.getIndex().equalsIgnoreCase("query_search_index") - || request.getIndex().equalsIgnoreCase("raw_cost_analysis_report_data_index")) { + || request.getIndex().equalsIgnoreCase("raw_cost_analysis_report_data_index") + || request.getIndex().equalsIgnoreCase("aggregated_cost_analysis_report_data_index")) { searchSourceBuilder.query(QueryBuilders.boolQuery().must(searchSourceBuilder.query())); } else { searchSourceBuilder.query( @@ -807,7 +809,7 @@ public class ElasticSearchClient implements SearchClient { return searchBuilder(queryBuilder, hb, from, size); } - private static SearchSourceBuilder buildRawCostAnalysisReportDataSearch(String query, int from, int size) { + private static SearchSourceBuilder buildCostAnalysisReportDataSearch(String query, int from, int size) { QueryStringQueryBuilder queryBuilder = QueryBuilders.queryStringQuery(query); return searchBuilder(queryBuilder, null, from, size); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/EsUnusedAssetsAggregator.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/EsUnusedAssetsAggregator.java index 9befce72feb..250f66f5ff5 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/EsUnusedAssetsAggregator.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/EsUnusedAssetsAggregator.java @@ -1,5 +1,6 @@ package org.openmetadata.service.search.elasticsearch; +import com.fasterxml.jackson.databind.ObjectMapper; import java.text.ParseException; import java.util.ArrayList; import java.util.HashMap; @@ -10,11 +11,14 @@ import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; import org.openmetadata.schema.dataInsight.DataInsightChartResult; import org.openmetadata.schema.dataInsight.type.UnusedAssets; +import org.openmetadata.schema.type.EntityReference; import org.openmetadata.service.dataInsight.DataInsightAggregatorInterface; // TODO: refactor this class and the interface in https://github.com/open-metadata/OpenMetadata/issues/13401 @Slf4j public class EsUnusedAssetsAggregator extends DataInsightAggregatorInterface { + final ObjectMapper mapper = new ObjectMapper(); + public EsUnusedAssetsAggregator(SearchHits hits, DataInsightChartResult.DataInsightChartType dataInsightChartType) { super(hits, dataInsightChartType); } @@ -39,18 +43,15 @@ public class EsUnusedAssetsAggregator extends DataInsightAggregatorInterface { for (SearchHit hit : this.hitsEs) { try { HashMap data = (HashMap) hit.getSourceAsMap().get("data"); - String fqn = ((HashMap) data.get("entity")).get("fullyQualifiedName"); Long lastAccessed = (Long) ((HashMap) ((HashMap) data.get("lifeCycle")).get("accessed")) .get("timestamp"); Double sizeInByte = (Double) data.get("sizeInByte"); - new UnusedAssets().withFullyQualifiedName(fqn).withLastAccessedAt(lastAccessed).withSizeInBytes(sizeInByte); - dataList.add( - new UnusedAssets() - .withFullyQualifiedName(fqn) - .withLastAccessedAt(lastAccessed) - .withSizeInBytes(sizeInByte)); + EntityReference entityReference = mapper.convertValue(data.get("entity"), EntityReference.class); + UnusedAssets unusedAssets = + new UnusedAssets().withEntity(entityReference).withLastAccessedAt(lastAccessed).withSizeInBytes(sizeInByte); + dataList.add(unusedAssets); } catch (Exception e) { LOG.error("Error while parsing hits for UnusedData chart from ES", e); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java index 1f804da1eeb..ea0e49ed5d5 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java @@ -291,8 +291,9 @@ public class OpenSearchClient implements SearchClient { searchSourceBuilder = buildSearchEntitySearch(request.getQuery(), request.getFrom(), request.getSize()); break; case "raw_cost_analysis_report_data_index": + case "aggregated_cost_analysis_report_data_index": searchSourceBuilder = - buildRawCostAnalysisReportDataSearch(request.getQuery(), request.getFrom(), request.getSize()); + buildCostAnalysisReportDataSearch(request.getQuery(), request.getFrom(), request.getSize()); break; default: searchSourceBuilder = buildAggregateSearchBuilder(request.getQuery(), request.getFrom(), request.getSize()); @@ -329,7 +330,8 @@ public class OpenSearchClient implements SearchClient { if (request.getIndex().equalsIgnoreCase("domain_search_index") || request.getIndex().equalsIgnoreCase("data_products_search_index") || request.getIndex().equalsIgnoreCase("query_search_index") - || request.getIndex().equalsIgnoreCase("raw_cost_analysis_report_data_index")) { + || request.getIndex().equalsIgnoreCase("raw_cost_analysis_report_data_index") + || request.getIndex().equalsIgnoreCase("aggregated_cost_analysis_report_data_index")) { searchSourceBuilder.query(QueryBuilders.boolQuery().must(searchSourceBuilder.query())); } else { searchSourceBuilder.query( @@ -799,7 +801,7 @@ public class OpenSearchClient implements SearchClient { return addAggregation(searchSourceBuilder); } - private static SearchSourceBuilder buildRawCostAnalysisReportDataSearch(String query, int from, int size) { + private static SearchSourceBuilder buildCostAnalysisReportDataSearch(String query, int from, int size) { QueryStringQueryBuilder queryBuilder = QueryBuilders.queryStringQuery(query); return searchBuilder(queryBuilder, null, from, size); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OsUnusedAssetsAggregator.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OsUnusedAssetsAggregator.java index 47108b82083..82bf3dab5fa 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OsUnusedAssetsAggregator.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OsUnusedAssetsAggregator.java @@ -1,5 +1,6 @@ package org.openmetadata.service.search.opensearch; +import com.fasterxml.jackson.databind.ObjectMapper; import java.text.ParseException; import java.util.ArrayList; import java.util.HashMap; @@ -8,12 +9,15 @@ import lombok.extern.slf4j.Slf4j; import org.apache.lucene.search.TotalHits; import org.openmetadata.schema.dataInsight.DataInsightChartResult; import org.openmetadata.schema.dataInsight.type.UnusedAssets; +import org.openmetadata.schema.type.EntityReference; import org.openmetadata.service.dataInsight.DataInsightAggregatorInterface; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; @Slf4j public class OsUnusedAssetsAggregator extends DataInsightAggregatorInterface { + final ObjectMapper mapper = new ObjectMapper(); + public OsUnusedAssetsAggregator(SearchHits hits, DataInsightChartResult.DataInsightChartType dataInsightChartType) { super(hits, dataInsightChartType); } @@ -38,18 +42,15 @@ public class OsUnusedAssetsAggregator extends DataInsightAggregatorInterface { for (SearchHit hit : this.hitsOs) { try { HashMap data = (HashMap) hit.getSourceAsMap().get("data"); - String fqn = ((HashMap) data.get("entity")).get("fullyQualifiedName"); Long lastAccessed = (Long) ((HashMap) ((HashMap) data.get("lifeCycle")).get("accessed")) .get("timestamp"); Double sizeInByte = (Double) data.get("sizeInByte"); - new UnusedAssets().withFullyQualifiedName(fqn).withLastAccessedAt(lastAccessed).withSizeInBytes(sizeInByte); - dataList.add( - new UnusedAssets() - .withFullyQualifiedName(fqn) - .withLastAccessedAt(lastAccessed) - .withSizeInBytes(sizeInByte)); + EntityReference entityReference = mapper.convertValue(data.get("entity"), EntityReference.class); + UnusedAssets unusedAssets = + new UnusedAssets().withEntity(entityReference).withLastAccessedAt(lastAccessed).withSizeInBytes(sizeInByte); + dataList.add(unusedAssets); } catch (Exception e) { LOG.error("Error while parsing hits for UnusedData chart from ES", e); } diff --git a/openmetadata-spec/src/main/resources/json/schema/analytics/reportDataType/aggregatedCostAnalysisReportData.json b/openmetadata-spec/src/main/resources/json/schema/analytics/reportDataType/aggregatedCostAnalysisReportData.json index 4548fb704a6..d0aae62f487 100644 --- a/openmetadata-spec/src/main/resources/json/schema/analytics/reportDataType/aggregatedCostAnalysisReportData.json +++ b/openmetadata-spec/src/main/resources/json/schema/analytics/reportDataType/aggregatedCostAnalysisReportData.json @@ -4,7 +4,7 @@ "$schema": "http://json-schema.org/draft-07/schema#", "title": "aggregatedCostAnalysisReportData", "type": "object", - "javaType": "org.openmetadata.schema.analytics.AggregatedCostAnalysisReportData.json", + "javaType": "org.openmetadata.schema.analytics.AggregatedCostAnalysisReportData", "description": "Aggregated data for Cost Analysis Report.", "definitions": { "dataAssetCount": { @@ -32,8 +32,7 @@ "type": "number" } }, - "additionalProperties": false, - "required": ["entity"] + "additionalProperties": false } }, "properties": { diff --git a/openmetadata-spec/src/main/resources/json/schema/analytics/reportDataType/rawCostAnalysisReportData.json b/openmetadata-spec/src/main/resources/json/schema/analytics/reportDataType/rawCostAnalysisReportData.json index cd5f9abc693..e4c752428e3 100644 --- a/openmetadata-spec/src/main/resources/json/schema/analytics/reportDataType/rawCostAnalysisReportData.json +++ b/openmetadata-spec/src/main/resources/json/schema/analytics/reportDataType/rawCostAnalysisReportData.json @@ -3,7 +3,7 @@ "$schema": "http://json-schema.org/draft-07/schema#", "title": "rawCostAnalysisReportData", "type": "object", - "javaType": "org.openmetadata.schema.analytics.RawCostAnalysisReportData.json", + "javaType": "org.openmetadata.schema.analytics.RawCostAnalysisReportData", "description": "Raw data for Cost Analysis Report.", "properties": { "entity": { diff --git a/openmetadata-spec/src/main/resources/json/schema/dataInsight/type/unusedAssets.json b/openmetadata-spec/src/main/resources/json/schema/dataInsight/type/unusedAssets.json index 3436e058682..a81099384aa 100644 --- a/openmetadata-spec/src/main/resources/json/schema/dataInsight/type/unusedAssets.json +++ b/openmetadata-spec/src/main/resources/json/schema/dataInsight/type/unusedAssets.json @@ -6,9 +6,9 @@ "type": "object", "javaType": "org.openmetadata.schema.dataInsight.type.UnusedAssets", "properties": { - "fullyQualifiedName": { - "description": "Fully qualified name of the asset", - "$ref": "../../type/basic.json#/definitions/fullyQualifiedEntityName" + "entity": { + "description": "Entity of the life cycle data", + "$ref": "../../type/entityReference.json" }, "sizeInBytes": { "description": "Size of the asset in bytes",