Added entity_ref instead of fqn in unused data assets aggregrate API (#13421)

2025-08-30 11:56:01 +00:00 · 2023-10-04 15:07:09 +05:30 · 2023-10-04 15:07:09 +05:30 · 13f2d3e25e
commit 13f2d3e25e
parent 0282574bdd
7 changed files with 32 additions and 27 deletions
--- a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java
@ -297,8 +297,9 @@ public class ElasticSearchClient implements SearchClient {
        searchSourceBuilder = buildDomainsSearch(request.getQuery(), request.getFrom(), request.getSize());
        break;
      case "raw_cost_analysis_report_data_index":
+      case "aggregated_cost_analysis_report_data_index":
        searchSourceBuilder =
-            buildRawCostAnalysisReportDataSearch(request.getQuery(), request.getFrom(), request.getSize());
+            buildCostAnalysisReportDataSearch(request.getQuery(), request.getFrom(), request.getSize());
        break;
      default:
        searchSourceBuilder = buildAggregateSearchBuilder(request.getQuery(), request.getFrom(), request.getSize());
@ -335,7 +336,8 @@ public class ElasticSearchClient implements SearchClient {
    if (request.getIndex().equalsIgnoreCase("domain_search_index")
        || request.getIndex().equalsIgnoreCase("data_products_search_index")
        || request.getIndex().equalsIgnoreCase("query_search_index")
-        || request.getIndex().equalsIgnoreCase("raw_cost_analysis_report_data_index")) {
+        || request.getIndex().equalsIgnoreCase("raw_cost_analysis_report_data_index")
+        || request.getIndex().equalsIgnoreCase("aggregated_cost_analysis_report_data_index")) {
      searchSourceBuilder.query(QueryBuilders.boolQuery().must(searchSourceBuilder.query()));
    } else {
      searchSourceBuilder.query(
@ -807,7 +809,7 @@ public class ElasticSearchClient implements SearchClient {
    return searchBuilder(queryBuilder, hb, from, size);
  }

-  private static SearchSourceBuilder buildRawCostAnalysisReportDataSearch(String query, int from, int size) {
+  private static SearchSourceBuilder buildCostAnalysisReportDataSearch(String query, int from, int size) {
    QueryStringQueryBuilder queryBuilder = QueryBuilders.queryStringQuery(query);
    return searchBuilder(queryBuilder, null, from, size);
  }
--- a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/EsUnusedAssetsAggregator.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/EsUnusedAssetsAggregator.java
@ -1,5 +1,6 @@
 package org.openmetadata.service.search.elasticsearch;

+import com.fasterxml.jackson.databind.ObjectMapper;
 import java.text.ParseException;
 import java.util.ArrayList;
 import java.util.HashMap;
@ -10,11 +11,14 @@ import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.SearchHits;
 import org.openmetadata.schema.dataInsight.DataInsightChartResult;
 import org.openmetadata.schema.dataInsight.type.UnusedAssets;
+import org.openmetadata.schema.type.EntityReference;
 import org.openmetadata.service.dataInsight.DataInsightAggregatorInterface;

 // TODO: refactor this class and the interface in https://github.com/open-metadata/OpenMetadata/issues/13401
@Slf4j
 public class EsUnusedAssetsAggregator extends DataInsightAggregatorInterface {
+  final ObjectMapper mapper = new ObjectMapper();
+
  public EsUnusedAssetsAggregator(SearchHits hits, DataInsightChartResult.DataInsightChartType dataInsightChartType) {
    super(hits, dataInsightChartType);
  }
@ -39,18 +43,15 @@ public class EsUnusedAssetsAggregator extends DataInsightAggregatorInterface {
    for (SearchHit hit : this.hitsEs) {
      try {
        HashMap<String, Object> data = (HashMap<String, Object>) hit.getSourceAsMap().get("data");
-        String fqn = ((HashMap<String, String>) data.get("entity")).get("fullyQualifiedName");
        Long lastAccessed =
            (Long)
                ((HashMap<String, Object>) ((HashMap<String, Object>) data.get("lifeCycle")).get("accessed"))
                    .get("timestamp");
        Double sizeInByte = (Double) data.get("sizeInByte");
-        new UnusedAssets().withFullyQualifiedName(fqn).withLastAccessedAt(lastAccessed).withSizeInBytes(sizeInByte);
-        dataList.add(
-            new UnusedAssets()
-                .withFullyQualifiedName(fqn)
-                .withLastAccessedAt(lastAccessed)
-                .withSizeInBytes(sizeInByte));
+        EntityReference entityReference = mapper.convertValue(data.get("entity"), EntityReference.class);
+        UnusedAssets unusedAssets =
+            new UnusedAssets().withEntity(entityReference).withLastAccessedAt(lastAccessed).withSizeInBytes(sizeInByte);
+        dataList.add(unusedAssets);
      } catch (Exception e) {
        LOG.error("Error while parsing hits for UnusedData chart from ES", e);
      }
--- a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java
@ -291,8 +291,9 @@ public class OpenSearchClient implements SearchClient {
        searchSourceBuilder = buildSearchEntitySearch(request.getQuery(), request.getFrom(), request.getSize());
        break;
      case "raw_cost_analysis_report_data_index":
+      case "aggregated_cost_analysis_report_data_index":
        searchSourceBuilder =
-            buildRawCostAnalysisReportDataSearch(request.getQuery(), request.getFrom(), request.getSize());
+            buildCostAnalysisReportDataSearch(request.getQuery(), request.getFrom(), request.getSize());
        break;
      default:
        searchSourceBuilder = buildAggregateSearchBuilder(request.getQuery(), request.getFrom(), request.getSize());
@ -329,7 +330,8 @@ public class OpenSearchClient implements SearchClient {
    if (request.getIndex().equalsIgnoreCase("domain_search_index")
        || request.getIndex().equalsIgnoreCase("data_products_search_index")
        || request.getIndex().equalsIgnoreCase("query_search_index")
-        || request.getIndex().equalsIgnoreCase("raw_cost_analysis_report_data_index")) {
+        || request.getIndex().equalsIgnoreCase("raw_cost_analysis_report_data_index")
+        || request.getIndex().equalsIgnoreCase("aggregated_cost_analysis_report_data_index")) {
      searchSourceBuilder.query(QueryBuilders.boolQuery().must(searchSourceBuilder.query()));
    } else {
      searchSourceBuilder.query(
@ -799,7 +801,7 @@ public class OpenSearchClient implements SearchClient {
    return addAggregation(searchSourceBuilder);
  }

-  private static SearchSourceBuilder buildRawCostAnalysisReportDataSearch(String query, int from, int size) {
+  private static SearchSourceBuilder buildCostAnalysisReportDataSearch(String query, int from, int size) {
    QueryStringQueryBuilder queryBuilder = QueryBuilders.queryStringQuery(query);
    return searchBuilder(queryBuilder, null, from, size);
  }
--- a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OsUnusedAssetsAggregator.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OsUnusedAssetsAggregator.java
@ -1,5 +1,6 @@
 package org.openmetadata.service.search.opensearch;

+import com.fasterxml.jackson.databind.ObjectMapper;
 import java.text.ParseException;
 import java.util.ArrayList;
 import java.util.HashMap;
@ -8,12 +9,15 @@ import lombok.extern.slf4j.Slf4j;
 import org.apache.lucene.search.TotalHits;
 import org.openmetadata.schema.dataInsight.DataInsightChartResult;
 import org.openmetadata.schema.dataInsight.type.UnusedAssets;
+import org.openmetadata.schema.type.EntityReference;
 import org.openmetadata.service.dataInsight.DataInsightAggregatorInterface;
 import org.opensearch.search.SearchHit;
 import org.opensearch.search.SearchHits;

@Slf4j
 public class OsUnusedAssetsAggregator extends DataInsightAggregatorInterface {
+  final ObjectMapper mapper = new ObjectMapper();
+
  public OsUnusedAssetsAggregator(SearchHits hits, DataInsightChartResult.DataInsightChartType dataInsightChartType) {
    super(hits, dataInsightChartType);
  }
@ -38,18 +42,15 @@ public class OsUnusedAssetsAggregator extends DataInsightAggregatorInterface {
    for (SearchHit hit : this.hitsOs) {
      try {
        HashMap<String, Object> data = (HashMap<String, Object>) hit.getSourceAsMap().get("data");
-        String fqn = ((HashMap<String, String>) data.get("entity")).get("fullyQualifiedName");
        Long lastAccessed =
            (Long)
                ((HashMap<String, Object>) ((HashMap<String, Object>) data.get("lifeCycle")).get("accessed"))
                    .get("timestamp");
        Double sizeInByte = (Double) data.get("sizeInByte");
-        new UnusedAssets().withFullyQualifiedName(fqn).withLastAccessedAt(lastAccessed).withSizeInBytes(sizeInByte);
-        dataList.add(
-            new UnusedAssets()
-                .withFullyQualifiedName(fqn)
-                .withLastAccessedAt(lastAccessed)
-                .withSizeInBytes(sizeInByte));
+        EntityReference entityReference = mapper.convertValue(data.get("entity"), EntityReference.class);
+        UnusedAssets unusedAssets =
+            new UnusedAssets().withEntity(entityReference).withLastAccessedAt(lastAccessed).withSizeInBytes(sizeInByte);
+        dataList.add(unusedAssets);
      } catch (Exception e) {
        LOG.error("Error while parsing hits for UnusedData chart from ES", e);
      }
--- a/openmetadata-spec/src/main/resources/json/schema/analytics/reportDataType/aggregatedCostAnalysisReportData.json
+++ b/openmetadata-spec/src/main/resources/json/schema/analytics/reportDataType/aggregatedCostAnalysisReportData.json
@ -4,7 +4,7 @@
  "$schema": "http://json-schema.org/draft-07/schema#",
  "title": "aggregatedCostAnalysisReportData",
  "type": "object",
-  "javaType": "org.openmetadata.schema.analytics.AggregatedCostAnalysisReportData.json",
+  "javaType": "org.openmetadata.schema.analytics.AggregatedCostAnalysisReportData",
  "description": "Aggregated data for Cost Analysis Report.",
  "definitions": {
    "dataAssetCount": {
@ -32,8 +32,7 @@
          "type": "number"
        }
      },
-      "additionalProperties": false,
-      "required": ["entity"]
+      "additionalProperties": false
    }
  },
  "properties": {
--- a/openmetadata-spec/src/main/resources/json/schema/analytics/reportDataType/rawCostAnalysisReportData.json
+++ b/openmetadata-spec/src/main/resources/json/schema/analytics/reportDataType/rawCostAnalysisReportData.json
@ -3,7 +3,7 @@
  "$schema": "http://json-schema.org/draft-07/schema#",
  "title": "rawCostAnalysisReportData",
  "type": "object",
-  "javaType": "org.openmetadata.schema.analytics.RawCostAnalysisReportData.json",
+  "javaType": "org.openmetadata.schema.analytics.RawCostAnalysisReportData",
  "description": "Raw data for Cost Analysis Report.",
  "properties": {
    "entity": {
--- a/openmetadata-spec/src/main/resources/json/schema/dataInsight/type/unusedAssets.json
+++ b/openmetadata-spec/src/main/resources/json/schema/dataInsight/type/unusedAssets.json
@ -6,9 +6,9 @@
  "type": "object",
  "javaType": "org.openmetadata.schema.dataInsight.type.UnusedAssets",
  "properties": {
-    "fullyQualifiedName": {
-      "description": "Fully qualified name of the asset",
-      "$ref": "../../type/basic.json#/definitions/fullyQualifiedEntityName"
+    "entity": {
+      "description": "Entity of the life cycle data",
+      "$ref": "../../type/entityReference.json"
    },
    "sizeInBytes": {
      "description": "Size of the asset in bytes",