yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-09-01 05:03:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

Added entity_ref instead of fqn in unused data assets aggregrate API (#13421)

Browse Source
This commit is contained in:
Onkar Ravgan 2023-10-04 15:07:09 +05:30 committed by GitHub
parent 0282574bdd
commit 13f2d3e25e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 32 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java
View File

@ -297,8 +297,9 @@ public class ElasticSearchClient implements SearchClient {
searchSourceBuilder = buildDomainsSearch(request.getQuery(), request.getFrom(), request.getSize()); searchSourceBuilder = buildDomainsSearch(request.getQuery(), request.getFrom(), request.getSize());
break; break;
case "raw_cost_analysis_report_data_index": case "raw_cost_analysis_report_data_index":
case "aggregated_cost_analysis_report_data_index":
searchSourceBuilder = searchSourceBuilder =
buildRawCostAnalysisReportDataSearch(request.getQuery(), request.getFrom(), request.getSize()); buildCostAnalysisReportDataSearch(request.getQuery(), request.getFrom(), request.getSize());
break; break;
default: default:
searchSourceBuilder = buildAggregateSearchBuilder(request.getQuery(), request.getFrom(), request.getSize()); searchSourceBuilder = buildAggregateSearchBuilder(request.getQuery(), request.getFrom(), request.getSize());
@ -335,7 +336,8 @@ public class ElasticSearchClient implements SearchClient {
if (request.getIndex().equalsIgnoreCase("domain_search_index") if (request.getIndex().equalsIgnoreCase("domain_search_index")
|| request.getIndex().equalsIgnoreCase("data_products_search_index") || request.getIndex().equalsIgnoreCase("data_products_search_index")
|| request.getIndex().equalsIgnoreCase("query_search_index") || request.getIndex().equalsIgnoreCase("query_search_index")
|| request.getIndex().equalsIgnoreCase("raw_cost_analysis_report_data_index")) { || request.getIndex().equalsIgnoreCase("raw_cost_analysis_report_data_index")
|| request.getIndex().equalsIgnoreCase("aggregated_cost_analysis_report_data_index")) {
searchSourceBuilder.query(QueryBuilders.boolQuery().must(searchSourceBuilder.query())); searchSourceBuilder.query(QueryBuilders.boolQuery().must(searchSourceBuilder.query()));
} else { } else {
searchSourceBuilder.query( searchSourceBuilder.query(
@ -807,7 +809,7 @@ public class ElasticSearchClient implements SearchClient {
return searchBuilder(queryBuilder, hb, from, size); return searchBuilder(queryBuilder, hb, from, size);
} }
private static SearchSourceBuilder buildRawCostAnalysisReportDataSearch(String query, int from, int size) { private static SearchSourceBuilder buildCostAnalysisReportDataSearch(String query, int from, int size) {
QueryStringQueryBuilder queryBuilder = QueryBuilders.queryStringQuery(query); QueryStringQueryBuilder queryBuilder = QueryBuilders.queryStringQuery(query);
return searchBuilder(queryBuilder, null, from, size); return searchBuilder(queryBuilder, null, from, size);
} }

15
openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/EsUnusedAssetsAggregator.java
View File

@ -1,5 +1,6 @@
package org.openmetadata.service.search.elasticsearch; package org.openmetadata.service.search.elasticsearch;
import com.fasterxml.jackson.databind.ObjectMapper;
import java.text.ParseException; import java.text.ParseException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
@ -10,11 +11,14 @@ import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.SearchHits;
import org.openmetadata.schema.dataInsight.DataInsightChartResult; import org.openmetadata.schema.dataInsight.DataInsightChartResult;
import org.openmetadata.schema.dataInsight.type.UnusedAssets; import org.openmetadata.schema.dataInsight.type.UnusedAssets;
import org.openmetadata.schema.type.EntityReference;
import org.openmetadata.service.dataInsight.DataInsightAggregatorInterface; import org.openmetadata.service.dataInsight.DataInsightAggregatorInterface;
// TODO: refactor this class and the interface in https://github.com/open-metadata/OpenMetadata/issues/13401 // TODO: refactor this class and the interface in https://github.com/open-metadata/OpenMetadata/issues/13401
@Slf4j @Slf4j
public class EsUnusedAssetsAggregator extends DataInsightAggregatorInterface { public class EsUnusedAssetsAggregator extends DataInsightAggregatorInterface {
final ObjectMapper mapper = new ObjectMapper();
public EsUnusedAssetsAggregator(SearchHits hits, DataInsightChartResult.DataInsightChartType dataInsightChartType) { public EsUnusedAssetsAggregator(SearchHits hits, DataInsightChartResult.DataInsightChartType dataInsightChartType) {
super(hits, dataInsightChartType); super(hits, dataInsightChartType);
} }
@ -39,18 +43,15 @@ public class EsUnusedAssetsAggregator extends DataInsightAggregatorInterface {
for (SearchHit hit : this.hitsEs) { for (SearchHit hit : this.hitsEs) {
try { try {
HashMap<String, Object> data = (HashMap<String, Object>) hit.getSourceAsMap().get("data"); HashMap<String, Object> data = (HashMap<String, Object>) hit.getSourceAsMap().get("data");
String fqn = ((HashMap<String, String>) data.get("entity")).get("fullyQualifiedName");
Long lastAccessed = Long lastAccessed =
(Long) (Long)
((HashMap<String, Object>) ((HashMap<String, Object>) data.get("lifeCycle")).get("accessed")) ((HashMap<String, Object>) ((HashMap<String, Object>) data.get("lifeCycle")).get("accessed"))
.get("timestamp"); .get("timestamp");
Double sizeInByte = (Double) data.get("sizeInByte"); Double sizeInByte = (Double) data.get("sizeInByte");
new UnusedAssets().withFullyQualifiedName(fqn).withLastAccessedAt(lastAccessed).withSizeInBytes(sizeInByte); EntityReference entityReference = mapper.convertValue(data.get("entity"), EntityReference.class);
dataList.add( UnusedAssets unusedAssets =
new UnusedAssets() new UnusedAssets().withEntity(entityReference).withLastAccessedAt(lastAccessed).withSizeInBytes(sizeInByte);
.withFullyQualifiedName(fqn) dataList.add(unusedAssets);
.withLastAccessedAt(lastAccessed)
.withSizeInBytes(sizeInByte));
} catch (Exception e) { } catch (Exception e) {
LOG.error("Error while parsing hits for UnusedData chart from ES", e); LOG.error("Error while parsing hits for UnusedData chart from ES", e);
} }

8
openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java
View File

@ -291,8 +291,9 @@ public class OpenSearchClient implements SearchClient {
searchSourceBuilder = buildSearchEntitySearch(request.getQuery(), request.getFrom(), request.getSize()); searchSourceBuilder = buildSearchEntitySearch(request.getQuery(), request.getFrom(), request.getSize());
break; break;
case "raw_cost_analysis_report_data_index": case "raw_cost_analysis_report_data_index":
case "aggregated_cost_analysis_report_data_index":
searchSourceBuilder = searchSourceBuilder =
buildRawCostAnalysisReportDataSearch(request.getQuery(), request.getFrom(), request.getSize()); buildCostAnalysisReportDataSearch(request.getQuery(), request.getFrom(), request.getSize());
break; break;
default: default:
searchSourceBuilder = buildAggregateSearchBuilder(request.getQuery(), request.getFrom(), request.getSize()); searchSourceBuilder = buildAggregateSearchBuilder(request.getQuery(), request.getFrom(), request.getSize());
@ -329,7 +330,8 @@ public class OpenSearchClient implements SearchClient {
if (request.getIndex().equalsIgnoreCase("domain_search_index") if (request.getIndex().equalsIgnoreCase("domain_search_index")
|| request.getIndex().equalsIgnoreCase("data_products_search_index") || request.getIndex().equalsIgnoreCase("data_products_search_index")
|| request.getIndex().equalsIgnoreCase("query_search_index") || request.getIndex().equalsIgnoreCase("query_search_index")
|| request.getIndex().equalsIgnoreCase("raw_cost_analysis_report_data_index")) { || request.getIndex().equalsIgnoreCase("raw_cost_analysis_report_data_index")
|| request.getIndex().equalsIgnoreCase("aggregated_cost_analysis_report_data_index")) {
searchSourceBuilder.query(QueryBuilders.boolQuery().must(searchSourceBuilder.query())); searchSourceBuilder.query(QueryBuilders.boolQuery().must(searchSourceBuilder.query()));
} else { } else {
searchSourceBuilder.query( searchSourceBuilder.query(
@ -799,7 +801,7 @@ public class OpenSearchClient implements SearchClient {
return addAggregation(searchSourceBuilder); return addAggregation(searchSourceBuilder);
} }
private static SearchSourceBuilder buildRawCostAnalysisReportDataSearch(String query, int from, int size) { private static SearchSourceBuilder buildCostAnalysisReportDataSearch(String query, int from, int size) {
QueryStringQueryBuilder queryBuilder = QueryBuilders.queryStringQuery(query); QueryStringQueryBuilder queryBuilder = QueryBuilders.queryStringQuery(query);
return searchBuilder(queryBuilder, null, from, size); return searchBuilder(queryBuilder, null, from, size);
} }

15
openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OsUnusedAssetsAggregator.java
View File

@ -1,5 +1,6 @@
package org.openmetadata.service.search.opensearch; package org.openmetadata.service.search.opensearch;
import com.fasterxml.jackson.databind.ObjectMapper;
import java.text.ParseException; import java.text.ParseException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
@ -8,12 +9,15 @@ import lombok.extern.slf4j.Slf4j;
import org.apache.lucene.search.TotalHits; import org.apache.lucene.search.TotalHits;
import org.openmetadata.schema.dataInsight.DataInsightChartResult; import org.openmetadata.schema.dataInsight.DataInsightChartResult;
import org.openmetadata.schema.dataInsight.type.UnusedAssets; import org.openmetadata.schema.dataInsight.type.UnusedAssets;
import org.openmetadata.schema.type.EntityReference;
import org.openmetadata.service.dataInsight.DataInsightAggregatorInterface; import org.openmetadata.service.dataInsight.DataInsightAggregatorInterface;
import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHit;
import org.opensearch.search.SearchHits; import org.opensearch.search.SearchHits;
@Slf4j @Slf4j
public class OsUnusedAssetsAggregator extends DataInsightAggregatorInterface { public class OsUnusedAssetsAggregator extends DataInsightAggregatorInterface {
final ObjectMapper mapper = new ObjectMapper();
public OsUnusedAssetsAggregator(SearchHits hits, DataInsightChartResult.DataInsightChartType dataInsightChartType) { public OsUnusedAssetsAggregator(SearchHits hits, DataInsightChartResult.DataInsightChartType dataInsightChartType) {
super(hits, dataInsightChartType); super(hits, dataInsightChartType);
} }
@ -38,18 +42,15 @@ public class OsUnusedAssetsAggregator extends DataInsightAggregatorInterface {
for (SearchHit hit : this.hitsOs) { for (SearchHit hit : this.hitsOs) {
try { try {
HashMap<String, Object> data = (HashMap<String, Object>) hit.getSourceAsMap().get("data"); HashMap<String, Object> data = (HashMap<String, Object>) hit.getSourceAsMap().get("data");
String fqn = ((HashMap<String, String>) data.get("entity")).get("fullyQualifiedName");
Long lastAccessed = Long lastAccessed =
(Long) (Long)
((HashMap<String, Object>) ((HashMap<String, Object>) data.get("lifeCycle")).get("accessed")) ((HashMap<String, Object>) ((HashMap<String, Object>) data.get("lifeCycle")).get("accessed"))
.get("timestamp"); .get("timestamp");
Double sizeInByte = (Double) data.get("sizeInByte"); Double sizeInByte = (Double) data.get("sizeInByte");
new UnusedAssets().withFullyQualifiedName(fqn).withLastAccessedAt(lastAccessed).withSizeInBytes(sizeInByte); EntityReference entityReference = mapper.convertValue(data.get("entity"), EntityReference.class);
dataList.add( UnusedAssets unusedAssets =
new UnusedAssets() new UnusedAssets().withEntity(entityReference).withLastAccessedAt(lastAccessed).withSizeInBytes(sizeInByte);
.withFullyQualifiedName(fqn) dataList.add(unusedAssets);
.withLastAccessedAt(lastAccessed)
.withSizeInBytes(sizeInByte));
} catch (Exception e) { } catch (Exception e) {
LOG.error("Error while parsing hits for UnusedData chart from ES", e); LOG.error("Error while parsing hits for UnusedData chart from ES", e);
} }

5
openmetadata-spec/src/main/resources/json/schema/analytics/reportDataType/aggregatedCostAnalysisReportData.json
View File

@ -4,7 +4,7 @@
"$schema": "http://json-schema.org/draft-07/schema#", "$schema": "http://json-schema.org/draft-07/schema#",
"title": "aggregatedCostAnalysisReportData", "title": "aggregatedCostAnalysisReportData",
"type": "object", "type": "object",
"javaType": "org.openmetadata.schema.analytics.AggregatedCostAnalysisReportData.json", "javaType": "org.openmetadata.schema.analytics.AggregatedCostAnalysisReportData",
"description": "Aggregated data for Cost Analysis Report.", "description": "Aggregated data for Cost Analysis Report.",
"definitions": { "definitions": {
"dataAssetCount": { "dataAssetCount": {
@ -32,8 +32,7 @@
"type": "number" "type": "number"
} }
}, },
"additionalProperties": false, "additionalProperties": false
"required": ["entity"]
} }
}, },
"properties": { "properties": {

2
openmetadata-spec/src/main/resources/json/schema/analytics/reportDataType/rawCostAnalysisReportData.json
View File

@ -3,7 +3,7 @@
"$schema": "http://json-schema.org/draft-07/schema#", "$schema": "http://json-schema.org/draft-07/schema#",
"title": "rawCostAnalysisReportData", "title": "rawCostAnalysisReportData",
"type": "object", "type": "object",
"javaType": "org.openmetadata.schema.analytics.RawCostAnalysisReportData.json", "javaType": "org.openmetadata.schema.analytics.RawCostAnalysisReportData",
"description": "Raw data for Cost Analysis Report.", "description": "Raw data for Cost Analysis Report.",
"properties": { "properties": {
"entity": { "entity": {

6
openmetadata-spec/src/main/resources/json/schema/dataInsight/type/unusedAssets.json
View File

@ -6,9 +6,9 @@
"type": "object", "type": "object",
"javaType": "org.openmetadata.schema.dataInsight.type.UnusedAssets", "javaType": "org.openmetadata.schema.dataInsight.type.UnusedAssets",
"properties": { "properties": {
"fullyQualifiedName": { "entity": {
"description": "Fully qualified name of the asset", "description": "Entity of the life cycle data",
"$ref": "../../type/basic.json#/definitions/fullyQualifiedEntityName" "$ref": "../../type/entityReference.json"
}, },
"sizeInBytes": { "sizeInBytes": {
"description": "Size of the asset in bytes", "description": "Size of the asset in bytes",