From 518fedd3fd31cabf4b1020ed02a045605dab2416 Mon Sep 17 00:00:00 2001 From: Teddy Date: Fri, 6 Oct 2023 08:42:38 +0200 Subject: [PATCH] Cost analysis charts (#13458) * feat: unused size report * feat: asset count report * feat: unused asset size * feat: used vs unused assets count * feat: used vs unused assets size * feat: clients + chart result json schema --- ...AggregatedUnusedAssetsCountAggregator.java | 56 +++++++++++++++++++ ...AggregatedUnusedAssetsSizeAggregator.java} | 17 ++++-- ...atedUsedvsUnusedAssetsCountAggregator.java | 52 +++++++++++++++++ ...gatedUsedvsUnusedAssetsSizeAggregator.java | 51 +++++++++++++++++ .../elasticsearch/ElasticSearchClient.java | 54 ++++++++++++++---- ...AggregatedUnusedAssetsCountAggregator.java | 40 +++++++++++++ ...AggregatedUnusedAssetsSizeAggregator.java} | 8 +-- ...atedUsedvsUnusedAssetsCountAggregator.java | 40 +++++++++++++ ...gatedUsedvsUnusedAssetsSizeAggregator.java | 40 +++++++++++++ .../search/opensearch/OpenSearchClient.java | 54 ++++++++++++++---- ...AggregatedUnusedAssetsCountAggregator.java | 40 +++++++++++++ ...AggregatedUnusedAssetsSizeAggregator.java} | 8 +-- ...atedUsedvsUnusedAssetsCountAggregator.java | 40 +++++++++++++ ...gatedUsedvsUnusedAssetsSizeAggregator.java | 40 +++++++++++++ .../aggregatedUnusedAssetsCount.json | 18 ++++++ ...s.json => aggregatedUnusedAssetsSize.json} | 19 ++++--- .../aggregatedUsedvsUnusedAssetsCount.json | 16 ++++++ .../aggregatedUsedvsUnusedAssetsSize.json | 16 ++++++ .../dataInsight/dataInsightChartResult.json | 10 +++- ....json => aggregatedUnusedAssetsCount.json} | 12 ++-- .../type/aggregatedUnusedAssetsSize.json | 39 +++++++++++++ .../aggregatedUsedVsUnusedAssetsCount.json | 31 ++++++++++ .../aggregatedUsedVsUnusedAssetsSize.json | 31 ++++++++++ 23 files changed, 684 insertions(+), 48 deletions(-) create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/dataInsight/AggregatedUnusedAssetsCountAggregator.java rename openmetadata-service/src/main/java/org/openmetadata/service/dataInsight/{AggregatedUnusedAssetsAggregator.java => AggregatedUnusedAssetsSizeAggregator.java} (69%) create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/dataInsight/AggregatedUsedvsUnusedAssetsCountAggregator.java create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/dataInsight/AggregatedUsedvsUnusedAssetsSizeAggregator.java create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/dataInsightAggregators/ElasticSearchAggregatedUnusedAssetsCountAggregator.java rename openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/dataInsightAggregators/{ElasticSearchAggregatedUnusedAssetsAggregator.java => ElasticSearchAggregatedUnusedAssetsSizeAggregator.java} (80%) create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/dataInsightAggregators/ElasticSearchAggregatedUsedvsUnusedAssetsCountAggregator.java create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/dataInsightAggregators/ElasticSearchAggregatedUsedvsUnusedAssetsSizeAggregator.java create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/dataInsightAggregator/OpenSearchAggregatedUnusedAssetsCountAggregator.java rename openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/dataInsightAggregator/{OpenSearchAggregatedUnusedAssetsAggregator.java => OpenSearchAggregatedUnusedAssetsSizeAggregator.java} (80%) create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/dataInsightAggregator/OpenSearchAggregatedUsedvsUnusedAssetsCountAggregator.java create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/dataInsightAggregator/OpenSearchAggregatedUsedvsUnusedAssetsSizeAggregator.java create mode 100644 openmetadata-service/src/main/resources/json/data/dataInsight/aggregatedUnusedAssetsCount.json rename openmetadata-service/src/main/resources/json/data/dataInsight/{aggregatedUnusedAssets.json => aggregatedUnusedAssetsSize.json} (56%) create mode 100644 openmetadata-service/src/main/resources/json/data/dataInsight/aggregatedUsedvsUnusedAssetsCount.json create mode 100644 openmetadata-service/src/main/resources/json/data/dataInsight/aggregatedUsedvsUnusedAssetsSize.json rename openmetadata-spec/src/main/resources/json/schema/dataInsight/type/{aggregatedUnusedAsserts.json => aggregatedUnusedAssetsCount.json} (79%) create mode 100644 openmetadata-spec/src/main/resources/json/schema/dataInsight/type/aggregatedUnusedAssetsSize.json create mode 100644 openmetadata-spec/src/main/resources/json/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsCount.json create mode 100644 openmetadata-spec/src/main/resources/json/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsSize.json diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/dataInsight/AggregatedUnusedAssetsCountAggregator.java b/openmetadata-service/src/main/java/org/openmetadata/service/dataInsight/AggregatedUnusedAssetsCountAggregator.java new file mode 100644 index 00000000000..ac3bd02e32a --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/dataInsight/AggregatedUnusedAssetsCountAggregator.java @@ -0,0 +1,56 @@ +package org.openmetadata.service.dataInsight; + +import java.text.ParseException; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import org.openmetadata.schema.dataInsight.type.AggregatedUnusedAssetsCount; + +public abstract class AggregatedUnusedAssetsCountAggregator implements DataInsightAggregatorInterface { + private final A aggregations; + + public AggregatedUnusedAssetsCountAggregator(A aggregations) { + this.aggregations = aggregations; + } + + @Override + public List aggregate() throws ParseException { + H histogramBucket = getHistogramBucket(this.aggregations); + List data = new ArrayList<>(); + for (B bucket : getBuckets(histogramBucket)) { + String dateTimeString = getKeyAsString(bucket); + Long timestamp = convertDatTimeStringToTimestamp(dateTimeString); + S threeDays = getAggregations(bucket, "threeDays"); + S sevenDays = getAggregations(bucket, "sevenDays"); + S fourteenDays = getAggregations(bucket, "fourteenDays"); + S thirtyDays = getAggregations(bucket, "thirtyDays"); + S sixtyDays = getAggregations(bucket, "sixtyDays"); + S totalUnused = getAggregations(bucket, "totalUnused"); + S totalUsed = getAggregations(bucket, "totalUsed"); + Double used = Objects.requireNonNullElse(getValue(totalUsed), 0.0); + Double unused = Objects.requireNonNullElse(getValue(totalUnused), 0.0); + Double total = used + unused; + + data.add( + new AggregatedUnusedAssetsCount() + .withTimestamp(timestamp) + .withThreeDays(getValue(threeDays)) + .withSevenDays(getValue(sevenDays)) + .withFourteenDays(getValue(fourteenDays)) + .withThirtyDays(getValue(thirtyDays)) + .withSixtyDays(getValue(sixtyDays)) + .withTotal(total)); + } + return data; + } + + protected abstract H getHistogramBucket(A aggregations); + + protected abstract List getBuckets(H histogramBucket); + + protected abstract String getKeyAsString(B bucket); + + protected abstract S getAggregations(B bucket, String key); + + protected abstract Double getValue(S aggregations); +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/dataInsight/AggregatedUnusedAssetsAggregator.java b/openmetadata-service/src/main/java/org/openmetadata/service/dataInsight/AggregatedUnusedAssetsSizeAggregator.java similarity index 69% rename from openmetadata-service/src/main/java/org/openmetadata/service/dataInsight/AggregatedUnusedAssetsAggregator.java rename to openmetadata-service/src/main/java/org/openmetadata/service/dataInsight/AggregatedUnusedAssetsSizeAggregator.java index df61f529ef4..d3f3e2842ec 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/dataInsight/AggregatedUnusedAssetsAggregator.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/dataInsight/AggregatedUnusedAssetsSizeAggregator.java @@ -3,12 +3,13 @@ package org.openmetadata.service.dataInsight; import java.text.ParseException; import java.util.ArrayList; import java.util.List; -import org.openmetadata.schema.dataInsight.type.AggregatedUnusedAssets; +import java.util.Objects; +import org.openmetadata.schema.dataInsight.type.AggregatedUnusedAssetsSize; -public abstract class AggregatedUnusedAssetsAggregator implements DataInsightAggregatorInterface { +public abstract class AggregatedUnusedAssetsSizeAggregator implements DataInsightAggregatorInterface { private final A aggregations; - public AggregatedUnusedAssetsAggregator(A aggregations) { + public AggregatedUnusedAssetsSizeAggregator(A aggregations) { this.aggregations = aggregations; } @@ -24,15 +25,21 @@ public abstract class AggregatedUnusedAssetsAggregator implements Da S fourteenDays = getAggregations(bucket, "fourteenDays"); S thirtyDays = getAggregations(bucket, "thirtyDays"); S sixtyDays = getAggregations(bucket, "sixtyDays"); + S totalUnused = getAggregations(bucket, "totalUnused"); + S totalUsed = getAggregations(bucket, "totalUsed"); + Double used = Objects.requireNonNullElse(getValue(totalUsed), 0.0); + Double unused = Objects.requireNonNullElse(getValue(totalUnused), 0.0); + Double total = unused + used; data.add( - new AggregatedUnusedAssets() + new AggregatedUnusedAssetsSize() .withTimestamp(timestamp) .withThreeDays(getValue(threeDays)) .withSevenDays(getValue(sevenDays)) .withFourteenDays(getValue(fourteenDays)) .withThirtyDays(getValue(thirtyDays)) - .withSixtyDays(getValue(sixtyDays))); + .withSixtyDays(getValue(sixtyDays)) + .withTotal(total)); } return data; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/dataInsight/AggregatedUsedvsUnusedAssetsCountAggregator.java b/openmetadata-service/src/main/java/org/openmetadata/service/dataInsight/AggregatedUsedvsUnusedAssetsCountAggregator.java new file mode 100644 index 00000000000..048a7ce25c9 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/dataInsight/AggregatedUsedvsUnusedAssetsCountAggregator.java @@ -0,0 +1,52 @@ +package org.openmetadata.service.dataInsight; + +import java.text.ParseException; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import org.openmetadata.schema.dataInsight.type.AggregatedUsedVsUnusedAssetsCount; + +public abstract class AggregatedUsedvsUnusedAssetsCountAggregator + implements DataInsightAggregatorInterface { + private final A aggregations; + + public AggregatedUsedvsUnusedAssetsCountAggregator(A aggregations) { + this.aggregations = aggregations; + } + + @Override + public List aggregate() throws ParseException { + H histogramBucket = getHistogramBucket(this.aggregations); + List data = new ArrayList<>(); + for (B bucket : getBuckets(histogramBucket)) { + String dateTimeString = getKeyAsString(bucket); + Long timestamp = convertDatTimeStringToTimestamp(dateTimeString); + S totalUnused = getAggregations(bucket, "totalUnused"); + S totalUsed = getAggregations(bucket, "totalUsed"); + Double used = Objects.requireNonNullElse(getValue(totalUsed), 0.0); + Double unused = Objects.requireNonNullElse(getValue(totalUnused), 0.0); + Double total = used + unused; + Double usedPercentage = used / total; + Double unusedPercentage = unused / total; + + data.add( + new AggregatedUsedVsUnusedAssetsCount() + .withTimestamp(timestamp) + .withUnused(unused) + .withUnusedPercentage(unusedPercentage) + .withUsed(used) + .withUsedPercentage(usedPercentage)); + } + return data; + } + + protected abstract H getHistogramBucket(A aggregations); + + protected abstract List getBuckets(H histogramBucket); + + protected abstract String getKeyAsString(B bucket); + + protected abstract S getAggregations(B bucket, String key); + + protected abstract Double getValue(S aggregations); +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/dataInsight/AggregatedUsedvsUnusedAssetsSizeAggregator.java b/openmetadata-service/src/main/java/org/openmetadata/service/dataInsight/AggregatedUsedvsUnusedAssetsSizeAggregator.java new file mode 100644 index 00000000000..5208caf76ee --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/dataInsight/AggregatedUsedvsUnusedAssetsSizeAggregator.java @@ -0,0 +1,51 @@ +package org.openmetadata.service.dataInsight; + +import java.text.ParseException; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import org.openmetadata.schema.dataInsight.type.AggregatedUsedVsUnusedAssetsSize; + +public abstract class AggregatedUsedvsUnusedAssetsSizeAggregator implements DataInsightAggregatorInterface { + private final A aggregations; + + public AggregatedUsedvsUnusedAssetsSizeAggregator(A aggregations) { + this.aggregations = aggregations; + } + + @Override + public List aggregate() throws ParseException { + H histogramBucket = getHistogramBucket(this.aggregations); + List data = new ArrayList<>(); + for (B bucket : getBuckets(histogramBucket)) { + String dateTimeString = getKeyAsString(bucket); + Long timestamp = convertDatTimeStringToTimestamp(dateTimeString); + S totalUnused = getAggregations(bucket, "totalUnused"); + S totalUsed = getAggregations(bucket, "totalUsed"); + Double used = Objects.requireNonNullElse(getValue(totalUsed), 0.0); + Double unused = Objects.requireNonNullElse(getValue(totalUnused), 0.0); + Double total = used + unused; + Double usedPercentage = used / total; + Double unusedPercentage = unused / total; + + data.add( + new AggregatedUsedVsUnusedAssetsSize() + .withTimestamp(timestamp) + .withUnused(unused) + .withUnusedPercentage(unusedPercentage) + .withUsed(used) + .withUsedPercentage(usedPercentage)); + } + return data; + } + + protected abstract H getHistogramBucket(A aggregations); + + protected abstract List getBuckets(H histogramBucket); + + protected abstract String getKeyAsString(B bucket); + + protected abstract S getAggregations(B bucket, String key); + + protected abstract Double getValue(S aggregations); +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java index 71cbc7e46ed..6bfe5b1367f 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java @@ -116,7 +116,10 @@ import org.openmetadata.service.jdbi3.DataInsightChartRepository; import org.openmetadata.service.search.SearchClient; import org.openmetadata.service.search.SearchRequest; import org.openmetadata.service.search.UpdateSearchEventsConstant; -import org.openmetadata.service.search.elasticsearch.dataInsightAggregators.ElasticSearchAggregatedUnusedAssetsAggregator; +import org.openmetadata.service.search.elasticsearch.dataInsightAggregators.ElasticSearchAggregatedUnusedAssetsCountAggregator; +import org.openmetadata.service.search.elasticsearch.dataInsightAggregators.ElasticSearchAggregatedUnusedAssetsSizeAggregator; +import org.openmetadata.service.search.elasticsearch.dataInsightAggregators.ElasticSearchAggregatedUsedvsUnusedAssetsCountAggregator; +import org.openmetadata.service.search.elasticsearch.dataInsightAggregators.ElasticSearchAggregatedUsedvsUnusedAssetsSizeAggregator; import org.openmetadata.service.search.elasticsearch.dataInsightAggregators.ElasticSearchDailyActiveUsersAggregator; import org.openmetadata.service.search.elasticsearch.dataInsightAggregators.ElasticSearchEntitiesDescriptionAggregator; import org.openmetadata.service.search.elasticsearch.dataInsightAggregators.ElasticSearchEntitiesOwnerAggregator; @@ -1186,8 +1189,14 @@ public class ElasticSearchClient implements SearchClient { return new ElasticSearchMostViewedEntitiesAggregator(aggregations.getAggregations()); case UNUSED_ASSETS: return new ElasticSearchUnusedAssetsAggregator(aggregations.getHits()); - case AGGREGATED_UNUSED_ASSETS: - return new ElasticSearchAggregatedUnusedAssetsAggregator(aggregations.getAggregations()); + case AGGREGATED_UNUSED_ASSETS_SIZE: + return new ElasticSearchAggregatedUnusedAssetsSizeAggregator(aggregations.getAggregations()); + case AGGREGATED_UNUSED_ASSETS_COUNT: + return new ElasticSearchAggregatedUnusedAssetsCountAggregator(aggregations.getAggregations()); + case AGGREGATED_USED_VS_UNUSED_ASSETS_COUNT: + return new ElasticSearchAggregatedUsedvsUnusedAssetsCountAggregator(aggregations.getAggregations()); + case AGGREGATED_USED_VS_UNUSED_ASSETS_SIZE: + return new ElasticSearchAggregatedUsedvsUnusedAssetsSizeAggregator(aggregations.getAggregations()); default: throw new IllegalArgumentException( String.format("No processor found for chart Type %s ", dataInsightChartType)); @@ -1306,23 +1315,48 @@ public class ElasticSearchClient implements SearchClient { termsAggregationBuilder .subAggregation(sumAggregationBuilder) .subAggregation(sumEntityCountAggregationBuilder)); - case AGGREGATED_UNUSED_ASSETS: + case AGGREGATED_UNUSED_ASSETS_SIZE: + case AGGREGATED_UNUSED_ASSETS_COUNT: + boolean isSize = + dataInsightChartName.equals(DataInsightChartResult.DataInsightChartType.AGGREGATED_UNUSED_ASSETS_SIZE); + String fieldType = isSize ? "size" : "count"; + String totalField = isSize ? "totalSize" : "totalCount"; SumAggregationBuilder threeDaysAgg = - AggregationBuilders.sum("threeDays").field("data.unusedDataAssets.threeDays"); + AggregationBuilders.sum("threeDays").field(String.format("data.unusedDataAssets.%s.threeDays", fieldType)); SumAggregationBuilder sevenDaysAgg = - AggregationBuilders.sum("sevenDays").field("data.unusedDataAssets.sevenDays"); + AggregationBuilders.sum("sevenDays").field(String.format("data.unusedDataAssets.%s.sevenDays", fieldType)); SumAggregationBuilder fourteenDaysAgg = - AggregationBuilders.sum("fourteenDays").field("data.unusedDataAssets.fourteenDays"); + AggregationBuilders.sum("fourteenDays") + .field(String.format("data.unusedDataAssets.%s.fourteenDays", fieldType)); SumAggregationBuilder thirtyDaysAgg = - AggregationBuilders.sum("thirtyDays").field("data.unusedDataAssets.thirtyDays"); + AggregationBuilders.sum("thirtyDays") + .field(String.format("data.unusedDataAssets.%s.thirtyDays", fieldType)); SumAggregationBuilder sixtyDaysAgg = - AggregationBuilders.sum("sixtyDays").field("data.unusedDataAssets.sixtyDays"); + AggregationBuilders.sum("sixtyDays").field(String.format("data.unusedDataAssets.%s.sixtyDays", fieldType)); + SumAggregationBuilder totalUnused = + AggregationBuilders.sum("totalUnused").field(String.format("data.unusedDataAssets.%s", totalField)); + SumAggregationBuilder totalUsed = + AggregationBuilders.sum("totalUsed").field(String.format("data.frequentlyUsedDataAssets.%s", totalField)); return dateHistogramAggregationBuilder .subAggregation(threeDaysAgg) .subAggregation(sevenDaysAgg) .subAggregation(fourteenDaysAgg) .subAggregation(thirtyDaysAgg) - .subAggregation(sixtyDaysAgg); + .subAggregation(sixtyDaysAgg) + .subAggregation(totalUnused) + .subAggregation(totalUsed); + case AGGREGATED_USED_VS_UNUSED_ASSETS_SIZE: + case AGGREGATED_USED_VS_UNUSED_ASSETS_COUNT: + boolean isSizeReport = + dataInsightChartName.equals( + DataInsightChartResult.DataInsightChartType.AGGREGATED_USED_VS_UNUSED_ASSETS_SIZE); + String totalFieldString = isSizeReport ? "totalSize" : "totalCount"; + SumAggregationBuilder totalUnusedAssets = + AggregationBuilders.sum("totalUnused").field(String.format("data.unusedDataAssets.%s", totalFieldString)); + SumAggregationBuilder totalUsedAssets = + AggregationBuilders.sum("totalUsed") + .field(String.format("data.frequentlyUsedDataAssets.%s", totalFieldString)); + return dateHistogramAggregationBuilder.subAggregation(totalUnusedAssets).subAggregation(totalUsedAssets); case PERCENTAGE_OF_SERVICES_WITH_DESCRIPTION: termsAggregationBuilder = AggregationBuilders.terms(DataInsightChartRepository.SERVICE_NAME) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/dataInsightAggregators/ElasticSearchAggregatedUnusedAssetsCountAggregator.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/dataInsightAggregators/ElasticSearchAggregatedUnusedAssetsCountAggregator.java new file mode 100644 index 00000000000..85faf233c20 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/dataInsightAggregators/ElasticSearchAggregatedUnusedAssetsCountAggregator.java @@ -0,0 +1,40 @@ +package org.openmetadata.service.search.elasticsearch.dataInsightAggregators; + +import java.util.List; +import org.elasticsearch.search.aggregations.Aggregations; +import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; +import org.elasticsearch.search.aggregations.bucket.histogram.Histogram.Bucket; +import org.elasticsearch.search.aggregations.metrics.Sum; +import org.openmetadata.service.dataInsight.AggregatedUnusedAssetsCountAggregator; + +public class ElasticSearchAggregatedUnusedAssetsCountAggregator + extends AggregatedUnusedAssetsCountAggregator { + public ElasticSearchAggregatedUnusedAssetsCountAggregator(Aggregations aggregations) { + super(aggregations); + } + + @Override + protected Histogram getHistogramBucket(Aggregations aggregations) { + return aggregations.get(TIMESTAMP); + } + + @Override + protected List getBuckets(Histogram histogramBucket) { + return histogramBucket.getBuckets(); + } + + @Override + protected String getKeyAsString(Bucket bucket) { + return bucket.getKeyAsString(); + } + + @Override + protected Sum getAggregations(Bucket bucket, String key) { + return bucket.getAggregations().get(key); + } + + @Override + protected Double getValue(Sum aggregations) { + return aggregations.getValue(); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/dataInsightAggregators/ElasticSearchAggregatedUnusedAssetsAggregator.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/dataInsightAggregators/ElasticSearchAggregatedUnusedAssetsSizeAggregator.java similarity index 80% rename from openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/dataInsightAggregators/ElasticSearchAggregatedUnusedAssetsAggregator.java rename to openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/dataInsightAggregators/ElasticSearchAggregatedUnusedAssetsSizeAggregator.java index f9ee48d905f..604952578f3 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/dataInsightAggregators/ElasticSearchAggregatedUnusedAssetsAggregator.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/dataInsightAggregators/ElasticSearchAggregatedUnusedAssetsSizeAggregator.java @@ -5,11 +5,11 @@ import org.elasticsearch.search.aggregations.Aggregations; import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; import org.elasticsearch.search.aggregations.bucket.histogram.Histogram.Bucket; import org.elasticsearch.search.aggregations.metrics.Sum; -import org.openmetadata.service.dataInsight.AggregatedUnusedAssetsAggregator; +import org.openmetadata.service.dataInsight.AggregatedUnusedAssetsSizeAggregator; -public class ElasticSearchAggregatedUnusedAssetsAggregator - extends AggregatedUnusedAssetsAggregator { - public ElasticSearchAggregatedUnusedAssetsAggregator(Aggregations aggregations) { +public class ElasticSearchAggregatedUnusedAssetsSizeAggregator + extends AggregatedUnusedAssetsSizeAggregator { + public ElasticSearchAggregatedUnusedAssetsSizeAggregator(Aggregations aggregations) { super(aggregations); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/dataInsightAggregators/ElasticSearchAggregatedUsedvsUnusedAssetsCountAggregator.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/dataInsightAggregators/ElasticSearchAggregatedUsedvsUnusedAssetsCountAggregator.java new file mode 100644 index 00000000000..b1bbd3c1119 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/dataInsightAggregators/ElasticSearchAggregatedUsedvsUnusedAssetsCountAggregator.java @@ -0,0 +1,40 @@ +package org.openmetadata.service.search.elasticsearch.dataInsightAggregators; + +import java.util.List; +import org.elasticsearch.search.aggregations.Aggregations; +import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; +import org.elasticsearch.search.aggregations.bucket.histogram.Histogram.Bucket; +import org.elasticsearch.search.aggregations.metrics.Sum; +import org.openmetadata.service.dataInsight.AggregatedUsedvsUnusedAssetsCountAggregator; + +public class ElasticSearchAggregatedUsedvsUnusedAssetsCountAggregator + extends AggregatedUsedvsUnusedAssetsCountAggregator { + public ElasticSearchAggregatedUsedvsUnusedAssetsCountAggregator(Aggregations aggregations) { + super(aggregations); + } + + @Override + protected Histogram getHistogramBucket(Aggregations aggregations) { + return aggregations.get(TIMESTAMP); + } + + @Override + protected List getBuckets(Histogram histogramBucket) { + return histogramBucket.getBuckets(); + } + + @Override + protected String getKeyAsString(Bucket bucket) { + return bucket.getKeyAsString(); + } + + @Override + protected Sum getAggregations(Bucket bucket, String key) { + return bucket.getAggregations().get(key); + } + + @Override + protected Double getValue(Sum aggregations) { + return aggregations.getValue(); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/dataInsightAggregators/ElasticSearchAggregatedUsedvsUnusedAssetsSizeAggregator.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/dataInsightAggregators/ElasticSearchAggregatedUsedvsUnusedAssetsSizeAggregator.java new file mode 100644 index 00000000000..38faccf3107 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/dataInsightAggregators/ElasticSearchAggregatedUsedvsUnusedAssetsSizeAggregator.java @@ -0,0 +1,40 @@ +package org.openmetadata.service.search.elasticsearch.dataInsightAggregators; + +import java.util.List; +import org.elasticsearch.search.aggregations.Aggregations; +import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; +import org.elasticsearch.search.aggregations.bucket.histogram.Histogram.Bucket; +import org.elasticsearch.search.aggregations.metrics.Sum; +import org.openmetadata.service.dataInsight.AggregatedUsedvsUnusedAssetsSizeAggregator; + +public class ElasticSearchAggregatedUsedvsUnusedAssetsSizeAggregator + extends AggregatedUsedvsUnusedAssetsSizeAggregator { + public ElasticSearchAggregatedUsedvsUnusedAssetsSizeAggregator(Aggregations aggregations) { + super(aggregations); + } + + @Override + protected Histogram getHistogramBucket(Aggregations aggregations) { + return aggregations.get(TIMESTAMP); + } + + @Override + protected List getBuckets(Histogram histogramBucket) { + return histogramBucket.getBuckets(); + } + + @Override + protected String getKeyAsString(Bucket bucket) { + return bucket.getKeyAsString(); + } + + @Override + protected Sum getAggregations(Bucket bucket, String key) { + return bucket.getAggregations().get(key); + } + + @Override + protected Double getValue(Sum aggregations) { + return aggregations.getValue(); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java index 779aa78af78..7f8bfe24a05 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java @@ -64,7 +64,10 @@ import org.openmetadata.service.search.indexes.TestCaseIndex; import org.openmetadata.service.search.indexes.TopicIndex; import org.openmetadata.service.search.indexes.UserIndex; import org.openmetadata.service.search.models.IndexMapping; -import org.openmetadata.service.search.opensearch.dataInsightAggregator.OpenSearchAggregatedUnusedAssetsAggregator; +import org.openmetadata.service.search.opensearch.dataInsightAggregator.OpenSearchAggregatedUnusedAssetsCountAggregator; +import org.openmetadata.service.search.opensearch.dataInsightAggregator.OpenSearchAggregatedUnusedAssetsSizeAggregator; +import org.openmetadata.service.search.opensearch.dataInsightAggregator.OpenSearchAggregatedUsedvsUnusedAssetsCountAggregator; +import org.openmetadata.service.search.opensearch.dataInsightAggregator.OpenSearchAggregatedUsedvsUnusedAssetsSizeAggregator; import org.openmetadata.service.search.opensearch.dataInsightAggregator.OpenSearchDailyActiveUsersAggregator; import org.openmetadata.service.search.opensearch.dataInsightAggregator.OpenSearchEntitiesDescriptionAggregator; import org.openmetadata.service.search.opensearch.dataInsightAggregator.OpenSearchEntitiesOwnerAggregator; @@ -1191,8 +1194,14 @@ public class OpenSearchClient implements SearchClient { return new OpenSearchMostViewedEntitiesAggregator(aggregations.getAggregations()); case UNUSED_ASSETS: return new OpenSearchUnusedAssetsAggregator(aggregations.getHits()); - case AGGREGATED_UNUSED_ASSETS: - return new OpenSearchAggregatedUnusedAssetsAggregator(aggregations.getAggregations()); + case AGGREGATED_UNUSED_ASSETS_SIZE: + return new OpenSearchAggregatedUnusedAssetsSizeAggregator(aggregations.getAggregations()); + case AGGREGATED_UNUSED_ASSETS_COUNT: + return new OpenSearchAggregatedUnusedAssetsCountAggregator(aggregations.getAggregations()); + case AGGREGATED_USED_VS_UNUSED_ASSETS_COUNT: + return new OpenSearchAggregatedUsedvsUnusedAssetsCountAggregator(aggregations.getAggregations()); + case AGGREGATED_USED_VS_UNUSED_ASSETS_SIZE: + return new OpenSearchAggregatedUsedvsUnusedAssetsSizeAggregator(aggregations.getAggregations()); default: throw new IllegalArgumentException( String.format("No processor found for chart Type %s ", dataInsightChartType)); @@ -1307,23 +1316,48 @@ public class OpenSearchClient implements SearchClient { termsAggregationBuilder .subAggregation(sumAggregationBuilder) .subAggregation(sumEntityCountAggregationBuilder)); - case AGGREGATED_UNUSED_ASSETS: + case AGGREGATED_UNUSED_ASSETS_COUNT: + case AGGREGATED_UNUSED_ASSETS_SIZE: + boolean isSize = + dataInsightChartName.equals(DataInsightChartResult.DataInsightChartType.AGGREGATED_UNUSED_ASSETS_SIZE); + String fieldType = isSize ? "size" : "count"; + String totalField = isSize ? "totalSize" : "totalCount"; SumAggregationBuilder threeDaysAgg = - AggregationBuilders.sum("threeDays").field("data.unusedDataAssets.threeDays"); + AggregationBuilders.sum("threeDays").field(String.format("data.unusedDataAssets.%s.threeDays", fieldType)); SumAggregationBuilder sevenDaysAgg = - AggregationBuilders.sum("sevenDays").field("data.unusedDataAssets.sevenDays"); + AggregationBuilders.sum("sevenDays").field(String.format("data.unusedDataAssets.%s.sevenDays", fieldType)); SumAggregationBuilder fourteenDaysAgg = - AggregationBuilders.sum("fourteenDays").field("data.unusedDataAssets.fourteenDays"); + AggregationBuilders.sum("fourteenDays") + .field(String.format("data.unusedDataAssets.%s.fourteenDays", fieldType)); SumAggregationBuilder thirtyDaysAgg = - AggregationBuilders.sum("thirtyDays").field("data.unusedDataAssets.thirtyDays"); + AggregationBuilders.sum("thirtyDays") + .field(String.format("data.unusedDataAssets.%s.thirtyDays", fieldType)); SumAggregationBuilder sixtyDaysAgg = - AggregationBuilders.sum("sixtyDays").field("data.unusedDataAssets.sixtyDays"); + AggregationBuilders.sum("sixtyDays").field(String.format("data.unusedDataAssets.%s.sixtyDays", fieldType)); + SumAggregationBuilder totalUnused = + AggregationBuilders.sum("totalUnused").field(String.format("data.unusedDataAssets.%s", totalField)); + SumAggregationBuilder totalUsed = + AggregationBuilders.sum("totalUsed").field(String.format("data.unusedDataAssets.%s", totalField)); return dateHistogramAggregationBuilder .subAggregation(threeDaysAgg) .subAggregation(sevenDaysAgg) .subAggregation(fourteenDaysAgg) .subAggregation(thirtyDaysAgg) - .subAggregation(sixtyDaysAgg); + .subAggregation(sixtyDaysAgg) + .subAggregation(totalUnused) + .subAggregation(totalUsed); + case AGGREGATED_USED_VS_UNUSED_ASSETS_SIZE: + case AGGREGATED_USED_VS_UNUSED_ASSETS_COUNT: + boolean isSizeReport = + dataInsightChartName.equals( + DataInsightChartResult.DataInsightChartType.AGGREGATED_USED_VS_UNUSED_ASSETS_SIZE); + String totalFieldString = isSizeReport ? "totalSize" : "totalCount"; + SumAggregationBuilder totalUnusedAssets = + AggregationBuilders.sum("totalUnused").field(String.format("data.unusedDataAssets.%s", totalFieldString)); + SumAggregationBuilder totalUsedAssets = + AggregationBuilders.sum("totalUsed") + .field(String.format("data.frequentlyUsedDataAssets.%s", totalFieldString)); + return dateHistogramAggregationBuilder.subAggregation(totalUnusedAssets).subAggregation(totalUsedAssets); case PERCENTAGE_OF_SERVICES_WITH_DESCRIPTION: termsAggregationBuilder = AggregationBuilders.terms(DataInsightChartRepository.SERVICE_NAME) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/dataInsightAggregator/OpenSearchAggregatedUnusedAssetsCountAggregator.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/dataInsightAggregator/OpenSearchAggregatedUnusedAssetsCountAggregator.java new file mode 100644 index 00000000000..30000d3e1f8 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/dataInsightAggregator/OpenSearchAggregatedUnusedAssetsCountAggregator.java @@ -0,0 +1,40 @@ +package org.openmetadata.service.search.opensearch.dataInsightAggregator; + +import java.util.List; +import org.openmetadata.service.dataInsight.AggregatedUnusedAssetsCountAggregator; +import org.opensearch.search.aggregations.Aggregations; +import org.opensearch.search.aggregations.bucket.histogram.Histogram; +import org.opensearch.search.aggregations.bucket.histogram.Histogram.Bucket; +import org.opensearch.search.aggregations.metrics.Sum; + +public class OpenSearchAggregatedUnusedAssetsCountAggregator + extends AggregatedUnusedAssetsCountAggregator { + public OpenSearchAggregatedUnusedAssetsCountAggregator(Aggregations aggregations) { + super(aggregations); + } + + @Override + protected Histogram getHistogramBucket(Aggregations aggregations) { + return aggregations.get(TIMESTAMP); + } + + @Override + protected List getBuckets(Histogram histogramBucket) { + return histogramBucket.getBuckets(); + } + + @Override + protected String getKeyAsString(Bucket bucket) { + return bucket.getKeyAsString(); + } + + @Override + protected Sum getAggregations(Bucket bucket, String key) { + return bucket.getAggregations().get(key); + } + + @Override + protected Double getValue(Sum aggregations) { + return aggregations.getValue(); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/dataInsightAggregator/OpenSearchAggregatedUnusedAssetsAggregator.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/dataInsightAggregator/OpenSearchAggregatedUnusedAssetsSizeAggregator.java similarity index 80% rename from openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/dataInsightAggregator/OpenSearchAggregatedUnusedAssetsAggregator.java rename to openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/dataInsightAggregator/OpenSearchAggregatedUnusedAssetsSizeAggregator.java index d67fe35499a..d61c81aeebd 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/dataInsightAggregator/OpenSearchAggregatedUnusedAssetsAggregator.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/dataInsightAggregator/OpenSearchAggregatedUnusedAssetsSizeAggregator.java @@ -1,15 +1,15 @@ package org.openmetadata.service.search.opensearch.dataInsightAggregator; import java.util.List; -import org.openmetadata.service.dataInsight.AggregatedUnusedAssetsAggregator; +import org.openmetadata.service.dataInsight.AggregatedUnusedAssetsSizeAggregator; import org.opensearch.search.aggregations.Aggregations; import org.opensearch.search.aggregations.bucket.histogram.Histogram; import org.opensearch.search.aggregations.bucket.histogram.Histogram.Bucket; import org.opensearch.search.aggregations.metrics.Sum; -public class OpenSearchAggregatedUnusedAssetsAggregator - extends AggregatedUnusedAssetsAggregator { - public OpenSearchAggregatedUnusedAssetsAggregator(Aggregations aggregations) { +public class OpenSearchAggregatedUnusedAssetsSizeAggregator + extends AggregatedUnusedAssetsSizeAggregator { + public OpenSearchAggregatedUnusedAssetsSizeAggregator(Aggregations aggregations) { super(aggregations); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/dataInsightAggregator/OpenSearchAggregatedUsedvsUnusedAssetsCountAggregator.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/dataInsightAggregator/OpenSearchAggregatedUsedvsUnusedAssetsCountAggregator.java new file mode 100644 index 00000000000..cb13d274ee8 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/dataInsightAggregator/OpenSearchAggregatedUsedvsUnusedAssetsCountAggregator.java @@ -0,0 +1,40 @@ +package org.openmetadata.service.search.opensearch.dataInsightAggregator; + +import java.util.List; +import org.openmetadata.service.dataInsight.AggregatedUsedvsUnusedAssetsCountAggregator; +import org.opensearch.search.aggregations.Aggregations; +import org.opensearch.search.aggregations.bucket.histogram.Histogram; +import org.opensearch.search.aggregations.bucket.histogram.Histogram.Bucket; +import org.opensearch.search.aggregations.metrics.Sum; + +public class OpenSearchAggregatedUsedvsUnusedAssetsCountAggregator + extends AggregatedUsedvsUnusedAssetsCountAggregator { + public OpenSearchAggregatedUsedvsUnusedAssetsCountAggregator(Aggregations aggregations) { + super(aggregations); + } + + @Override + protected Histogram getHistogramBucket(Aggregations aggregations) { + return aggregations.get(TIMESTAMP); + } + + @Override + protected List getBuckets(Histogram histogramBucket) { + return histogramBucket.getBuckets(); + } + + @Override + protected String getKeyAsString(Bucket bucket) { + return bucket.getKeyAsString(); + } + + @Override + protected Sum getAggregations(Bucket bucket, String key) { + return bucket.getAggregations().get(key); + } + + @Override + protected Double getValue(Sum aggregations) { + return aggregations.getValue(); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/dataInsightAggregator/OpenSearchAggregatedUsedvsUnusedAssetsSizeAggregator.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/dataInsightAggregator/OpenSearchAggregatedUsedvsUnusedAssetsSizeAggregator.java new file mode 100644 index 00000000000..17e46cd0bae --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/dataInsightAggregator/OpenSearchAggregatedUsedvsUnusedAssetsSizeAggregator.java @@ -0,0 +1,40 @@ +package org.openmetadata.service.search.opensearch.dataInsightAggregator; + +import java.util.List; +import org.openmetadata.service.dataInsight.AggregatedUsedvsUnusedAssetsSizeAggregator; +import org.opensearch.search.aggregations.Aggregations; +import org.opensearch.search.aggregations.bucket.histogram.Histogram; +import org.opensearch.search.aggregations.bucket.histogram.Histogram.Bucket; +import org.opensearch.search.aggregations.metrics.Sum; + +public class OpenSearchAggregatedUsedvsUnusedAssetsSizeAggregator + extends AggregatedUsedvsUnusedAssetsSizeAggregator { + public OpenSearchAggregatedUsedvsUnusedAssetsSizeAggregator(Aggregations aggregations) { + super(aggregations); + } + + @Override + protected Histogram getHistogramBucket(Aggregations aggregations) { + return aggregations.get(TIMESTAMP); + } + + @Override + protected List getBuckets(Histogram histogramBucket) { + return histogramBucket.getBuckets(); + } + + @Override + protected String getKeyAsString(Bucket bucket) { + return bucket.getKeyAsString(); + } + + @Override + protected Sum getAggregations(Bucket bucket, String key) { + return bucket.getAggregations().get(key); + } + + @Override + protected Double getValue(Sum aggregations) { + return aggregations.getValue(); + } +} diff --git a/openmetadata-service/src/main/resources/json/data/dataInsight/aggregatedUnusedAssetsCount.json b/openmetadata-service/src/main/resources/json/data/dataInsight/aggregatedUnusedAssetsCount.json new file mode 100644 index 00000000000..9e137126ca5 --- /dev/null +++ b/openmetadata-service/src/main/resources/json/data/dataInsight/aggregatedUnusedAssetsCount.json @@ -0,0 +1,18 @@ +{ + "name": "AggregatedUnusedAssetsCount", + "fullyQualifiedName": "AggregatedUnusedAssetsCount", + "displayName": "Unused Assets Count", + "description": "Displays the total count of unused assets over time.", + "dataIndexType": "aggregated_cost_analysis_report_data_index", + "dimensions": [ + {"name": "timestamp","chartDataType":"INT"} + ], + "metrics": [ + {"name": "threeDays", "displayName": "3 Days", "chartDataType": "FLOAT"}, + {"name": "sevenDays", "displayName": "7 Days", "chartDataType": "FLOAT"}, + {"name": "fourteenDays", "displayName": "14 Days", "chartDataType": "FLOAT"}, + {"name": "thirtyDays", "displayName": "30 Days", "chartDataType": "FLOAT"}, + {"name": "sixtyDays", "displayName": "60 Days", "chartDataType": "FLOAT"}, + {"name": "totalAssetCount", "displayName": "Total Asset Counts", "chartDataType": "FLOAT"} + ] +} \ No newline at end of file diff --git a/openmetadata-service/src/main/resources/json/data/dataInsight/aggregatedUnusedAssets.json b/openmetadata-service/src/main/resources/json/data/dataInsight/aggregatedUnusedAssetsSize.json similarity index 56% rename from openmetadata-service/src/main/resources/json/data/dataInsight/aggregatedUnusedAssets.json rename to openmetadata-service/src/main/resources/json/data/dataInsight/aggregatedUnusedAssetsSize.json index 79c463585e3..29afd5b6025 100644 --- a/openmetadata-service/src/main/resources/json/data/dataInsight/aggregatedUnusedAssets.json +++ b/openmetadata-service/src/main/resources/json/data/dataInsight/aggregatedUnusedAssetsSize.json @@ -1,17 +1,18 @@ { - "name": "AggregatedUnusedAssets", - "fullyQualifiedName": "AggregatedUnusedAssets", - "displayName": "Aggregated Unused Assets", - "description": "Displays the list of unused assets over time", + "name": "AggregatedUnusedAssetsSize", + "fullyQualifiedName": "AggregatedUnusedAssetsSize", + "displayName": "Unused Assets Size", + "description": "Displays the total size of unused assets over time.", "dataIndexType": "aggregated_cost_analysis_report_data_index", "dimensions": [ {"name": "timestamp","chartDataType":"INT"} ], "metrics": [ - {"name": "threeDays", "displayName": "3 Days", "chartDataType": "INT"}, - {"name": "sevenDays", "displayName": "7 Days", "chartDataType": "INT"}, - {"name": "fourteenDays", "displayName": "14 Days", "chartDataType": "INT"}, - {"name": "thirtyDays", "displayName": "30 Days", "chartDataType": "INT"}, - {"name": "sixtyDays", "displayName": "60 Days", "chartDataType": "INT"} + {"name": "threeDays", "displayName": "3 Days", "chartDataType": "FLOAT"}, + {"name": "sevenDays", "displayName": "7 Days", "chartDataType": "FLOAT"}, + {"name": "fourteenDays", "displayName": "14 Days", "chartDataType": "FLOAT"}, + {"name": "thirtyDays", "displayName": "30 Days", "chartDataType": "FLOAT"}, + {"name": "sixtyDays", "displayName": "60 Days", "chartDataType": "FLOAT"}, + {"name": "totalAssetSize", "displayName": "Total Asset Size", "chartDataType": "FLOAT"} ] } \ No newline at end of file diff --git a/openmetadata-service/src/main/resources/json/data/dataInsight/aggregatedUsedvsUnusedAssetsCount.json b/openmetadata-service/src/main/resources/json/data/dataInsight/aggregatedUsedvsUnusedAssetsCount.json new file mode 100644 index 00000000000..3e4ecf5771d --- /dev/null +++ b/openmetadata-service/src/main/resources/json/data/dataInsight/aggregatedUsedvsUnusedAssetsCount.json @@ -0,0 +1,16 @@ +{ + "name": "AggregatedUsedvsUnusedAssetsCount", + "fullyQualifiedName": "AggregatedUsedvsUnusedAssetsCount", + "displayName": "Used vs Unused Assets Count", + "description": "Displays the count of used vs unused assets over time.", + "dataIndexType": "aggregated_cost_analysis_report_data_index", + "dimensions": [ + {"name": "timestamp","chartDataType":"INT"} + ], + "metrics": [ + {"name": "UnusedPercentage", "displayName": "Percentage of Unused Assets (Count)", "chartDataType": "FLOAT"}, + {"name": "UsedPercentage", "displayName": "Percentage of Used Assets (Count)", "chartDataType": "FLOAT"}, + {"name": "Unused", "displayName": "Unused Assets (Count)", "chartDataType": "INT"}, + {"name": "Used", "displayName": "Used Assets (Count)", "chartDataType": "INT"} + ] +} \ No newline at end of file diff --git a/openmetadata-service/src/main/resources/json/data/dataInsight/aggregatedUsedvsUnusedAssetsSize.json b/openmetadata-service/src/main/resources/json/data/dataInsight/aggregatedUsedvsUnusedAssetsSize.json new file mode 100644 index 00000000000..9b8dd5b4e68 --- /dev/null +++ b/openmetadata-service/src/main/resources/json/data/dataInsight/aggregatedUsedvsUnusedAssetsSize.json @@ -0,0 +1,16 @@ +{ + "name": "AggregatedUsedvsUnusedAssetsSize", + "fullyQualifiedName": "AggregatedUsedvsUnusedAssetsSize", + "displayName": "Used vs Unused Assets Size", + "description": "Displays the size of used vs unused assets over time.", + "dataIndexType": "aggregated_cost_analysis_report_data_index", + "dimensions": [ + {"name": "timestamp","chartDataType":"INT"} + ], + "metrics": [ + {"name": "UnusedPercentage", "displayName": "Percentage of Unused Assets (Storage Size)", "chartDataType": "FLOAT"}, + {"name": "UsedPercentage", "displayName": "Percentage of Used Assets (Storage Size)", "chartDataType": "FLOAT"}, + {"name": "Unused", "displayName": "Unused Assets (Storage Size)", "chartDataType": "INT"}, + {"name": "Used", "displayName": "Used Assets (Storage Size)", "chartDataType": "INT"} + ] +} \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/dataInsight/dataInsightChartResult.json b/openmetadata-spec/src/main/resources/json/schema/dataInsight/dataInsightChartResult.json index aa780a2c822..7ef5d2f6e5b 100644 --- a/openmetadata-spec/src/main/resources/json/schema/dataInsight/dataInsightChartResult.json +++ b/openmetadata-spec/src/main/resources/json/schema/dataInsight/dataInsightChartResult.json @@ -21,7 +21,10 @@ "PercentageOfServicesWithDescription", "PercentageOfServicesWithOwner", "UnusedAssets", - "AggregatedUnusedAssets" + "AggregatedUnusedAssetsSize", + "AggregatedUnusedAssetsCount", + "AggregatedUsedVsUnusedAssetsSize", + "AggregatedUsedVsUnusedAssetsCount" ] } }, @@ -50,7 +53,10 @@ {"$ref": "type/percentageOfServicesWithDescription.json"}, {"$ref": "type/percentageOfServicesWithOwner.json"}, {"$ref": "type/unusedAssets.json"}, - {"$ref": "type/aggregatedUnusedAsserts.json"} + {"$ref": "type/aggregatedUnusedAssetsSize.json"}, + {"$ref": "type/aggregatedUnusedAssetsCount.json"}, + {"$ref": "type/aggregatedUsedVsUnusedAssetsSize.json"}, + {"$ref": "type/aggregatedUsedVsUnusedAssetsCount.json"} ] } } diff --git a/openmetadata-spec/src/main/resources/json/schema/dataInsight/type/aggregatedUnusedAsserts.json b/openmetadata-spec/src/main/resources/json/schema/dataInsight/type/aggregatedUnusedAssetsCount.json similarity index 79% rename from openmetadata-spec/src/main/resources/json/schema/dataInsight/type/aggregatedUnusedAsserts.json rename to openmetadata-spec/src/main/resources/json/schema/dataInsight/type/aggregatedUnusedAssetsCount.json index df6cd55c573..eebdf5a4486 100644 --- a/openmetadata-spec/src/main/resources/json/schema/dataInsight/type/aggregatedUnusedAsserts.json +++ b/openmetadata-spec/src/main/resources/json/schema/dataInsight/type/aggregatedUnusedAssetsCount.json @@ -1,10 +1,10 @@ { - "$id": "https://open-metadata.org/schema/dataInsight/type/aggregatedUnusedAssets.json", + "$id": "https://open-metadata.org/schema/dataInsight/type/aggregatedUnusedAssetsCount.json", "$schema": "http://json-schema.org/draft-07/schema#", - "title": "AggregatedUnusedAssets", - "description": "AggregatedUnusedAssets data blob", + "title": "AggregatedUnusedAssetsCount", + "description": "AggregatedUnusedAssetsCount data blob", "type": "object", - "javaType": "org.openmetadata.schema.dataInsight.type.AggregatedUnusedAssets", + "javaType": "org.openmetadata.schema.dataInsight.type.AggregatedUnusedAssetsCount", "properties": { "timestamp": { "description": "timestamp", @@ -29,6 +29,10 @@ "sixtyDays": { "description": "Number of unused assets in the last 60 days", "type": "number" + }, + "total": { + "description": "Total number of unused assets", + "type": "number" } }, "additionalProperties": false diff --git a/openmetadata-spec/src/main/resources/json/schema/dataInsight/type/aggregatedUnusedAssetsSize.json b/openmetadata-spec/src/main/resources/json/schema/dataInsight/type/aggregatedUnusedAssetsSize.json new file mode 100644 index 00000000000..b34bd50e46d --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/dataInsight/type/aggregatedUnusedAssetsSize.json @@ -0,0 +1,39 @@ +{ + "$id": "https://open-metadata.org/schema/dataInsight/type/aggregatedUnusedAssetsSize.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "AggregatedUnusedAssetsSize", + "description": "AggregatedUnusedAssetsSize data blob", + "type": "object", + "javaType": "org.openmetadata.schema.dataInsight.type.AggregatedUnusedAssetsSize", + "properties": { + "timestamp": { + "description": "timestamp", + "$ref": "../../type/basic.json#/definitions/timestamp" + }, + "threeDays": { + "description": "Size of unused assets in the last 3 days", + "type": "number" + }, + "sevenDays": { + "description": "Size of unused assets in the last 7 days", + "type": "number" + }, + "fourteenDays": { + "description": "Size of unused assets in the last 14 days", + "type": "number" + }, + "thirtyDays": { + "description": "Size of unused assets in the last 30 days", + "type": "number" + }, + "sixtyDays": { + "description": "Size of unused assets in the last 60 days", + "type": "number" + }, + "total": { + "description": "Total size of unused assets", + "type": "number" + } + }, + "additionalProperties": false +} \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsCount.json b/openmetadata-spec/src/main/resources/json/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsCount.json new file mode 100644 index 00000000000..c8158b24bc2 --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsCount.json @@ -0,0 +1,31 @@ +{ + "$id": "https://open-metadata.org/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsCount.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "AggregatedUsedVsUnusedAssetsCount", + "description": "AggregatedUsedVsUnusedAssetsCount data blob", + "type": "object", + "javaType": "org.openmetadata.schema.dataInsight.type.AggregatedUsedVsUnusedAssetsCount", + "properties": { + "timestamp": { + "description": "timestamp", + "$ref": "../../type/basic.json#/definitions/timestamp" + }, + "UnusedPercentage": { + "description": "Percentage of the count of unused assets (last access >= 3 days)", + "type": "number" + }, + "UsedPercentage": { + "description": "Percentage of the count of used assets (last access < 3 days)", + "type": "number" + }, + "Unused": { + "description": "Count of unused assets (last access >= 3 days)", + "type": "number" + }, + "Used": { + "description": "Count of used assets (last access < 3 days)", + "type": "number" + } + }, + "additionalProperties": false +} \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsSize.json b/openmetadata-spec/src/main/resources/json/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsSize.json new file mode 100644 index 00000000000..9e92fa553ce --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsSize.json @@ -0,0 +1,31 @@ +{ + "$id": "https://open-metadata.org/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsSize.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "AggregatedUsedVsUnusedAssetsSize", + "description": "AggregatedUsedVsUnusedAssetsSize data blob", + "type": "object", + "javaType": "org.openmetadata.schema.dataInsight.type.AggregatedUsedVsUnusedAssetsSize", + "properties": { + "timestamp": { + "description": "timestamp", + "$ref": "../../type/basic.json#/definitions/timestamp" + }, + "UnusedPercentage": { + "description": "Percentage of the size of unused assets (last access >= 3 days)", + "type": "number" + }, + "UsedPercentage": { + "description": "Percentage of the size of used assets (last access < 3 days)", + "type": "number" + }, + "Unused": { + "description": "Size of unused assets (last access >= 3 days)", + "type": "number" + }, + "Used": { + "description": "Size of used assets (last access < 3 days)", + "type": "number" + } + }, + "additionalProperties": false +} \ No newline at end of file