diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/dataInsightAggregators/ElasticSearchDynamicChartAggregatorInterface.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/dataInsightAggregators/ElasticSearchDynamicChartAggregatorInterface.java index d30eb685df3..634622b6f37 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/dataInsightAggregators/ElasticSearchDynamicChartAggregatorInterface.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/dataInsightAggregators/ElasticSearchDynamicChartAggregatorInterface.java @@ -7,12 +7,14 @@ import es.org.elasticsearch.action.search.SearchResponse; import es.org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import es.org.elasticsearch.index.query.QueryBuilder; import es.org.elasticsearch.index.query.QueryBuilders; +import es.org.elasticsearch.search.aggregations.AbstractAggregationBuilder; import es.org.elasticsearch.search.aggregations.Aggregation; import es.org.elasticsearch.search.aggregations.AggregationBuilders; import es.org.elasticsearch.search.aggregations.bucket.filter.ParsedFilter; -import es.org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder; import es.org.elasticsearch.search.aggregations.bucket.histogram.Histogram; import es.org.elasticsearch.search.aggregations.bucket.histogram.ParsedDateHistogram; +import es.org.elasticsearch.search.aggregations.bucket.terms.ParsedTerms; +import es.org.elasticsearch.search.aggregations.bucket.terms.Terms; import es.org.elasticsearch.search.aggregations.metrics.ParsedCardinality; import es.org.elasticsearch.search.aggregations.metrics.ParsedSingleValueNumericMetricsAggregation; import es.org.elasticsearch.search.aggregations.metrics.ParsedValueCount; @@ -54,7 +56,7 @@ public interface ElasticSearchDynamicChartAggregatorInterface { static void getDateHistogramByFormula( String formula, QueryBuilder filter, - DateHistogramAggregationBuilder dateHistogramAggregationBuilder, + AbstractAggregationBuilder aggregationBuilder, List formulas) { Pattern pattern = Pattern.compile(DataInsightSystemChartRepository.FORMULA_FUNC_REGEX); Matcher matcher = pattern.matcher(formula); @@ -82,15 +84,15 @@ public interface ElasticSearchDynamicChartAggregatorInterface { } else { queryBuilder = QueryBuilders.queryStringQuery(matcher.group(5)); } - dateHistogramAggregationBuilder.subAggregation( + aggregationBuilder.subAggregation( AggregationBuilders.filter("filer" + index, queryBuilder).subAggregation(subAgg)); holder.setQuery(matcher.group(5)); } else { if (filter != null) { - dateHistogramAggregationBuilder.subAggregation( + aggregationBuilder.subAggregation( AggregationBuilders.filter("filer" + index, filter).subAggregation(subAgg)); } else { - dateHistogramAggregationBuilder.subAggregation(subAgg); + aggregationBuilder.subAggregation(subAgg); } } formulas.add(holder); @@ -111,23 +113,33 @@ public interface ElasticSearchDynamicChartAggregatorInterface { } boolean evaluate = true; Double day = null; + String term = null; for (int i = 0; i < holder.size(); i++) { if (result.get(i).getCount() == null) { evaluate = false; break; } day = result.get(i).getDay(); + term = result.get(i).getTerm(); formulaCopy = formulaCopy.replace(holder.get(i).getFormula(), result.get(i).getCount().toString()); } if (evaluate && formulaCopy.matches(DataInsightSystemChartRepository.NUMERIC_VALIDATION_REGEX) - && day != null) { + && (day != null || term != null)) { Expression expression = CompiledRule.parseExpression(formulaCopy); Double value = (Double) expression.getValue(); if (!value.isNaN() && !value.isInfinite()) { - finalList.add( - new DataInsightCustomChartResult().withCount(value).withGroup(group).withDay(day)); + if (day != null) { + finalList.add( + new DataInsightCustomChartResult().withCount(value).withGroup(group).withDay(day)); + } else { + finalList.add( + new DataInsightCustomChartResult() + .withCount(value) + .withGroup(group) + .withTerm(term)); + } } } } @@ -139,7 +151,7 @@ public interface ElasticSearchDynamicChartAggregatorInterface { String formula, String field, String filter, - DateHistogramAggregationBuilder dateHistogramAggregationBuilder, + AbstractAggregationBuilder aggregationBuilder, List formulas) throws IOException { if (formula != null) { @@ -150,9 +162,9 @@ public interface ElasticSearchDynamicChartAggregatorInterface { .xContent() .createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, filter); QueryBuilder queryFilter = SearchSourceBuilder.fromXContent(filterParser).query(); - getDateHistogramByFormula(formula, queryFilter, dateHistogramAggregationBuilder, formulas); + getDateHistogramByFormula(formula, queryFilter, aggregationBuilder, formulas); } else { - getDateHistogramByFormula(formula, null, dateHistogramAggregationBuilder, formulas); + getDateHistogramByFormula(formula, null, aggregationBuilder, formulas); } return; } @@ -165,10 +177,10 @@ public interface ElasticSearchDynamicChartAggregatorInterface { .xContent() .createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, filter); QueryBuilder queryFilter = SearchSourceBuilder.fromXContent(filterParser).query(); - dateHistogramAggregationBuilder.subAggregation( + aggregationBuilder.subAggregation( AggregationBuilders.filter("filer", queryFilter).subAggregation(subAgg)); } else { - dateHistogramAggregationBuilder.subAggregation(subAgg); + aggregationBuilder.subAggregation(subAgg); } } @@ -204,17 +216,30 @@ public interface ElasticSearchDynamicChartAggregatorInterface { List aggregations, String group) { List> results = new ArrayList<>(); for (Aggregation arg : aggregations) { - ParsedDateHistogram parsedDateHistogram = (ParsedDateHistogram) arg; - for (Histogram.Bucket bucket : parsedDateHistogram.getBuckets()) { - List subResults = new ArrayList<>(); - for (Aggregation subAggr : bucket.getAggregations().asList()) { - addByAggregationType( - subAggr, - subResults, - (double) ((ZonedDateTime) bucket.getKey()).toInstant().toEpochMilli(), - group); + if (arg instanceof ParsedTerms) { + ParsedTerms parsedTerms = (ParsedTerms) arg; + for (Terms.Bucket bucket : parsedTerms.getBuckets()) { + List subResults = new ArrayList<>(); + for (Aggregation subAggr : bucket.getAggregations().asList()) { + addByAggregationType( + subAggr, subResults, String.valueOf(bucket.getKey()), group, false); + } + results.add(subResults); + } + } else { + ParsedDateHistogram parsedDateHistogram = (ParsedDateHistogram) arg; + for (Histogram.Bucket bucket : parsedDateHistogram.getBuckets()) { + List subResults = new ArrayList<>(); + for (Aggregation subAggr : bucket.getAggregations().asList()) { + addByAggregationType( + subAggr, + subResults, + String.valueOf(((ZonedDateTime) bucket.getKey()).toInstant().toEpochMilli()), + group, + true); + } + results.add(subResults); } - results.add(subResults); } } return results; @@ -223,28 +248,43 @@ public interface ElasticSearchDynamicChartAggregatorInterface { private void addByAggregationType( Aggregation subAggr, List diChartResults, - Double day, - String group) { + String key, + String group, + boolean isTimeStamp) { if (subAggr instanceof ParsedValueCount) - addProcessedSubResult((ParsedValueCount) subAggr, diChartResults, day, group); + addProcessedSubResult((ParsedValueCount) subAggr, diChartResults, key, group, isTimeStamp); else if (subAggr instanceof ParsedCardinality) - addProcessedSubResult((ParsedCardinality) subAggr, diChartResults, day, group); + addProcessedSubResult((ParsedCardinality) subAggr, diChartResults, key, group, isTimeStamp); else if (subAggr instanceof ParsedSingleValueNumericMetricsAggregation) addProcessedSubResult( - (ParsedSingleValueNumericMetricsAggregation) subAggr, diChartResults, day, group); + (ParsedSingleValueNumericMetricsAggregation) subAggr, + diChartResults, + key, + group, + isTimeStamp); else if (subAggr instanceof ParsedFilter) - addProcessedSubResult((ParsedFilter) subAggr, diChartResults, day, group); + addProcessedSubResult((ParsedFilter) subAggr, diChartResults, key, group, isTimeStamp); + } + + private DataInsightCustomChartResult getDIChartResult( + Double value, String key, String group, boolean isTimestamp) { + if (isTimestamp) + return new DataInsightCustomChartResult() + .withCount(value) + .withDay(Double.valueOf(key)) + .withGroup(group); + return new DataInsightCustomChartResult().withCount(value).withGroup(group).withTerm(key); } private void addProcessedSubResult( ParsedValueCount aggregation, List diChartResults, - Double day, - String group) { + String key, + String group, + boolean isTimeStamp) { Double value = Double.valueOf((double) aggregation.getValue()); if (!Double.isInfinite(value) && !Double.isNaN(value)) { - DataInsightCustomChartResult diChartResult = - new DataInsightCustomChartResult().withCount(value).withDay(day).withGroup(group); + DataInsightCustomChartResult diChartResult = getDIChartResult(value, key, group, isTimeStamp); diChartResults.add(diChartResult); } } @@ -252,12 +292,12 @@ public interface ElasticSearchDynamicChartAggregatorInterface { private void addProcessedSubResult( ParsedCardinality aggregation, List diChartResults, - Double day, - String group) { + String key, + String group, + boolean isTimeStamp) { Double value = Double.valueOf((double) aggregation.getValue()); if (!Double.isInfinite(value) && !Double.isNaN(value)) { - DataInsightCustomChartResult diChartResult = - new DataInsightCustomChartResult().withCount(value).withDay(day).withGroup(group); + DataInsightCustomChartResult diChartResult = getDIChartResult(value, key, group, isTimeStamp); diChartResults.add(diChartResult); } } @@ -265,12 +305,12 @@ public interface ElasticSearchDynamicChartAggregatorInterface { private void addProcessedSubResult( ParsedSingleValueNumericMetricsAggregation aggregation, List diChartResults, - Double day, - String group) { + String key, + String group, + boolean isTimeStamp) { Double value = aggregation.value(); if (!Double.isInfinite(value) && !Double.isNaN(value)) { - DataInsightCustomChartResult diChartResult = - new DataInsightCustomChartResult().withCount(value).withDay(day).withGroup(group); + DataInsightCustomChartResult diChartResult = getDIChartResult(value, key, group, isTimeStamp); diChartResults.add(diChartResult); } } @@ -278,10 +318,11 @@ public interface ElasticSearchDynamicChartAggregatorInterface { private void addProcessedSubResult( ParsedFilter aggregation, List diChartResults, - Double day, - String group) { + String key, + String group, + boolean isTimeStamp) { for (Aggregation agg : aggregation.getAggregations().asList()) { - addByAggregationType(agg, diChartResults, day, group); + addByAggregationType(agg, diChartResults, key, group, isTimeStamp); } } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/dataInsightAggregators/ElasticSearchLineChartAggregator.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/dataInsightAggregators/ElasticSearchLineChartAggregator.java index 811e2f36058..fcf136544b8 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/dataInsightAggregators/ElasticSearchLineChartAggregator.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/dataInsightAggregators/ElasticSearchLineChartAggregator.java @@ -4,10 +4,10 @@ import es.org.elasticsearch.action.search.SearchRequest; import es.org.elasticsearch.action.search.SearchResponse; import es.org.elasticsearch.index.query.QueryBuilder; import es.org.elasticsearch.index.query.RangeQueryBuilder; +import es.org.elasticsearch.search.aggregations.AbstractAggregationBuilder; import es.org.elasticsearch.search.aggregations.Aggregation; import es.org.elasticsearch.search.aggregations.AggregationBuilders; import es.org.elasticsearch.search.aggregations.Aggregations; -import es.org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder; import es.org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; import es.org.elasticsearch.search.aggregations.bucket.terms.IncludeExclude; import es.org.elasticsearch.search.aggregations.bucket.terms.ParsedTerms; @@ -34,21 +34,35 @@ public class ElasticSearchLineChartAggregator @NotNull DataInsightCustomChart diChart, long start, long end, List formulas) throws IOException { LineChart lineChart = JsonUtils.convertValue(diChart.getChartDetails(), LineChart.class); - DateHistogramAggregationBuilder dateHistogramAggregationBuilder = - AggregationBuilders.dateHistogram("1") - .field(DataInsightSystemChartRepository.TIMESTAMP_FIELD) - .calendarInterval(DateHistogramInterval.DAY); + AbstractAggregationBuilder aggregationBuilder; + + if (lineChart.getxAxisField() != null + && !lineChart.getxAxisField().equals(DataInsightSystemChartRepository.TIMESTAMP_FIELD)) { + aggregationBuilder = + AggregationBuilders.terms("1").field(lineChart.getxAxisField()).size(1000); + + // in case of horizontal axis only process data of 24 hr prior to end time + start = end - MILLISECONDS_IN_DAY; + + } else { + aggregationBuilder = + AggregationBuilders.dateHistogram("1") + .field(DataInsightSystemChartRepository.TIMESTAMP_FIELD) + .calendarInterval(DateHistogramInterval.DAY); + } + populateDateHistogram( lineChart.getFunction(), lineChart.getFormula(), lineChart.getField(), lineChart.getFilter(), - dateHistogramAggregationBuilder, + aggregationBuilder, formulas); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - QueryBuilder queryFilter = new RangeQueryBuilder("@timestamp").gte(start).lte(end); + QueryBuilder queryFilter = + new RangeQueryBuilder(DataInsightSystemChartRepository.TIMESTAMP_FIELD).gte(start).lte(end); if (lineChart.getGroupBy() != null) { String[] includeArr = null; @@ -61,7 +75,7 @@ public class ElasticSearchLineChartAggregator } TermsAggregationBuilder termsAggregationBuilder = AggregationBuilders.terms("0").field(lineChart.getGroupBy()).size(1000); - termsAggregationBuilder.subAggregation(dateHistogramAggregationBuilder); + termsAggregationBuilder.subAggregation(aggregationBuilder); if (includeArr != null || excludeArr != null) { IncludeExclude includeExclude = new IncludeExclude(includeArr, excludeArr); termsAggregationBuilder.includeExclude(includeExclude); @@ -69,7 +83,7 @@ public class ElasticSearchLineChartAggregator searchSourceBuilder.size(0); searchSourceBuilder.aggregation(termsAggregationBuilder); } else { - searchSourceBuilder.aggregation(dateHistogramAggregationBuilder); + searchSourceBuilder.aggregation(aggregationBuilder); } searchSourceBuilder.query(queryFilter); es.org.elasticsearch.action.search.SearchRequest searchRequest = diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/dataInsightAggregator/OpenSearchDynamicChartAggregatorInterface.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/dataInsightAggregator/OpenSearchDynamicChartAggregatorInterface.java index 5dc625ad430..adf1f9d09b4 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/dataInsightAggregator/OpenSearchDynamicChartAggregatorInterface.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/dataInsightAggregator/OpenSearchDynamicChartAggregatorInterface.java @@ -24,12 +24,14 @@ import os.org.opensearch.common.xcontent.XContentParser; import os.org.opensearch.common.xcontent.XContentType; import os.org.opensearch.index.query.QueryBuilder; import os.org.opensearch.index.query.QueryBuilders; +import os.org.opensearch.search.aggregations.AbstractAggregationBuilder; import os.org.opensearch.search.aggregations.Aggregation; import os.org.opensearch.search.aggregations.AggregationBuilders; import os.org.opensearch.search.aggregations.bucket.filter.ParsedFilter; -import os.org.opensearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder; import os.org.opensearch.search.aggregations.bucket.histogram.Histogram; import os.org.opensearch.search.aggregations.bucket.histogram.ParsedDateHistogram; +import os.org.opensearch.search.aggregations.bucket.terms.ParsedTerms; +import os.org.opensearch.search.aggregations.bucket.terms.Terms; import os.org.opensearch.search.aggregations.metrics.ParsedCardinality; import os.org.opensearch.search.aggregations.metrics.ParsedSingleValueNumericMetricsAggregation; import os.org.opensearch.search.aggregations.metrics.ParsedValueCount; @@ -54,7 +56,7 @@ public interface OpenSearchDynamicChartAggregatorInterface { static void getDateHistogramByFormula( String formula, QueryBuilder filter, - DateHistogramAggregationBuilder dateHistogramAggregationBuilder, + AbstractAggregationBuilder aggregationBuilder, List formulas) { Pattern pattern = Pattern.compile(DataInsightSystemChartRepository.FORMULA_FUNC_REGEX); Matcher matcher = pattern.matcher(formula); @@ -82,15 +84,15 @@ public interface OpenSearchDynamicChartAggregatorInterface { } else { queryBuilder = QueryBuilders.queryStringQuery(matcher.group(5)); } - dateHistogramAggregationBuilder.subAggregation( + aggregationBuilder.subAggregation( AggregationBuilders.filter("filer" + index, queryBuilder).subAggregation(subAgg)); holder.setQuery(matcher.group(5)); } else { if (filter != null) { - dateHistogramAggregationBuilder.subAggregation( + aggregationBuilder.subAggregation( AggregationBuilders.filter("filer" + index, filter).subAggregation(subAgg)); } else { - dateHistogramAggregationBuilder.subAggregation(subAgg); + aggregationBuilder.subAggregation(subAgg); } } formulas.add(holder); @@ -111,23 +113,33 @@ public interface OpenSearchDynamicChartAggregatorInterface { } boolean evaluate = true; Double day = null; + String term = null; for (int i = 0; i < holder.size(); i++) { if (result.get(i).getCount() == null) { evaluate = false; break; } day = result.get(i).getDay(); + term = result.get(i).getTerm(); formulaCopy = formulaCopy.replace(holder.get(i).getFormula(), result.get(i).getCount().toString()); } if (evaluate && formulaCopy.matches(DataInsightSystemChartRepository.NUMERIC_VALIDATION_REGEX) - && day != null) { + && (day != null || term != null)) { Expression expression = CompiledRule.parseExpression(formulaCopy); Double value = (Double) expression.getValue(); if (!value.isNaN() && !value.isInfinite()) { - finalList.add( - new DataInsightCustomChartResult().withCount(value).withGroup(group).withDay(day)); + if (day != null) { + finalList.add( + new DataInsightCustomChartResult().withCount(value).withGroup(group).withDay(day)); + } else { + finalList.add( + new DataInsightCustomChartResult() + .withCount(value) + .withGroup(group) + .withTerm(term)); + } } } } @@ -139,19 +151,20 @@ public interface OpenSearchDynamicChartAggregatorInterface { String formula, String field, String filter, - DateHistogramAggregationBuilder dateHistogramAggregationBuilder, + AbstractAggregationBuilder aggregationBuilder, List formulas) throws IOException { if (formula != null) { + if (filter != null && !filter.equals("{}")) { XContentParser filterParser = XContentType.JSON .xContent() .createParser(X_CONTENT_REGISTRY, LoggingDeprecationHandler.INSTANCE, filter); QueryBuilder queryFilter = SearchSourceBuilder.fromXContent(filterParser).query(); - getDateHistogramByFormula(formula, queryFilter, dateHistogramAggregationBuilder, formulas); + getDateHistogramByFormula(formula, queryFilter, aggregationBuilder, formulas); } else { - getDateHistogramByFormula(formula, null, dateHistogramAggregationBuilder, formulas); + getDateHistogramByFormula(formula, null, aggregationBuilder, formulas); } return; } @@ -164,10 +177,10 @@ public interface OpenSearchDynamicChartAggregatorInterface { .xContent() .createParser(X_CONTENT_REGISTRY, LoggingDeprecationHandler.INSTANCE, filter); QueryBuilder queryFilter = SearchSourceBuilder.fromXContent(filterParser).query(); - dateHistogramAggregationBuilder.subAggregation( + aggregationBuilder.subAggregation( AggregationBuilders.filter("filer", queryFilter).subAggregation(subAgg)); } else { - dateHistogramAggregationBuilder.subAggregation(subAgg); + aggregationBuilder.subAggregation(subAgg); } } @@ -203,17 +216,30 @@ public interface OpenSearchDynamicChartAggregatorInterface { List aggregations, String group) { List> results = new ArrayList<>(); for (Aggregation arg : aggregations) { - ParsedDateHistogram parsedDateHistogram = (ParsedDateHistogram) arg; - for (Histogram.Bucket bucket : parsedDateHistogram.getBuckets()) { - List subResults = new ArrayList<>(); - for (Aggregation subAggr : bucket.getAggregations().asList()) { - addByAggregationType( - subAggr, - subResults, - (double) ((ZonedDateTime) bucket.getKey()).toInstant().toEpochMilli(), - group); + if (arg instanceof ParsedTerms) { + ParsedTerms parsedTerms = (ParsedTerms) arg; + for (Terms.Bucket bucket : parsedTerms.getBuckets()) { + List subResults = new ArrayList<>(); + for (Aggregation subAggr : bucket.getAggregations().asList()) { + addByAggregationType( + subAggr, subResults, String.valueOf(bucket.getKey()), group, false); + } + results.add(subResults); + } + } else { + ParsedDateHistogram parsedDateHistogram = (ParsedDateHistogram) arg; + for (Histogram.Bucket bucket : parsedDateHistogram.getBuckets()) { + List subResults = new ArrayList<>(); + for (Aggregation subAggr : bucket.getAggregations().asList()) { + addByAggregationType( + subAggr, + subResults, + String.valueOf(((ZonedDateTime) bucket.getKey()).toInstant().toEpochMilli()), + group, + true); + } + results.add(subResults); } - results.add(subResults); } } return results; @@ -222,28 +248,43 @@ public interface OpenSearchDynamicChartAggregatorInterface { private void addByAggregationType( Aggregation subAggr, List diChartResults, - Double day, - String group) { + String key, + String group, + boolean isTimeStamp) { if (subAggr instanceof ParsedValueCount) - addProcessedSubResult((ParsedValueCount) subAggr, diChartResults, day, group); + addProcessedSubResult((ParsedValueCount) subAggr, diChartResults, key, group, isTimeStamp); else if (subAggr instanceof ParsedCardinality) - addProcessedSubResult((ParsedCardinality) subAggr, diChartResults, day, group); + addProcessedSubResult((ParsedCardinality) subAggr, diChartResults, key, group, isTimeStamp); else if (subAggr instanceof ParsedSingleValueNumericMetricsAggregation) addProcessedSubResult( - (ParsedSingleValueNumericMetricsAggregation) subAggr, diChartResults, day, group); + (ParsedSingleValueNumericMetricsAggregation) subAggr, + diChartResults, + key, + group, + isTimeStamp); else if (subAggr instanceof ParsedFilter) - addProcessedSubResult((ParsedFilter) subAggr, diChartResults, day, group); + addProcessedSubResult((ParsedFilter) subAggr, diChartResults, key, group, isTimeStamp); + } + + private DataInsightCustomChartResult getDIChartResult( + Double value, String key, String group, boolean isTimestamp) { + if (isTimestamp) + return new DataInsightCustomChartResult() + .withCount(value) + .withDay(Double.valueOf(key)) + .withGroup(group); + return new DataInsightCustomChartResult().withCount(value).withGroup(group).withTerm(key); } private void addProcessedSubResult( ParsedValueCount aggregation, List diChartResults, - Double day, - String group) { + String key, + String group, + boolean isTimeStamp) { Double value = Double.valueOf((double) aggregation.getValue()); if (!Double.isInfinite(value) && !Double.isNaN(value)) { - DataInsightCustomChartResult diChartResult = - new DataInsightCustomChartResult().withCount(value).withDay(day).withGroup(group); + DataInsightCustomChartResult diChartResult = getDIChartResult(value, key, group, isTimeStamp); diChartResults.add(diChartResult); } } @@ -251,12 +292,12 @@ public interface OpenSearchDynamicChartAggregatorInterface { private void addProcessedSubResult( ParsedCardinality aggregation, List diChartResults, - Double day, - String group) { + String key, + String group, + boolean isTimeStamp) { Double value = Double.valueOf((double) aggregation.getValue()); if (!Double.isInfinite(value) && !Double.isNaN(value)) { - DataInsightCustomChartResult diChartResult = - new DataInsightCustomChartResult().withCount(value).withDay(day).withGroup(group); + DataInsightCustomChartResult diChartResult = getDIChartResult(value, key, group, isTimeStamp); diChartResults.add(diChartResult); } } @@ -264,12 +305,12 @@ public interface OpenSearchDynamicChartAggregatorInterface { private void addProcessedSubResult( ParsedSingleValueNumericMetricsAggregation aggregation, List diChartResults, - Double day, - String group) { + String key, + String group, + boolean isTimeStamp) { Double value = aggregation.value(); if (!Double.isInfinite(value) && !Double.isNaN(value)) { - DataInsightCustomChartResult diChartResult = - new DataInsightCustomChartResult().withCount(value).withDay(day).withGroup(group); + DataInsightCustomChartResult diChartResult = getDIChartResult(value, key, group, isTimeStamp); diChartResults.add(diChartResult); } } @@ -277,10 +318,11 @@ public interface OpenSearchDynamicChartAggregatorInterface { private void addProcessedSubResult( ParsedFilter aggregation, List diChartResults, - Double day, - String group) { + String key, + String group, + boolean isTimeStamp) { for (Aggregation agg : aggregation.getAggregations().asList()) { - addByAggregationType(agg, diChartResults, day, group); + addByAggregationType(agg, diChartResults, key, group, isTimeStamp); } } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/dataInsightAggregator/OpenSearchLineChartAggregator.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/dataInsightAggregator/OpenSearchLineChartAggregator.java index fc21c3a8828..2522778538e 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/dataInsightAggregator/OpenSearchLineChartAggregator.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/dataInsightAggregator/OpenSearchLineChartAggregator.java @@ -17,10 +17,10 @@ import os.org.opensearch.action.search.SearchRequest; import os.org.opensearch.action.search.SearchResponse; import os.org.opensearch.index.query.QueryBuilder; import os.org.opensearch.index.query.RangeQueryBuilder; +import os.org.opensearch.search.aggregations.AbstractAggregationBuilder; import os.org.opensearch.search.aggregations.Aggregation; import os.org.opensearch.search.aggregations.AggregationBuilders; import os.org.opensearch.search.aggregations.Aggregations; -import os.org.opensearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder; import os.org.opensearch.search.aggregations.bucket.histogram.DateHistogramInterval; import os.org.opensearch.search.aggregations.bucket.terms.IncludeExclude; import os.org.opensearch.search.aggregations.bucket.terms.ParsedTerms; @@ -33,21 +33,35 @@ public class OpenSearchLineChartAggregator implements OpenSearchDynamicChartAggr @NotNull DataInsightCustomChart diChart, long start, long end, List formulas) throws IOException { LineChart lineChart = JsonUtils.convertValue(diChart.getChartDetails(), LineChart.class); - DateHistogramAggregationBuilder dateHistogramAggregationBuilder = - AggregationBuilders.dateHistogram("1") - .field(DataInsightSystemChartRepository.TIMESTAMP_FIELD) - .calendarInterval(DateHistogramInterval.DAY); + AbstractAggregationBuilder aggregationBuilder; + + if (lineChart.getxAxisField() != null + && !lineChart.getxAxisField().equals(DataInsightSystemChartRepository.TIMESTAMP_FIELD)) { + aggregationBuilder = + AggregationBuilders.terms("1").field(lineChart.getxAxisField()).size(1000); + + // in case of horizontal axis only process data of 24 hr prior to end time + start = end - MILLISECONDS_IN_DAY; + + } else { + aggregationBuilder = + AggregationBuilders.dateHistogram("1") + .field(DataInsightSystemChartRepository.TIMESTAMP_FIELD) + .calendarInterval(DateHistogramInterval.DAY); + } + populateDateHistogram( lineChart.getFunction(), lineChart.getFormula(), lineChart.getField(), lineChart.getFilter(), - dateHistogramAggregationBuilder, + aggregationBuilder, formulas); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - QueryBuilder queryFilter = new RangeQueryBuilder("@timestamp").gte(start).lte(end); + QueryBuilder queryFilter = + new RangeQueryBuilder(DataInsightSystemChartRepository.TIMESTAMP_FIELD).gte(start).lte(end); if (lineChart.getGroupBy() != null) { String[] includeArr = null; @@ -60,7 +74,7 @@ public class OpenSearchLineChartAggregator implements OpenSearchDynamicChartAggr } TermsAggregationBuilder termsAggregationBuilder = AggregationBuilders.terms("0").field(lineChart.getGroupBy()).size(1000); - termsAggregationBuilder.subAggregation(dateHistogramAggregationBuilder); + termsAggregationBuilder.subAggregation(aggregationBuilder); if (includeArr != null || excludeArr != null) { IncludeExclude includeExclude = new IncludeExclude(includeArr, excludeArr); termsAggregationBuilder.includeExclude(includeExclude); @@ -68,7 +82,7 @@ public class OpenSearchLineChartAggregator implements OpenSearchDynamicChartAggr searchSourceBuilder.size(0); searchSourceBuilder.aggregation(termsAggregationBuilder); } else { - searchSourceBuilder.aggregation(dateHistogramAggregationBuilder); + searchSourceBuilder.aggregation(aggregationBuilder); } searchSourceBuilder.query(queryFilter); os.org.opensearch.action.search.SearchRequest searchRequest = diff --git a/openmetadata-spec/src/main/resources/json/schema/dataInsight/custom/dataInsightCustomChartResult.json b/openmetadata-spec/src/main/resources/json/schema/dataInsight/custom/dataInsightCustomChartResult.json index 5fe808b3b0e..79c7665f696 100644 --- a/openmetadata-spec/src/main/resources/json/schema/dataInsight/custom/dataInsightCustomChartResult.json +++ b/openmetadata-spec/src/main/resources/json/schema/dataInsight/custom/dataInsightCustomChartResult.json @@ -17,6 +17,10 @@ "group": { "description": "Group of Result", "type": "string" + }, + "term": { + "description": "Term of Result, used in case of horizontal axis not timestamp", + "type": "string" } }, "additionalProperties": false diff --git a/openmetadata-spec/src/main/resources/json/schema/dataInsight/custom/lineChart.json b/openmetadata-spec/src/main/resources/json/schema/dataInsight/custom/lineChart.json index 92ed4d2c2cd..ec80e0a0e0f 100644 --- a/openmetadata-spec/src/main/resources/json/schema/dataInsight/custom/lineChart.json +++ b/openmetadata-spec/src/main/resources/json/schema/dataInsight/custom/lineChart.json @@ -61,6 +61,11 @@ }, "kpiDetails": { "$ref": "dataInsightCustomChart.json#/definitions/kpiDetails" + }, + "xAxisField":{ + "description": "X-axis field for the data insight chart.", + "type": "string", + "default": "@timestamp" } }, "additionalProperties": false