yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-08-29 19:35:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

GEN-1664: DI Make Horizontal Axis Configurable (#18134)

Browse Source
This commit is contained in:
Mayur Singal 2024-10-28 11:11:15 +05:30 committed by GitHub
parent 21c78390e9
commit ef77535f17
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 226 additions and 106 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

109
openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/dataInsightAggregators/ElasticSearchDynamicChartAggregatorInterface.java
View File

@ -7,12 +7,14 @@ import es.org.elasticsearch.action.search.SearchResponse;
import es.org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
import es.org.elasticsearch.index.query.QueryBuilder;
import es.org.elasticsearch.index.query.QueryBuilders;
import es.org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
import es.org.elasticsearch.search.aggregations.Aggregation;
import es.org.elasticsearch.search.aggregations.AggregationBuilders;
import es.org.elasticsearch.search.aggregations.bucket.filter.ParsedFilter;
import es.org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder;
import es.org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
import es.org.elasticsearch.search.aggregations.bucket.histogram.ParsedDateHistogram;
import es.org.elasticsearch.search.aggregations.bucket.terms.ParsedTerms;
import es.org.elasticsearch.search.aggregations.bucket.terms.Terms;
import es.org.elasticsearch.search.aggregations.metrics.ParsedCardinality;
import es.org.elasticsearch.search.aggregations.metrics.ParsedSingleValueNumericMetricsAggregation;
import es.org.elasticsearch.search.aggregations.metrics.ParsedValueCount;
@ -54,7 +56,7 @@ public interface ElasticSearchDynamicChartAggregatorInterface {
static void getDateHistogramByFormula(
String formula,
QueryBuilder filter,
DateHistogramAggregationBuilder dateHistogramAggregationBuilder,
AbstractAggregationBuilder aggregationBuilder,
List<FormulaHolder> formulas) {
Pattern pattern = Pattern.compile(DataInsightSystemChartRepository.FORMULA_FUNC_REGEX);
Matcher matcher = pattern.matcher(formula);
@ -82,15 +84,15 @@ public interface ElasticSearchDynamicChartAggregatorInterface {
} else {
queryBuilder = QueryBuilders.queryStringQuery(matcher.group(5));
}
dateHistogramAggregationBuilder.subAggregation(
aggregationBuilder.subAggregation(
AggregationBuilders.filter("filer" + index, queryBuilder).subAggregation(subAgg));
holder.setQuery(matcher.group(5));
} else {
if (filter != null) {
dateHistogramAggregationBuilder.subAggregation(
aggregationBuilder.subAggregation(
AggregationBuilders.filter("filer" + index, filter).subAggregation(subAgg));
} else {
dateHistogramAggregationBuilder.subAggregation(subAgg);
aggregationBuilder.subAggregation(subAgg);
}
}
formulas.add(holder);
@ -111,23 +113,33 @@ public interface ElasticSearchDynamicChartAggregatorInterface {
}
boolean evaluate = true;
Double day = null;
String term = null;
for (int i = 0; i < holder.size(); i++) {
if (result.get(i).getCount() == null) {
evaluate = false;
break;
}
day = result.get(i).getDay();
term = result.get(i).getTerm();
formulaCopy =
formulaCopy.replace(holder.get(i).getFormula(), result.get(i).getCount().toString());
}
if (evaluate
&& formulaCopy.matches(DataInsightSystemChartRepository.NUMERIC_VALIDATION_REGEX)
&& day != null) {
&& (day != null || term != null)) {
Expression expression = CompiledRule.parseExpression(formulaCopy);
Double value = (Double) expression.getValue();
if (!value.isNaN() && !value.isInfinite()) {
if (day != null) {
finalList.add(
new DataInsightCustomChartResult().withCount(value).withGroup(group).withDay(day));
} else {
finalList.add(
new DataInsightCustomChartResult()
.withCount(value)
.withGroup(group)
.withTerm(term));
}
}
}
}
@ -139,7 +151,7 @@ public interface ElasticSearchDynamicChartAggregatorInterface {
String formula,
String field,
String filter,
DateHistogramAggregationBuilder dateHistogramAggregationBuilder,
AbstractAggregationBuilder aggregationBuilder,
List<FormulaHolder> formulas)
throws IOException {
if (formula != null) {
@ -150,9 +162,9 @@ public interface ElasticSearchDynamicChartAggregatorInterface {
.xContent()
.createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, filter);
QueryBuilder queryFilter = SearchSourceBuilder.fromXContent(filterParser).query();
getDateHistogramByFormula(formula, queryFilter, dateHistogramAggregationBuilder, formulas);
getDateHistogramByFormula(formula, queryFilter, aggregationBuilder, formulas);
} else {
getDateHistogramByFormula(formula, null, dateHistogramAggregationBuilder, formulas);
getDateHistogramByFormula(formula, null, aggregationBuilder, formulas);
}
return;
}
@ -165,10 +177,10 @@ public interface ElasticSearchDynamicChartAggregatorInterface {
.xContent()
.createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, filter);
QueryBuilder queryFilter = SearchSourceBuilder.fromXContent(filterParser).query();
dateHistogramAggregationBuilder.subAggregation(
aggregationBuilder.subAggregation(
AggregationBuilders.filter("filer", queryFilter).subAggregation(subAgg));
} else {
dateHistogramAggregationBuilder.subAggregation(subAgg);
aggregationBuilder.subAggregation(subAgg);
}
}
@ -204,6 +216,17 @@ public interface ElasticSearchDynamicChartAggregatorInterface {
List<Aggregation> aggregations, String group) {
List<List<DataInsightCustomChartResult>> results = new ArrayList<>();
for (Aggregation arg : aggregations) {
if (arg instanceof ParsedTerms) {
ParsedTerms parsedTerms = (ParsedTerms) arg;
for (Terms.Bucket bucket : parsedTerms.getBuckets()) {
List<DataInsightCustomChartResult> subResults = new ArrayList<>();
for (Aggregation subAggr : bucket.getAggregations().asList()) {
addByAggregationType(
subAggr, subResults, String.valueOf(bucket.getKey()), group, false);
}
results.add(subResults);
}
} else {
ParsedDateHistogram parsedDateHistogram = (ParsedDateHistogram) arg;
for (Histogram.Bucket bucket : parsedDateHistogram.getBuckets()) {
List<DataInsightCustomChartResult> subResults = new ArrayList<>();
@ -211,40 +234,57 @@ public interface ElasticSearchDynamicChartAggregatorInterface {
addByAggregationType(
subAggr,
subResults,
(double) ((ZonedDateTime) bucket.getKey()).toInstant().toEpochMilli(),
group);
String.valueOf(((ZonedDateTime) bucket.getKey()).toInstant().toEpochMilli()),
group,
true);
}
results.add(subResults);
}
}
}
return results;
}
private void addByAggregationType(
Aggregation subAggr,
List<DataInsightCustomChartResult> diChartResults,
Double day,
String group) {
String key,
String group,
boolean isTimeStamp) {
if (subAggr instanceof ParsedValueCount)
addProcessedSubResult((ParsedValueCount) subAggr, diChartResults, day, group);
addProcessedSubResult((ParsedValueCount) subAggr, diChartResults, key, group, isTimeStamp);
else if (subAggr instanceof ParsedCardinality)
addProcessedSubResult((ParsedCardinality) subAggr, diChartResults, day, group);
addProcessedSubResult((ParsedCardinality) subAggr, diChartResults, key, group, isTimeStamp);
else if (subAggr instanceof ParsedSingleValueNumericMetricsAggregation)
addProcessedSubResult(
(ParsedSingleValueNumericMetricsAggregation) subAggr, diChartResults, day, group);
(ParsedSingleValueNumericMetricsAggregation) subAggr,
diChartResults,
key,
group,
isTimeStamp);
else if (subAggr instanceof ParsedFilter)
addProcessedSubResult((ParsedFilter) subAggr, diChartResults, day, group);
addProcessedSubResult((ParsedFilter) subAggr, diChartResults, key, group, isTimeStamp);
}
private DataInsightCustomChartResult getDIChartResult(
Double value, String key, String group, boolean isTimestamp) {
if (isTimestamp)
return new DataInsightCustomChartResult()
.withCount(value)
.withDay(Double.valueOf(key))
.withGroup(group);
return new DataInsightCustomChartResult().withCount(value).withGroup(group).withTerm(key);
}
private void addProcessedSubResult(
ParsedValueCount aggregation,
List<DataInsightCustomChartResult> diChartResults,
Double day,
String group) {
String key,
String group,
boolean isTimeStamp) {
Double value = Double.valueOf((double) aggregation.getValue());
if (!Double.isInfinite(value) && !Double.isNaN(value)) {
DataInsightCustomChartResult diChartResult =
new DataInsightCustomChartResult().withCount(value).withDay(day).withGroup(group);
DataInsightCustomChartResult diChartResult = getDIChartResult(value, key, group, isTimeStamp);
diChartResults.add(diChartResult);
}
}
@ -252,12 +292,12 @@ public interface ElasticSearchDynamicChartAggregatorInterface {
private void addProcessedSubResult(
ParsedCardinality aggregation,
List<DataInsightCustomChartResult> diChartResults,
Double day,
String group) {
String key,
String group,
boolean isTimeStamp) {
Double value = Double.valueOf((double) aggregation.getValue());
if (!Double.isInfinite(value) && !Double.isNaN(value)) {
DataInsightCustomChartResult diChartResult =
new DataInsightCustomChartResult().withCount(value).withDay(day).withGroup(group);
DataInsightCustomChartResult diChartResult = getDIChartResult(value, key, group, isTimeStamp);
diChartResults.add(diChartResult);
}
}
@ -265,12 +305,12 @@ public interface ElasticSearchDynamicChartAggregatorInterface {
private void addProcessedSubResult(
ParsedSingleValueNumericMetricsAggregation aggregation,
List<DataInsightCustomChartResult> diChartResults,
Double day,
String group) {
String key,
String group,
boolean isTimeStamp) {
Double value = aggregation.value();
if (!Double.isInfinite(value) && !Double.isNaN(value)) {
DataInsightCustomChartResult diChartResult =
new DataInsightCustomChartResult().withCount(value).withDay(day).withGroup(group);
DataInsightCustomChartResult diChartResult = getDIChartResult(value, key, group, isTimeStamp);
diChartResults.add(diChartResult);
}
}
@ -278,10 +318,11 @@ public interface ElasticSearchDynamicChartAggregatorInterface {
private void addProcessedSubResult(
ParsedFilter aggregation,
List<DataInsightCustomChartResult> diChartResults,
Double day,
String group) {
String key,
String group,
boolean isTimeStamp) {
for (Aggregation agg : aggregation.getAggregations().asList()) {
addByAggregationType(agg, diChartResults, day, group);
addByAggregationType(agg, diChartResults, key, group, isTimeStamp);
}
}
}

26
openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/dataInsightAggregators/ElasticSearchLineChartAggregator.java
View File

@ -4,10 +4,10 @@ import es.org.elasticsearch.action.search.SearchRequest;
import es.org.elasticsearch.action.search.SearchResponse;
import es.org.elasticsearch.index.query.QueryBuilder;
import es.org.elasticsearch.index.query.RangeQueryBuilder;
import es.org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
import es.org.elasticsearch.search.aggregations.Aggregation;
import es.org.elasticsearch.search.aggregations.AggregationBuilders;
import es.org.elasticsearch.search.aggregations.Aggregations;
import es.org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder;
import es.org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
import es.org.elasticsearch.search.aggregations.bucket.terms.IncludeExclude;
import es.org.elasticsearch.search.aggregations.bucket.terms.ParsedTerms;
@ -34,21 +34,35 @@ public class ElasticSearchLineChartAggregator
@NotNull DataInsightCustomChart diChart, long start, long end, List<FormulaHolder> formulas)
throws IOException {
LineChart lineChart = JsonUtils.convertValue(diChart.getChartDetails(), LineChart.class);
DateHistogramAggregationBuilder dateHistogramAggregationBuilder =
AbstractAggregationBuilder aggregationBuilder;
if (lineChart.getxAxisField() != null
&& !lineChart.getxAxisField().equals(DataInsightSystemChartRepository.TIMESTAMP_FIELD)) {
aggregationBuilder =
AggregationBuilders.terms("1").field(lineChart.getxAxisField()).size(1000);
// in case of horizontal axis only process data of 24 hr prior to end time
start = end - MILLISECONDS_IN_DAY;
} else {
aggregationBuilder =
AggregationBuilders.dateHistogram("1")
.field(DataInsightSystemChartRepository.TIMESTAMP_FIELD)
.calendarInterval(DateHistogramInterval.DAY);
}
populateDateHistogram(
lineChart.getFunction(),
lineChart.getFormula(),
lineChart.getField(),
lineChart.getFilter(),
dateHistogramAggregationBuilder,
aggregationBuilder,
formulas);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
QueryBuilder queryFilter = new RangeQueryBuilder("@timestamp").gte(start).lte(end);
QueryBuilder queryFilter =
new RangeQueryBuilder(DataInsightSystemChartRepository.TIMESTAMP_FIELD).gte(start).lte(end);
if (lineChart.getGroupBy() != null) {
String[] includeArr = null;
@ -61,7 +75,7 @@ public class ElasticSearchLineChartAggregator
}
TermsAggregationBuilder termsAggregationBuilder =
AggregationBuilders.terms("0").field(lineChart.getGroupBy()).size(1000);
termsAggregationBuilder.subAggregation(dateHistogramAggregationBuilder);
termsAggregationBuilder.subAggregation(aggregationBuilder);
if (includeArr != null || excludeArr != null) {
IncludeExclude includeExclude = new IncludeExclude(includeArr, excludeArr);
termsAggregationBuilder.includeExclude(includeExclude);
@ -69,7 +83,7 @@ public class ElasticSearchLineChartAggregator
searchSourceBuilder.size(0);
searchSourceBuilder.aggregation(termsAggregationBuilder);
} else {
searchSourceBuilder.aggregation(dateHistogramAggregationBuilder);
searchSourceBuilder.aggregation(aggregationBuilder);
}
searchSourceBuilder.query(queryFilter);
es.org.elasticsearch.action.search.SearchRequest searchRequest =

110
openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/dataInsightAggregator/OpenSearchDynamicChartAggregatorInterface.java
View File

@ -24,12 +24,14 @@ import os.org.opensearch.common.xcontent.XContentParser;
import os.org.opensearch.common.xcontent.XContentType;
import os.org.opensearch.index.query.QueryBuilder;
import os.org.opensearch.index.query.QueryBuilders;
import os.org.opensearch.search.aggregations.AbstractAggregationBuilder;
import os.org.opensearch.search.aggregations.Aggregation;
import os.org.opensearch.search.aggregations.AggregationBuilders;
import os.org.opensearch.search.aggregations.bucket.filter.ParsedFilter;
import os.org.opensearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder;
import os.org.opensearch.search.aggregations.bucket.histogram.Histogram;
import os.org.opensearch.search.aggregations.bucket.histogram.ParsedDateHistogram;
import os.org.opensearch.search.aggregations.bucket.terms.ParsedTerms;
import os.org.opensearch.search.aggregations.bucket.terms.Terms;
import os.org.opensearch.search.aggregations.metrics.ParsedCardinality;
import os.org.opensearch.search.aggregations.metrics.ParsedSingleValueNumericMetricsAggregation;
import os.org.opensearch.search.aggregations.metrics.ParsedValueCount;
@ -54,7 +56,7 @@ public interface OpenSearchDynamicChartAggregatorInterface {
static void getDateHistogramByFormula(
String formula,
QueryBuilder filter,
DateHistogramAggregationBuilder dateHistogramAggregationBuilder,
AbstractAggregationBuilder aggregationBuilder,
List<FormulaHolder> formulas) {
Pattern pattern = Pattern.compile(DataInsightSystemChartRepository.FORMULA_FUNC_REGEX);
Matcher matcher = pattern.matcher(formula);
@ -82,15 +84,15 @@ public interface OpenSearchDynamicChartAggregatorInterface {
} else {
queryBuilder = QueryBuilders.queryStringQuery(matcher.group(5));
}
dateHistogramAggregationBuilder.subAggregation(
aggregationBuilder.subAggregation(
AggregationBuilders.filter("filer" + index, queryBuilder).subAggregation(subAgg));
holder.setQuery(matcher.group(5));
} else {
if (filter != null) {
dateHistogramAggregationBuilder.subAggregation(
aggregationBuilder.subAggregation(
AggregationBuilders.filter("filer" + index, filter).subAggregation(subAgg));
} else {
dateHistogramAggregationBuilder.subAggregation(subAgg);
aggregationBuilder.subAggregation(subAgg);
}
}
formulas.add(holder);
@ -111,23 +113,33 @@ public interface OpenSearchDynamicChartAggregatorInterface {
}
boolean evaluate = true;
Double day = null;
String term = null;
for (int i = 0; i < holder.size(); i++) {
if (result.get(i).getCount() == null) {
evaluate = false;
break;
}
day = result.get(i).getDay();
term = result.get(i).getTerm();
formulaCopy =
formulaCopy.replace(holder.get(i).getFormula(), result.get(i).getCount().toString());
}
if (evaluate
&& formulaCopy.matches(DataInsightSystemChartRepository.NUMERIC_VALIDATION_REGEX)
&& day != null) {
&& (day != null || term != null)) {
Expression expression = CompiledRule.parseExpression(formulaCopy);
Double value = (Double) expression.getValue();
if (!value.isNaN() && !value.isInfinite()) {
if (day != null) {
finalList.add(
new DataInsightCustomChartResult().withCount(value).withGroup(group).withDay(day));
} else {
finalList.add(
new DataInsightCustomChartResult()
.withCount(value)
.withGroup(group)
.withTerm(term));
}
}
}
}
@ -139,19 +151,20 @@ public interface OpenSearchDynamicChartAggregatorInterface {
String formula,
String field,
String filter,
DateHistogramAggregationBuilder dateHistogramAggregationBuilder,
AbstractAggregationBuilder aggregationBuilder,
List<FormulaHolder> formulas)
throws IOException {
if (formula != null) {
if (filter != null && !filter.equals("{}")) {
XContentParser filterParser =
XContentType.JSON
.xContent()
.createParser(X_CONTENT_REGISTRY, LoggingDeprecationHandler.INSTANCE, filter);
QueryBuilder queryFilter = SearchSourceBuilder.fromXContent(filterParser).query();
getDateHistogramByFormula(formula, queryFilter, dateHistogramAggregationBuilder, formulas);
getDateHistogramByFormula(formula, queryFilter, aggregationBuilder, formulas);
} else {
getDateHistogramByFormula(formula, null, dateHistogramAggregationBuilder, formulas);
getDateHistogramByFormula(formula, null, aggregationBuilder, formulas);
}
return;
}
@ -164,10 +177,10 @@ public interface OpenSearchDynamicChartAggregatorInterface {
.xContent()
.createParser(X_CONTENT_REGISTRY, LoggingDeprecationHandler.INSTANCE, filter);
QueryBuilder queryFilter = SearchSourceBuilder.fromXContent(filterParser).query();
dateHistogramAggregationBuilder.subAggregation(
aggregationBuilder.subAggregation(
AggregationBuilders.filter("filer", queryFilter).subAggregation(subAgg));
} else {
dateHistogramAggregationBuilder.subAggregation(subAgg);
aggregationBuilder.subAggregation(subAgg);
}
}
@ -203,6 +216,17 @@ public interface OpenSearchDynamicChartAggregatorInterface {
List<Aggregation> aggregations, String group) {
List<List<DataInsightCustomChartResult>> results = new ArrayList<>();
for (Aggregation arg : aggregations) {
if (arg instanceof ParsedTerms) {
ParsedTerms parsedTerms = (ParsedTerms) arg;
for (Terms.Bucket bucket : parsedTerms.getBuckets()) {
List<DataInsightCustomChartResult> subResults = new ArrayList<>();
for (Aggregation subAggr : bucket.getAggregations().asList()) {
addByAggregationType(
subAggr, subResults, String.valueOf(bucket.getKey()), group, false);
}
results.add(subResults);
}
} else {
ParsedDateHistogram parsedDateHistogram = (ParsedDateHistogram) arg;
for (Histogram.Bucket bucket : parsedDateHistogram.getBuckets()) {
List<DataInsightCustomChartResult> subResults = new ArrayList<>();
@ -210,40 +234,57 @@ public interface OpenSearchDynamicChartAggregatorInterface {
addByAggregationType(
subAggr,
subResults,
(double) ((ZonedDateTime) bucket.getKey()).toInstant().toEpochMilli(),
group);
String.valueOf(((ZonedDateTime) bucket.getKey()).toInstant().toEpochMilli()),
group,
true);
}
results.add(subResults);
}
}
}
return results;
}
private void addByAggregationType(
Aggregation subAggr,
List<DataInsightCustomChartResult> diChartResults,
Double day,
String group) {
String key,
String group,
boolean isTimeStamp) {
if (subAggr instanceof ParsedValueCount)
addProcessedSubResult((ParsedValueCount) subAggr, diChartResults, day, group);
addProcessedSubResult((ParsedValueCount) subAggr, diChartResults, key, group, isTimeStamp);
else if (subAggr instanceof ParsedCardinality)
addProcessedSubResult((ParsedCardinality) subAggr, diChartResults, day, group);
addProcessedSubResult((ParsedCardinality) subAggr, diChartResults, key, group, isTimeStamp);
else if (subAggr instanceof ParsedSingleValueNumericMetricsAggregation)
addProcessedSubResult(
(ParsedSingleValueNumericMetricsAggregation) subAggr, diChartResults, day, group);
(ParsedSingleValueNumericMetricsAggregation) subAggr,
diChartResults,
key,
group,
isTimeStamp);
else if (subAggr instanceof ParsedFilter)
addProcessedSubResult((ParsedFilter) subAggr, diChartResults, day, group);
addProcessedSubResult((ParsedFilter) subAggr, diChartResults, key, group, isTimeStamp);
}
private DataInsightCustomChartResult getDIChartResult(
Double value, String key, String group, boolean isTimestamp) {
if (isTimestamp)
return new DataInsightCustomChartResult()
.withCount(value)
.withDay(Double.valueOf(key))
.withGroup(group);
return new DataInsightCustomChartResult().withCount(value).withGroup(group).withTerm(key);
}
private void addProcessedSubResult(
ParsedValueCount aggregation,
List<DataInsightCustomChartResult> diChartResults,
Double day,
String group) {
String key,
String group,
boolean isTimeStamp) {
Double value = Double.valueOf((double) aggregation.getValue());
if (!Double.isInfinite(value) && !Double.isNaN(value)) {
DataInsightCustomChartResult diChartResult =
new DataInsightCustomChartResult().withCount(value).withDay(day).withGroup(group);
DataInsightCustomChartResult diChartResult = getDIChartResult(value, key, group, isTimeStamp);
diChartResults.add(diChartResult);
}
}
@ -251,12 +292,12 @@ public interface OpenSearchDynamicChartAggregatorInterface {
private void addProcessedSubResult(
ParsedCardinality aggregation,
List<DataInsightCustomChartResult> diChartResults,
Double day,
String group) {
String key,
String group,
boolean isTimeStamp) {
Double value = Double.valueOf((double) aggregation.getValue());
if (!Double.isInfinite(value) && !Double.isNaN(value)) {
DataInsightCustomChartResult diChartResult =
new DataInsightCustomChartResult().withCount(value).withDay(day).withGroup(group);
DataInsightCustomChartResult diChartResult = getDIChartResult(value, key, group, isTimeStamp);
diChartResults.add(diChartResult);
}
}
@ -264,12 +305,12 @@ public interface OpenSearchDynamicChartAggregatorInterface {
private void addProcessedSubResult(
ParsedSingleValueNumericMetricsAggregation aggregation,
List<DataInsightCustomChartResult> diChartResults,
Double day,
String group) {
String key,
String group,
boolean isTimeStamp) {
Double value = aggregation.value();
if (!Double.isInfinite(value) && !Double.isNaN(value)) {
DataInsightCustomChartResult diChartResult =
new DataInsightCustomChartResult().withCount(value).withDay(day).withGroup(group);
DataInsightCustomChartResult diChartResult = getDIChartResult(value, key, group, isTimeStamp);
diChartResults.add(diChartResult);
}
}
@ -277,10 +318,11 @@ public interface OpenSearchDynamicChartAggregatorInterface {
private void addProcessedSubResult(
ParsedFilter aggregation,
List<DataInsightCustomChartResult> diChartResults,
Double day,
String group) {
String key,
String group,
boolean isTimeStamp) {
for (Aggregation agg : aggregation.getAggregations().asList()) {
addByAggregationType(agg, diChartResults, day, group);
addByAggregationType(agg, diChartResults, key, group, isTimeStamp);
}
}
}

26
openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/dataInsightAggregator/OpenSearchLineChartAggregator.java
View File

@ -17,10 +17,10 @@ import os.org.opensearch.action.search.SearchRequest;
import os.org.opensearch.action.search.SearchResponse;
import os.org.opensearch.index.query.QueryBuilder;
import os.org.opensearch.index.query.RangeQueryBuilder;
import os.org.opensearch.search.aggregations.AbstractAggregationBuilder;
import os.org.opensearch.search.aggregations.Aggregation;
import os.org.opensearch.search.aggregations.AggregationBuilders;
import os.org.opensearch.search.aggregations.Aggregations;
import os.org.opensearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder;
import os.org.opensearch.search.aggregations.bucket.histogram.DateHistogramInterval;
import os.org.opensearch.search.aggregations.bucket.terms.IncludeExclude;
import os.org.opensearch.search.aggregations.bucket.terms.ParsedTerms;
@ -33,21 +33,35 @@ public class OpenSearchLineChartAggregator implements OpenSearchDynamicChartAggr
@NotNull DataInsightCustomChart diChart, long start, long end, List<FormulaHolder> formulas)
throws IOException {
LineChart lineChart = JsonUtils.convertValue(diChart.getChartDetails(), LineChart.class);
DateHistogramAggregationBuilder dateHistogramAggregationBuilder =
AbstractAggregationBuilder aggregationBuilder;
if (lineChart.getxAxisField() != null
&& !lineChart.getxAxisField().equals(DataInsightSystemChartRepository.TIMESTAMP_FIELD)) {
aggregationBuilder =
AggregationBuilders.terms("1").field(lineChart.getxAxisField()).size(1000);
// in case of horizontal axis only process data of 24 hr prior to end time
start = end - MILLISECONDS_IN_DAY;
} else {
aggregationBuilder =
AggregationBuilders.dateHistogram("1")
.field(DataInsightSystemChartRepository.TIMESTAMP_FIELD)
.calendarInterval(DateHistogramInterval.DAY);
}
populateDateHistogram(
lineChart.getFunction(),
lineChart.getFormula(),
lineChart.getField(),
lineChart.getFilter(),
dateHistogramAggregationBuilder,
aggregationBuilder,
formulas);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
QueryBuilder queryFilter = new RangeQueryBuilder("@timestamp").gte(start).lte(end);
QueryBuilder queryFilter =
new RangeQueryBuilder(DataInsightSystemChartRepository.TIMESTAMP_FIELD).gte(start).lte(end);
if (lineChart.getGroupBy() != null) {
String[] includeArr = null;
@ -60,7 +74,7 @@ public class OpenSearchLineChartAggregator implements OpenSearchDynamicChartAggr
}
TermsAggregationBuilder termsAggregationBuilder =
AggregationBuilders.terms("0").field(lineChart.getGroupBy()).size(1000);
termsAggregationBuilder.subAggregation(dateHistogramAggregationBuilder);
termsAggregationBuilder.subAggregation(aggregationBuilder);
if (includeArr != null || excludeArr != null) {
IncludeExclude includeExclude = new IncludeExclude(includeArr, excludeArr);
termsAggregationBuilder.includeExclude(includeExclude);
@ -68,7 +82,7 @@ public class OpenSearchLineChartAggregator implements OpenSearchDynamicChartAggr
searchSourceBuilder.size(0);
searchSourceBuilder.aggregation(termsAggregationBuilder);
} else {
searchSourceBuilder.aggregation(dateHistogramAggregationBuilder);
searchSourceBuilder.aggregation(aggregationBuilder);
}
searchSourceBuilder.query(queryFilter);
os.org.opensearch.action.search.SearchRequest searchRequest =

4
openmetadata-spec/src/main/resources/json/schema/dataInsight/custom/dataInsightCustomChartResult.json
View File

@ -17,6 +17,10 @@
"group": {
"description": "Group of Result",
"type": "string"
},
"term": {
"description": "Term of Result, used in case of horizontal axis not timestamp",
"type": "string"
}
},
"additionalProperties": false

5
openmetadata-spec/src/main/resources/json/schema/dataInsight/custom/lineChart.json
View File

@ -61,6 +61,11 @@
},
"kpiDetails": {
"$ref": "dataInsightCustomChart.json#/definitions/kpiDetails"
},
"xAxisField":{
"description": "X-axis field for the data insight chart.",
"type": "string",
"default": "@timestamp"
}
},
"additionalProperties": false