yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-08-30 03:46:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

MINOR: DI Simplify Formulas (#17382)

Browse Source 
* MINOR: DI Similify Formulas

* checkstyle
This commit is contained in:
Mayur Singal 2024-08-13 17:42:46 +05:30 committed by GitHub
parent feab12422b
commit ac2264e9c7
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 45 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataInsightSystemChartRepository.java
View File

@ -21,7 +21,7 @@ public class DataInsightSystemChartRepository extends EntityRepository<DataInsig
public static final String DI_SEARCH_INDEX = "di-data-assets-*";
public static final String FORMULA_FUNC_REGEX =
"\\b(count|sum|min|max|avg)+\\(k='([^']*)',?\\s*(q='([^']*)')?\\)?";
"\\b(count|sum|min|max|avg)+\\((k='([^']*)')?,?\\s*(q='([^']*)')?\\)?";
public static final String NUMERIC_VALIDATION_REGEX = "[\\d\\.+-\\/\\*\\(\\)\s]+";

17
openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/dataInsightAggregators/ElasticSearchDynamicChartAggregatorInterface.java
View File

@ -70,23 +70,28 @@ public interface ElasticSearchDynamicChartAggregatorInterface {
FormulaHolder holder = new FormulaHolder();
holder.setFormula(matcher.group());
holder.setFunction(Function.valueOf(matcher.group(1).toUpperCase()));
holder.setField(matcher.group(2));
String field;
if (matcher.group(3) != null) {
field = matcher.group(3);
} else {
field = "id.keyword";
}
ValuesSourceAggregationBuilder subAgg =
getSubAggregationsByFunction(
Function.valueOf(matcher.group(1).toUpperCase()), matcher.group(2), index);
if (matcher.group(4) != null) {
Function.valueOf(matcher.group(1).toUpperCase()), field, index);
if (matcher.group(5) != null) {
QueryBuilder queryBuilder;
if (filter != null) {
queryBuilder =
QueryBuilders.boolQuery()
.must(QueryBuilders.queryStringQuery(matcher.group(4)))
.must(QueryBuilders.queryStringQuery(matcher.group(5)))
.must(filter);
} else {
queryBuilder = QueryBuilders.queryStringQuery(matcher.group(4));
queryBuilder = QueryBuilders.queryStringQuery(matcher.group(5));
}
dateHistogramAggregationBuilder.subAggregation(
AggregationBuilders.filter("filer" + index, queryBuilder).subAggregation(subAgg));
holder.setQuery(matcher.group(4));
holder.setQuery(matcher.group(5));
} else {
if (filter != null) {
dateHistogramAggregationBuilder.subAggregation(

17
openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/dataInsightAggregator/OpenSearchDynamicChartAggregatorInterface.java
View File

@ -70,23 +70,28 @@ public interface OpenSearchDynamicChartAggregatorInterface {
FormulaHolder holder = new FormulaHolder();
holder.setFormula(matcher.group());
holder.setFunction(Function.valueOf(matcher.group(1).toUpperCase()));
holder.setField(matcher.group(2));
String field;
if (matcher.group(3) != null) {
field = matcher.group(3);
} else {
field = "id.keyword";
}
ValuesSourceAggregationBuilder subAgg =
getSubAggregationsByFunction(
Function.valueOf(matcher.group(1).toUpperCase()), matcher.group(2), index);
if (matcher.group(4) != null) {
Function.valueOf(matcher.group(1).toUpperCase()), field, index);
if (matcher.group(5) != null) {
QueryBuilder queryBuilder;
if (filter != null) {
queryBuilder =
QueryBuilders.boolQuery()
.must(QueryBuilders.queryStringQuery(matcher.group(4)))
.must(QueryBuilders.queryStringQuery(matcher.group(5)))
.must(filter);
} else {
queryBuilder = QueryBuilders.queryStringQuery(matcher.group(4));
queryBuilder = QueryBuilders.queryStringQuery(matcher.group(5));
}
dateHistogramAggregationBuilder.subAggregation(
AggregationBuilders.filter("filer" + index, queryBuilder).subAggregation(subAgg));
holder.setQuery(matcher.group(4));
holder.setQuery(matcher.group(5));
} else {
if (filter != null) {
dateHistogramAggregationBuilder.subAggregation(

11
openmetadata-service/src/test/java/org/openmetadata/service/search/elasticsearch/ElasticSearchDynamicChartAggregatorTest.java
View File

@ -130,6 +130,11 @@ public class ElasticSearchDynamicChartAggregatorTest extends OpenMetadataApplica
summaryCard.put("formula", "count(k='id.keyword')");
assertTrue(compareRequest(cardString, summaryCard));
Map<String, Object> summaryCard1 = new LinkedHashMap<>();
summaryCard1.put("type", "SummaryCard");
summaryCard1.put("formula", "count()");
assertTrue(compareRequest(cardString, summaryCard1));
String lineString =
"{\"query\":{\"range\":{\"@timestamp\":{\"from\":\"2024-07-16T03:54:31Z\",\"to\":\"2024-07-23T01:34:31Z\",\"include_lower\":true,\"include_upper\":true,\"boost\":1.0}}},\"aggregations\":{\"1\":{\"date_histogram\":{\"field\":\"@timestamp\",\"calendar_interval\":\"1d\",\"offset\":0,\"order\":{\"_key\":\"asc\"},\"keyed\":false,\"min_doc_count\":0},\"aggregations\":{\"id.keyword0\":{\"value_count\":{\"field\":\"id.keyword\"}}}}}}";
Map<String, Object> lineChart = new LinkedHashMap<>();
@ -169,6 +174,12 @@ public class ElasticSearchDynamicChartAggregatorTest extends OpenMetadataApplica
lineChart4.put("groupBy", "entityType.keyword");
assertTrue(compareRequest(lineString4, lineChart4));
Map<String, Object> lineChart41 = new LinkedHashMap<>();
lineChart41.put("type", "LineChart");
lineChart41.put("formula", "count(q='hasDescription: 1')+count(q='owner.name.keyword: *')");
lineChart41.put("groupBy", "entityType.keyword");
assertTrue(compareRequest(lineString4, lineChart41));
String lineString5 =
"{\"size\":0,\"query\":{\"range\":{\"@timestamp\":{\"from\":\"2024-07-16T03:54:31Z\",\"to\":\"2024-07-23T01:34:31Z\",\"include_lower\":true,\"include_upper\":true,\"boost\":1.0}}},\"aggregations\":{\"0\":{\"terms\":{\"field\":\"entityType.keyword\",\"size\":20,\"min_doc_count\":1,\"shard_min_doc_count\":0,\"show_term_doc_count_error\":false,\"order\":[{\"_count\":\"desc\"},{\"_key\":\"asc\"}]},\"aggregations\":{\"1\":{\"date_histogram\":{\"field\":\"@timestamp\",\"calendar_interval\":\"1d\",\"offset\":0,\"order\":{\"_key\":\"asc\"},\"keyed\":false,\"min_doc_count\":0},\"aggregations\":{\"filer0\":{\"filter\":{\"bool\":{\"must\":[{\"query_string\":{\"query\":\"hasDescription: 1\",\"fields\":[],\"type\":\"best_fields\",\"default_operator\":\"or\",\"max_determinized_states\":10000,\"enable_position_increments\":true,\"fuzziness\":\"AUTO\",\"fuzzy_prefix_length\":0,\"fuzzy_max_expansions\":50,\"phrase_slop\":0,\"escape\":false,\"auto_generate_synonyms_phrase_query\":true,\"fuzzy_transpositions\":true,\"boost\":1.0}},{\"bool\":{\"must\":[{\"bool\":{\"should\":[{\"term\":{\"owners.displayName.keyword\":{\"value\":\"admin\",\"boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}}],\"adjust_pure_negative\":true,\"boost\":1.0}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"aggregations\":{\"id.keyword0\":{\"value_count\":{\"field\":\"id.keyword\"}}}},\"filer1\":{\"filter\":{\"bool\":{\"must\":[{\"query_string\":{\"query\":\"owner.name.keyword: *\",\"fields\":[],\"type\":\"best_fields\",\"default_operator\":\"or\",\"max_determinized_states\":10000,\"enable_position_increments\":true,\"fuzziness\":\"AUTO\",\"fuzzy_prefix_length\":0,\"fuzzy_max_expansions\":50,\"phrase_slop\":0,\"escape\":false,\"auto_generate_synonyms_phrase_query\":true,\"fuzzy_transpositions\":true,\"boost\":1.0}},{\"bool\":{\"must\":[{\"bool\":{\"should\":[{\"term\":{\"owners.displayName.keyword\":{\"value\":\"admin\",\"boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}}],\"adjust_pure_negative\":true,\"boost\":1.0}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"aggregations\":{\"id.keyword1\":{\"value_count\":{\"field\":\"id.keyword\"}}}}}}}}}}";
Map<String, Object> lineChart5 = new LinkedHashMap<>();

11
openmetadata-service/src/test/java/org/openmetadata/service/search/opensearch/OpenSearchDynamicChartAggregatorTest.java
View File

@ -129,6 +129,11 @@ public class OpenSearchDynamicChartAggregatorTest extends OpenMetadataApplicatio
summaryCard.put("formula", "count(k='id.keyword')");
assertTrue(compareRequest(cardString, summaryCard));
Map<String, Object> summaryCard1 = new LinkedHashMap<>();
summaryCard1.put("type", "SummaryCard");
summaryCard1.put("formula", "count()");
assertTrue(compareRequest(cardString, summaryCard1));
String lineString =
"{\"query\":{\"range\":{\"@timestamp\":{\"from\":\"2024-07-16T03:54:31Z\",\"to\":\"2024-07-23T01:34:31Z\",\"include_lower\":true,\"include_upper\":true,\"boost\":1.0}}},\"aggregations\":{\"1\":{\"date_histogram\":{\"field\":\"@timestamp\",\"calendar_interval\":\"1d\",\"offset\":0,\"order\":{\"_key\":\"asc\"},\"keyed\":false,\"min_doc_count\":0},\"aggregations\":{\"id.keyword0\":{\"value_count\":{\"field\":\"id.keyword\"}}}}}}";
Map<String, Object> lineChart = new LinkedHashMap<>();
@ -168,6 +173,12 @@ public class OpenSearchDynamicChartAggregatorTest extends OpenMetadataApplicatio
lineChart4.put("groupBy", "entityType.keyword");
assertTrue(compareRequest(lineString4, lineChart4));
Map<String, Object> lineChart41 = new LinkedHashMap<>();
lineChart41.put("type", "LineChart");
lineChart41.put("formula", "count(q='hasDescription: 1')+count(q='owner.name.keyword: *')");
lineChart41.put("groupBy", "entityType.keyword");
assertTrue(compareRequest(lineString4, lineChart41));
String lineString5 =
"{\"size\":0,\"query\":{\"range\":{\"@timestamp\":{\"from\":\"2024-07-16T03:54:31Z\",\"to\":\"2024-07-23T01:34:31Z\",\"include_lower\":true,\"include_upper\":true,\"boost\":1.0}}},\"aggregations\":{\"0\":{\"terms\":{\"field\":\"entityType.keyword\",\"size\":20,\"min_doc_count\":1,\"shard_min_doc_count\":0,\"show_term_doc_count_error\":false,\"order\":[{\"_count\":\"desc\"},{\"_key\":\"asc\"}]},\"aggregations\":{\"1\":{\"date_histogram\":{\"field\":\"@timestamp\",\"calendar_interval\":\"1d\",\"offset\":0,\"order\":{\"_key\":\"asc\"},\"keyed\":false,\"min_doc_count\":0},\"aggregations\":{\"filer0\":{\"filter\":{\"bool\":{\"must\":[{\"query_string\":{\"query\":\"hasDescription: 1\",\"fields\":[],\"type\":\"best_fields\",\"default_operator\":\"or\",\"max_determinized_states\":10000,\"enable_position_increments\":true,\"fuzziness\":\"AUTO\",\"fuzzy_prefix_length\":0,\"fuzzy_max_expansions\":50,\"phrase_slop\":0,\"escape\":false,\"auto_generate_synonyms_phrase_query\":true,\"fuzzy_transpositions\":true,\"boost\":1.0}},{\"bool\":{\"must\":[{\"bool\":{\"should\":[{\"term\":{\"owners.displayName.keyword\":{\"value\":\"admin\",\"boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}}],\"adjust_pure_negative\":true,\"boost\":1.0}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"aggregations\":{\"id.keyword0\":{\"value_count\":{\"field\":\"id.keyword\"}}}},\"filer1\":{\"filter\":{\"bool\":{\"must\":[{\"query_string\":{\"query\":\"owner.name.keyword: *\",\"fields\":[],\"type\":\"best_fields\",\"default_operator\":\"or\",\"max_determinized_states\":10000,\"enable_position_increments\":true,\"fuzziness\":\"AUTO\",\"fuzzy_prefix_length\":0,\"fuzzy_max_expansions\":50,\"phrase_slop\":0,\"escape\":false,\"auto_generate_synonyms_phrase_query\":true,\"fuzzy_transpositions\":true,\"boost\":1.0}},{\"bool\":{\"must\":[{\"bool\":{\"should\":[{\"term\":{\"owners.displayName.keyword\":{\"value\":\"admin\",\"boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}}],\"adjust_pure_negative\":true,\"boost\":1.0}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"aggregations\":{\"id.keyword1\":{\"value_count\":{\"field\":\"id.keyword\"}}}}}}}}}}";
Map<String, Object> lineChart5 = new LinkedHashMap<>();