diff --git a/web/app/controllers/api/v1/AdvSearch.java b/web/app/controllers/api/v1/AdvSearch.java index a0fabf06a8..da4c6fd767 100644 --- a/web/app/controllers/api/v1/AdvSearch.java +++ b/web/app/controllers/api/v1/AdvSearch.java @@ -16,8 +16,10 @@ package controllers.api.v1; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import dao.AdvSearchDAO; +import dao.SearchDAO; import org.apache.commons.lang3.StringUtils; import play.Logger; +import play.Play; import play.libs.Json; import play.mvc.Controller; import play.mvc.Result; @@ -137,17 +139,46 @@ public class AdvSearch extends Controller } } result.put("status", "ok"); + String searchEngine = Play.application().configuration().getString(SearchDAO.WHEREHOWS_SEARCH_ENGINE__KEY); + if (searchOpt != null && searchOpt.has("category")) { String category = searchOpt.get("category").asText(); if(category.equalsIgnoreCase("flow")) { - result.set("result", Json.toJson(AdvSearchDAO.searchFlows(searchOpt, page, size))); + if(StringUtils.isNotBlank(searchEngine) && searchEngine.equalsIgnoreCase("elasticsearch")) + { + result.set("result", Json.toJson(AdvSearchDAO.elasticSearchFlowJobs(searchOpt, page, size))); + } + else + { + result.set("result", Json.toJson(AdvSearchDAO.searchFlows(searchOpt, page, size))); + } return ok(result); } + else if(category.equalsIgnoreCase("metric")) + { + if(StringUtils.isNotBlank(searchEngine) && searchEngine.equalsIgnoreCase("elasticsearch")) + { + result.set("result", Json.toJson(AdvSearchDAO.elasticSearchMetric(searchOpt, page, size))); + } + else + { + result.set("result", Json.toJson(AdvSearchDAO.searchMetrics(searchOpt, page, size))); + } + return ok(result); + } + } - result.set("result", Json.toJson(AdvSearchDAO.search(searchOpt, page, size))); + if(StringUtils.isNotBlank(searchEngine) && searchEngine.equalsIgnoreCase("elasticsearch")) + { + result.set("result", Json.toJson(AdvSearchDAO.elasticSearch(searchOpt, page, size))); + } + else + { + result.set("result", Json.toJson(AdvSearchDAO.search(searchOpt, page, size))); + } return ok(result); } diff --git a/web/app/controllers/api/v1/Search.java b/web/app/controllers/api/v1/Search.java index af110ebb9e..07fdb6c664 100644 --- a/web/app/controllers/api/v1/Search.java +++ b/web/app/controllers/api/v1/Search.java @@ -17,6 +17,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import dao.SearchDAO; import models.DatasetColumn; +import play.Play; import play.api.libs.json.JsValue; import play.libs.Json; import play.mvc.Controller; @@ -92,51 +93,71 @@ public class Search extends Controller { category = "datasets"; } - if (StringUtils.isBlank(source)) + if (StringUtils.isBlank(source) || source.equalsIgnoreCase("all") || source.equalsIgnoreCase("default")) { - source = "all"; - } - else if (source.equalsIgnoreCase("default")) - { - source = "all"; - isDefault = true; + source = null; } + + String searchEngine = Play.application().configuration().getString(SearchDAO.WHEREHOWS_SEARCH_ENGINE__KEY); + if (category.toLowerCase().equalsIgnoreCase("metric")) { - result.set("result", SearchDAO.getPagedMetricByKeyword(category, keyword, page, size)); + if(StringUtils.isNotBlank(searchEngine) && searchEngine.equalsIgnoreCase("elasticsearch")) + { + result.set("result", SearchDAO.elasticSearchMetricByKeyword(category, keyword, page, size)); + + } + else + { + result.set("result", SearchDAO.getPagedMetricByKeyword(category, keyword, page, size)); + } } else if (category.toLowerCase().equalsIgnoreCase("flows")) { - result.set("result", SearchDAO.getPagedFlowByKeyword(category, keyword, page, size)); + if(StringUtils.isNotBlank(searchEngine) && searchEngine.equalsIgnoreCase("elasticsearch")) + { + result.set("result", SearchDAO.elasticSearchFlowByKeyword(category, keyword, page, size)); + + } + else + { + result.set("result", SearchDAO.getPagedFlowByKeyword(category, keyword, page, size)); + } } else if (category.toLowerCase().equalsIgnoreCase("jobs")) { - result.set("result", SearchDAO.getPagedJobByKeyword(category, keyword, page, size)); + if(StringUtils.isNotBlank(searchEngine) && searchEngine.equalsIgnoreCase("elasticsearch")) + { + result.set("result", SearchDAO.elasticSearchFlowByKeyword(category, keyword, page, size)); + + } + else + { + result.set("result", SearchDAO.getPagedJobByKeyword(category, keyword, page, size)); + } } else if (category.toLowerCase().equalsIgnoreCase("comments")) { - result.set("result", SearchDAO.getPagedCommentsByKeyword(category, keyword, page, size)); + if(StringUtils.isNotBlank(searchEngine) && searchEngine.equalsIgnoreCase("elasticsearch")) + { + result.set("result", SearchDAO.elasticSearchDatasetByKeyword(category, keyword, null, page, size)); + } + else + { + result.set("result", SearchDAO.getPagedCommentsByKeyword(category, keyword, page, size)); + } + } else { - ObjectNode node = SearchDAO.getPagedDatasetByKeyword(category, keyword, source, page, size); - if (isDefault && node != null && node.has("count")) + if(StringUtils.isNotBlank(searchEngine) && searchEngine.equalsIgnoreCase("elasticsearch")) { - Long count = node.get("count").asLong(); - if (count != null && count == 0) - { - node = SearchDAO.getPagedFlowByKeyword("flows", keyword, page, size); - if (node!= null && node.has("count")) - { - Long flowCount = node.get("count").asLong(); - if (flowCount != null && flowCount == 0) - { - node = SearchDAO.getPagedJobByKeyword("jobs", keyword, page, size); - } - } - } + result.set("result", SearchDAO.elasticSearchDatasetByKeyword(category, keyword, source, page, size)); + } + else + { + result.set("result", SearchDAO.getPagedDatasetByKeyword(category, keyword, source, page, size)); } - result.set("result", node); } return ok(result); diff --git a/web/app/dao/AdvSearchDAO.java b/web/app/dao/AdvSearchDAO.java index baac238b4d..653b014a94 100644 --- a/web/app/dao/AdvSearchDAO.java +++ b/web/app/dao/AdvSearchDAO.java @@ -17,6 +17,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import models.Dataset; import models.FlowJob; +import models.Metric; import org.apache.commons.lang3.StringUtils; import org.springframework.dao.EmptyResultDataAccessException; import org.springframework.jdbc.core.JdbcTemplate; @@ -26,12 +27,12 @@ import org.springframework.transaction.TransactionStatus; import org.springframework.transaction.support.TransactionCallback; import org.springframework.transaction.support.TransactionTemplate; import play.Logger; +import play.Play; +import play.libs.F; import play.libs.Json; -import java.util.Arrays; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; +import play.libs.WS; + +import java.util.*; public class AdvSearchDAO extends AbstractMySQLOpenSourceDAO { @@ -100,6 +101,14 @@ public class AdvSearchDAO extends AbstractMySQLOpenSourceDAO "FROM flow_job j JOIN flow f on j.app_id = f.app_id AND j.flow_id = f.flow_id " + "JOIN cfg_application a on j.app_id = a.app_id "; + public final static String ADV_SEARCH_METRIC = "SELECT SQL_CALC_FOUND_ROWS metric_id, " + + "metric_name, metric_description, dashboard_name, metric_group, metric_category, " + + "metric_sub_category, metric_level, metric_source_type, metric_source, " + + "metric_source_dataset_id, metric_ref_id_type, metric_ref_id, metric_type, metric_grain, " + + "metric_display_factor, metric_display_factor_sym, metric_good_direction, " + + "metric_formula, dimensions, owners, tags, urn, metric_url, wiki_url, scm_url, 0 as watch_id " + + "FROM dict_business_metric "; + public static List getDatasetSources() @@ -193,6 +202,246 @@ public class AdvSearchDAO extends AbstractMySQLOpenSourceDAO return getJdbcTemplate().queryForList(GET_JOB_NAMES, String.class); } + public static ObjectNode elasticSearch(JsonNode searchOpt, int page, int size) + { + ObjectNode resultNode = Json.newObject(); + Long count = 0L; + List pagedDatasets = new ArrayList(); + ObjectNode queryNode = Json.newObject(); + queryNode.put("from", (page-1)*size); + queryNode.put("size", size); + + JsonNode searchNode = utils.Search.generateDatasetAdvSearchQueryString(searchOpt); + + if (searchNode != null && searchNode.isContainerNode()) + { + queryNode.put("query", searchNode); + } + F.Promise < WS.Response> responsePromise = WS.url( + Play.application().configuration().getString( + SearchDAO.ELASTICSEARCH_DATASET_URL_KEY)).post(queryNode); + JsonNode responseNode = responsePromise.get().asJson(); + + resultNode.put("page", page); + resultNode.put("category", "Datasets"); + resultNode.put("itemsPerPage", size); + + if (responseNode != null && responseNode.isContainerNode() && responseNode.has("hits")) { + JsonNode hitsNode = responseNode.get("hits"); + if (hitsNode != null) { + if (hitsNode.has("total")) { + count = hitsNode.get("total").asLong(); + } + if (hitsNode.has("hits")) { + JsonNode dataNode = hitsNode.get("hits"); + if (dataNode != null && dataNode.isArray()) { + Iterator arrayIterator = dataNode.elements(); + if (arrayIterator != null) { + while (arrayIterator.hasNext()) { + JsonNode node = arrayIterator.next(); + if (node.isContainerNode() && node.has("_id")) { + Dataset dataset = new Dataset(); + dataset.id = node.get("_id").asLong(); + if (node.has("_source")) { + JsonNode sourceNode = node.get("_source"); + if (sourceNode != null) { + if (sourceNode.has("name")) { + dataset.name = sourceNode.get("name").asText(); + } + if (sourceNode.has("source")) { + dataset.source = sourceNode.get("source").asText(); + } + if (sourceNode.has("urn")) { + dataset.urn = sourceNode.get("urn").asText(); + } + if (sourceNode.has("schema")) { + dataset.schema = sourceNode.get("schema").asText(); + } + } + } + pagedDatasets.add(dataset); + } + } + } + } + } + } + } + resultNode.put("count", count); + resultNode.put("totalPages", (int)Math.ceil(count/((double)size))); + resultNode.set("data", Json.toJson(pagedDatasets)); + return resultNode; + } + + public static ObjectNode elasticSearchMetric(JsonNode searchOpt, int page, int size) + { + ObjectNode resultNode = Json.newObject(); + Long count = 0L; + List pagedMetrics = new ArrayList(); + ObjectNode queryNode = Json.newObject(); + queryNode.put("from", (page-1)*size); + queryNode.put("size", size); + + JsonNode searchNode = utils.Search.generateMetricAdvSearchQueryString(searchOpt); + + if (searchNode != null && searchNode.isContainerNode()) + { + queryNode.put("query", searchNode); + } + + F.Promise < WS.Response> responsePromise = WS.url(Play.application().configuration().getString( + SearchDAO.ELASTICSEARCH_METRIC_URL_KEY)).post(queryNode); + JsonNode responseNode = responsePromise.get().asJson(); + + resultNode.put("page", page); + resultNode.put("category", "Metrics"); + resultNode.put("isMetrics", true); + resultNode.put("itemsPerPage", size); + + if (responseNode != null && responseNode.isContainerNode() && responseNode.has("hits")) { + JsonNode hitsNode = responseNode.get("hits"); + if (hitsNode != null) { + if (hitsNode.has("total")) { + count = hitsNode.get("total").asLong(); + } + if (hitsNode.has("hits")) { + JsonNode dataNode = hitsNode.get("hits"); + if (dataNode != null && dataNode.isArray()) { + Iterator arrayIterator = dataNode.elements(); + if (arrayIterator != null) { + while (arrayIterator.hasNext()) { + JsonNode node = arrayIterator.next(); + if (node.isContainerNode() && node.has("_id")) { + Metric metric = new Metric(); + metric.id = node.get("_id").asInt(); + if (node.has("_source")) { + JsonNode sourceNode = node.get("_source"); + if (sourceNode != null) { + if (sourceNode.has("metric_name")) { + metric.name = sourceNode.get("metric_name").asText(); + } + if (sourceNode.has("metric_description")) { + metric.description = sourceNode.get("metric_description").asText(); + } + if (sourceNode.has("dashboard_name")) { + metric.dashboardName = sourceNode.get("dashboard_name").asText(); + } + if (sourceNode.has("metric_group")) { + metric.group = sourceNode.get("metric_group").asText(); + } + if (sourceNode.has("metric_category")) { + metric.category = sourceNode.get("metric_category").asText(); + } + if (sourceNode.has("urn")) { + metric.urn = sourceNode.get("urn").asText(); + } + if (sourceNode.has("metric_source")) { + metric.source = sourceNode.get("metric_source").asText(); + if (StringUtils.isBlank(metric.source)) + { + metric.source = null; + } + } + metric.schema = sourceNode.toString(); + } + } + pagedMetrics.add(metric); + } + } + } + } + } + } + } + resultNode.put("count", count); + resultNode.put("totalPages", (int)Math.ceil(count/((double)size))); + resultNode.set("data", Json.toJson(pagedMetrics)); + return resultNode; + } + + public static ObjectNode elasticSearchFlowJobs(JsonNode searchOpt, int page, int size) + { + ObjectNode resultNode = Json.newObject(); + Long count = 0L; + List pagedFlows = new ArrayList(); + ObjectNode queryNode = Json.newObject(); + queryNode.put("from", (page-1)*size); + queryNode.put("size", size); + + JsonNode searchNode = utils.Search.generateFlowJobAdvSearchQueryString(searchOpt); + + if (searchNode != null && searchNode.isContainerNode()) + { + queryNode.put("query", searchNode); + } + + F.Promise < WS.Response> responsePromise = WS.url(Play.application().configuration().getString( + SearchDAO.ELASTICSEARCH_FLOW_URL_KEY)).post(queryNode); + JsonNode responseNode = responsePromise.get().asJson(); + + resultNode.put("page", page); + resultNode.put("category", "Flows"); + resultNode.put("isFlowJob", true); + resultNode.put("itemsPerPage", size); + + if (responseNode != null && responseNode.isContainerNode() && responseNode.has("hits")) { + JsonNode hitsNode = responseNode.get("hits"); + if (hitsNode != null) { + if (hitsNode.has("total")) { + count = hitsNode.get("total").asLong(); + } + if (hitsNode.has("hits")) { + JsonNode dataNode = hitsNode.get("hits"); + if (dataNode != null && dataNode.isArray()) { + Iterator arrayIterator = dataNode.elements(); + if (arrayIterator != null) { + while (arrayIterator.hasNext()) { + JsonNode node = arrayIterator.next(); + if (node.isContainerNode() && node.has("_id")) { + FlowJob flowJob = new FlowJob(); + if (node.has("_source")) { + JsonNode sourceNode = node.get("_source"); + if (sourceNode != null) { + if (sourceNode.has("app_code")) { + flowJob.appCode = sourceNode.get("app_code").asText(); + } + if (sourceNode.has("app_id")) { + flowJob.appId = sourceNode.get("app_id").asInt(); + } + if (sourceNode.has("flow_id")) { + flowJob.flowId = sourceNode.get("flow_id").asLong(); + } + if (sourceNode.has("flow_name")) { + flowJob.flowName = sourceNode.get("flow_name").asText(); + flowJob.displayName = flowJob.flowName; + } + if (sourceNode.has("flow_path")) { + flowJob.flowPath = sourceNode.get("flow_path").asText(); + } + if (sourceNode.has("flow_group")) { + flowJob.flowGroup = sourceNode.get("flow_group").asText(); + } + flowJob.link = "#/flows/" + flowJob.appCode + "/" + + flowJob.flowGroup + "/" + Long.toString(flowJob.flowId) + "/page/1"; + flowJob.path = flowJob.appCode + "/" + flowJob.flowPath; + + flowJob.schema = sourceNode.toString(); + } + } + pagedFlows.add(flowJob); + } + } + } + } + } + } + } + resultNode.put("count", count); + resultNode.put("totalPages", (int)Math.ceil(count/((double)size))); + resultNode.set("data", Json.toJson(pagedFlows)); + return resultNode; + } + public static ObjectNode search(JsonNode searchOpt, int page, int size) { ObjectNode resultNode = Json.newObject(); @@ -1291,4 +1540,495 @@ public class AdvSearchDAO extends AbstractMySQLOpenSourceDAO return resultNode; } + public static ObjectNode searchMetrics(JsonNode searchOpt, int page, int size) + { + ObjectNode resultNode = Json.newObject(); + int count = 0; + List dashboardInList = new ArrayList(); + List dashboardNotInList = new ArrayList(); + List groupInList = new ArrayList(); + List groupNotInList = new ArrayList(); + List categoryInList = new ArrayList(); + List categoryNotInList = new ArrayList(); + List metricInList = new ArrayList(); + List metricNotInList = new ArrayList(); + + if (searchOpt != null && (searchOpt.isContainerNode())) + { + if (searchOpt.has("dashboard")) { + JsonNode dashboardNode = searchOpt.get("dashboard"); + if (dashboardNode != null && dashboardNode.isContainerNode()) + { + if (dashboardNode.has("in")) + { + JsonNode dashboardInNode = dashboardNode.get("in"); + if (dashboardInNode != null) + { + String dashboardInStr = dashboardInNode.asText(); + if (StringUtils.isNotBlank(dashboardInStr)) + { + String[] dashboardInArray = dashboardInStr.split(","); + if (dashboardInArray != null) + { + for(String value : dashboardInArray) + { + if (StringUtils.isNotBlank(value)) + { + dashboardInList.add(value.trim()); + } + } + } + } + } + } + if (dashboardNode.has("not")) + { + JsonNode dashboardNotInNode = dashboardNode.get("not"); + if (dashboardNotInNode != null) + { + String dashboardNotInStr = dashboardNotInNode.asText(); + if (StringUtils.isNotBlank(dashboardNotInStr)) + { + String[] dashboardNotInArray = dashboardNotInStr.split(","); + if (dashboardNotInArray != null) + { + for(String value : dashboardNotInArray) + { + if (StringUtils.isNotBlank(value)) + { + dashboardNotInList.add(value.trim()); + } + } + } + } + } + } + } + } + + if (searchOpt.has("group")) { + JsonNode groupNode = searchOpt.get("group"); + if (groupNode != null && groupNode.isContainerNode()) + { + if (groupNode.has("in")) + { + JsonNode groupInNode = groupNode.get("in"); + if (groupInNode != null) + { + String groupInStr = groupInNode.asText(); + if (StringUtils.isNotBlank(groupInStr)) + { + String[] groupInArray = groupInStr.split(","); + if (groupInArray != null) + { + for(String value : groupInArray) + { + if (StringUtils.isNotBlank(value)) + { + groupInList.add(value.trim()); + } + } + } + } + } + } + if (groupNode.has("not")) + { + JsonNode groupNotInNode = groupNode.get("not"); + if (groupNotInNode != null) + { + String groupNotInStr = groupNotInNode.asText(); + if (StringUtils.isNotBlank(groupNotInStr)) + { + String[] groupNotInArray = groupNotInStr.split(","); + if (groupNotInArray != null) + { + for(String value : groupNotInArray) + { + if (StringUtils.isNotBlank(value)) + { + groupNotInList.add(value.trim()); + } + } + } + } + } + } + } + } + + if (searchOpt.has("cat")) { + JsonNode categoryNode = searchOpt.get("cat"); + if (categoryNode != null && categoryNode.isContainerNode()) + { + if (categoryNode.has("in")) + { + JsonNode categoryInNode = categoryNode.get("in"); + if (categoryInNode != null) + { + String categoryInStr = categoryInNode.asText(); + if (StringUtils.isNotBlank(categoryInStr)) + { + String[] categoryInArray = categoryInStr.split(","); + if (categoryInArray != null) + { + for(String value : categoryInArray) + { + if (StringUtils.isNotBlank(value)) + { + categoryInList.add(value.trim()); + } + } + } + } + } + } + if (categoryNode.has("not")) + { + JsonNode categoryNotInNode = categoryNode.get("not"); + if (categoryNotInNode != null) + { + String categoryNotInStr = categoryNotInNode.asText(); + if (StringUtils.isNotBlank(categoryNotInStr)) + { + String[] categoryNotInArray = categoryNotInStr.split(","); + if (categoryNotInArray != null) + { + for(String value : categoryNotInArray) + { + if (StringUtils.isNotBlank(value)) + { + categoryNotInList.add(value.trim()); + } + } + } + } + } + } + } + } + + if (searchOpt.has("metric")) { + JsonNode metricNode = searchOpt.get("metric"); + if (metricNode != null && metricNode.isContainerNode()) + { + if (metricNode.has("in")) + { + JsonNode metricInNode = metricNode.get("in"); + if (metricInNode != null) + { + String metricInStr = metricInNode.asText(); + if (StringUtils.isNotBlank(metricInStr)) + { + String[] metricInArray = metricInStr.split(","); + if (metricInArray != null) + { + for(String value : metricInArray) + { + if (StringUtils.isNotBlank(value)) + { + metricInList.add(value.trim()); + } + } + } + } + } + } + if (metricNode.has("not")) + { + JsonNode metricNotInNode = metricNode.get("not"); + if (metricNotInNode != null) + { + String metricNotInStr = metricNotInNode.asText(); + if (StringUtils.isNotBlank(metricNotInStr)) + { + String[] metricNotInArray = metricNotInStr.split(","); + if (metricNotInArray != null) + { + for(String value : metricNotInArray) + { + if (StringUtils.isNotBlank(value)) + { + metricNotInList.add(value.trim()); + } + } + } + } + } + } + } + } + + boolean needAndKeyword = false; + + final List pagedMetrics = new ArrayList(); + final JdbcTemplate jdbcTemplate = getJdbcTemplate(); + javax.sql.DataSource ds = jdbcTemplate.getDataSource(); + DataSourceTransactionManager tm = new DataSourceTransactionManager(ds); + + TransactionTemplate txTemplate = new TransactionTemplate(tm); + + ObjectNode result; + String query = ADV_SEARCH_METRIC; + + if (dashboardInList.size() > 0 || dashboardNotInList.size() > 0) + { + boolean dashboardNeedAndKeyword = false; + if (dashboardInList.size() > 0) + { + int indexForDashboardInList = 0; + for (String dashboard : dashboardInList) + { + if (indexForDashboardInList == 0) + { + query += "WHERE dashboard_name in ('" + dashboard + "'"; + } + else + { + query += ", '" + dashboard + "'"; + } + indexForDashboardInList++; + } + query += ") "; + dashboardNeedAndKeyword = true; + } + if (dashboardNotInList.size() > 0) + { + if (dashboardNeedAndKeyword) + { + query += " AND "; + } + else + { + query += " WHERE "; + } + int indexForDashboardNotInList = 0; + for (String dashboard : dashboardNotInList) + { + if (indexForDashboardNotInList == 0) + { + query += "dashboard_name not in ('" + dashboard + "'"; + } + else + { + query += ", '" + dashboard + "'"; + } + indexForDashboardNotInList++; + } + query += ") "; + } + needAndKeyword = true; + } + + if (groupInList.size() > 0 || groupNotInList.size() > 0) + { + if (needAndKeyword) + { + query += " AND "; + } + else + { + query += " WHERE "; + } + query += "( "; + boolean groupNeedAndKeyword = false; + if (groupInList.size() > 0) + { + query += "( "; + int indexForGroupInList = 0; + for (String group : groupInList) + { + if (indexForGroupInList == 0) + { + query += "metric_group LIKE '%" + group + "%'"; + } + else + { + query += " or metric_group LIKE '%" + group + "%'"; + } + indexForGroupInList++; + } + query += ") "; + groupNeedAndKeyword = true; + } + if (groupNotInList.size() > 0) + { + if (groupNeedAndKeyword) + { + query += " AND "; + } + query += "( "; + int indexForGroupNotInList = 0; + for (String group : groupNotInList) + { + if (indexForGroupNotInList == 0) + { + query += "metric_group NOT LIKE '%" + group + "%'"; + } + else + { + query += " and metric_group NOT LIKE '%" + group + "%'"; + } + indexForGroupNotInList++; + } + query += ") "; + } + query += ") "; + needAndKeyword = true; + } + + if (categoryInList.size() > 0 || categoryNotInList.size() > 0) + { + if (needAndKeyword) + { + query += " AND "; + } + else + { + query += " WHERE "; + } + query += "( "; + boolean categoryNeedAndKeyword = false; + if (categoryInList.size() > 0) + { + int indexForCategoryInList = 0; + query += "( "; + for (String category : categoryInList) + { + if (indexForCategoryInList == 0) + { + query += "metric_category LIKE '%" + category + "%'"; + } + else + { + query += " or metric_category LIKE '%" + category + "%'"; + } + indexForCategoryInList++; + } + query += ") "; + categoryNeedAndKeyword = true; + } + if (categoryNotInList.size() > 0) + { + if (categoryNeedAndKeyword) + { + query += " AND "; + } + query += "( "; + int indexForCategoryNotInList = 0; + for (String category : categoryNotInList) + { + if (indexForCategoryNotInList == 0) + { + query += "metric_category NOT LIKE '%" + category + "%'"; + } + else + { + query += " and metric_category NOT LIKE '%" + category + "%'"; + } + indexForCategoryNotInList++; + } + query += ") "; + } + query += ") "; + needAndKeyword = true; + } + + if (metricInList.size() > 0 || metricNotInList.size() > 0) + { + if (needAndKeyword) + { + query += " AND "; + } + else + { + query += " WHERE "; + } + query += "( "; + boolean metricNeedAndKeyword = false; + if (metricInList.size() > 0) + { + int indexForMetricInList = 0; + query += " ( "; + for (String metric : metricInList) + { + if (indexForMetricInList == 0) + { + query += "metric_name LIKE '%" + metric + "%'"; + } + else + { + query += " or metric_name LIKE '%" + metric + "%'"; + } + indexForMetricInList++; + } + query += ") "; + metricNeedAndKeyword = true; + } + if (metricNotInList.size() > 0) + { + if (metricNeedAndKeyword) + { + query += " AND "; + } + query += "( "; + int indexForMetricNotInList = 0; + for (String metric : metricNotInList) + { + if (indexForMetricNotInList == 0) + { + query += "metric_name NOT LIKE '%" + metric + "%'"; + } + else + { + query += " and metric_name NOT LIKE '%" + metric + "%'"; + } + indexForMetricNotInList++; + } + query += ") "; + } + query += " )"; + } + + query += " LIMIT " + (page-1)*size + ", " + size; + final String queryString = query; + + result = txTemplate.execute(new TransactionCallback() + { + public ObjectNode doInTransaction(TransactionStatus status) + { + List pagedMetrics = jdbcTemplate.query(queryString, new MetricRowMapper()); + + long count = 0; + try { + count = jdbcTemplate.queryForObject( + "SELECT FOUND_ROWS()", + Long.class); + } + catch(EmptyResultDataAccessException e) + { + Logger.error("Exception = " + e.getMessage()); + } + + ObjectNode resultNode = Json.newObject(); + resultNode.put("count", count); + resultNode.put("page", page); + resultNode.put("isMetrics", true); + resultNode.put("itemsPerPage", size); + resultNode.put("totalPages", (int)Math.ceil(count/((double)size))); + resultNode.set("data", Json.toJson(pagedMetrics)); + + return resultNode; + } + }); + return result; + } + resultNode.put("count", 0); + resultNode.put("page", page); + resultNode.put("itemsPerPage", size); + resultNode.put("totalPages", 0); + resultNode.set("data", Json.toJson("")); + return resultNode; + } + } diff --git a/web/app/dao/SearchDAO.java b/web/app/dao/SearchDAO.java index 2c0af78015..1345b07ae9 100644 --- a/web/app/dao/SearchDAO.java +++ b/web/app/dao/SearchDAO.java @@ -13,13 +13,11 @@ */ package dao; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.stream.Collectors; import java.util.stream.Stream; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.commons.lang3.StringUtils; import org.springframework.dao.EmptyResultDataAccessException; @@ -29,12 +27,23 @@ import org.springframework.transaction.TransactionStatus; import org.springframework.transaction.support.TransactionCallback; import org.springframework.transaction.support.TransactionTemplate; import play.Logger; +import play.Play; +import play.libs.F; import play.libs.Json; import play.cache.Cache; import models.*; +import play.libs.WS; public class SearchDAO extends AbstractMySQLOpenSourceDAO { + public static String ELASTICSEARCH_DATASET_URL_KEY = "elasticsearch.dataset.url"; + + public static String ELASTICSEARCH_METRIC_URL_KEY = "elasticsearch.metric.url"; + + public static String ELASTICSEARCH_FLOW_URL_KEY = "elasticsearch.flow.url"; + + public static String WHEREHOWS_SEARCH_ENGINE__KEY = "search.engine"; + public final static String SEARCH_DATASET_WITH_PAGINATION = "SELECT SQL_CALC_FOUND_ROWS " + "id, `name`, `schema`, `source`, `urn`, FROM_UNIXTIME(source_modified_time) as modified, " + "rank_01 + rank_02 + rank_03 + rank_04 + rank_05 + rank_06 + rank_07 + rank_08 + rank_09 as rank " + @@ -169,6 +178,324 @@ public class SearchDAO extends AbstractMySQLOpenSourceDAO return cachedAutoCompleteList; } + public static JsonNode elasticSearchDatasetByKeyword( + String category, + String keywords, + String source, + int page, + int size) + { + ObjectNode queryNode = Json.newObject(); + queryNode.put("from", (page-1)*size); + queryNode.put("size", size); + JsonNode responseNode = null; + ObjectNode keywordNode = null; + + try + { + keywordNode = utils.Search.generateElasticSearchQueryString(category, source, keywords); + } + catch(Exception e) + { + Logger.error("Elastic search dataset input query is not JSON format. Error message :" + e.getMessage()); + } + + if (keywordNode != null) + { + queryNode.put("query", keywordNode); + F.Promise < WS.Response> responsePromise = WS.url(Play.application().configuration().getString( + SearchDAO.ELASTICSEARCH_DATASET_URL_KEY)).post(queryNode); + responseNode = responsePromise.get().asJson(); + } + + ObjectNode resultNode = Json.newObject(); + Long count = 0L; + List pagedDatasets = new ArrayList(); + resultNode.put("page", page); + resultNode.put("category", category); + resultNode.put("source", source); + resultNode.put("itemsPerPage", size); + resultNode.put("keywords", keywords); + + if (responseNode != null && responseNode.isContainerNode() && responseNode.has("hits")) + { + JsonNode hitsNode = responseNode.get("hits"); + if (hitsNode != null) + { + if (hitsNode.has("total")) + { + count = hitsNode.get("total").asLong(); + } + if (hitsNode.has("hits")) + { + JsonNode dataNode = hitsNode.get("hits"); + if (dataNode != null && dataNode.isArray()) + { + Iterator arrayIterator = dataNode.elements(); + if (arrayIterator != null) + { + while (arrayIterator.hasNext()) + { + JsonNode node = arrayIterator.next(); + if (node.isContainerNode() && node.has("_id")) + { + Dataset dataset = new Dataset(); + dataset.id = node.get("_id").asLong(); + if (node.has("_source")) + { + JsonNode sourceNode = node.get("_source"); + if (sourceNode != null) + { + if (sourceNode.has("name")) + { + dataset.name = sourceNode.get("name").asText(); + } + if (sourceNode.has("source")) + { + dataset.source = sourceNode.get("source").asText(); + } + if (sourceNode.has("urn")) + { + dataset.urn = sourceNode.get("urn").asText(); + } + if (sourceNode.has("schema")) + { + dataset.schema = sourceNode.get("schema").asText(); + } + } + } + pagedDatasets.add(dataset); + } + } + } + + } + } + + } + } + resultNode.put("count", count); + resultNode.put("totalPages", (int)Math.ceil(count/((double)size))); + resultNode.set("data", Json.toJson(pagedDatasets)); + return resultNode; + } + + public static JsonNode elasticSearchMetricByKeyword( + String category, + String keywords, + int page, + int size) + { + ObjectNode queryNode = Json.newObject(); + queryNode.put("from", (page-1)*size); + queryNode.put("size", size); + JsonNode responseNode = null; + ObjectNode keywordNode = null; + + try + { + keywordNode = utils.Search.generateElasticSearchQueryString(category, null, keywords); + } + catch(Exception e) + { + Logger.error("Elastic search metric input query is not JSON format. Error message :" + e.getMessage()); + } + + if (keywordNode != null) + { + queryNode.put("query", keywordNode); + F.Promise < WS.Response> responsePromise = WS.url(Play.application().configuration().getString( + SearchDAO.ELASTICSEARCH_METRIC_URL_KEY)).post(queryNode); + responseNode = responsePromise.get().asJson(); + } + + ObjectNode resultNode = Json.newObject(); + Long count = 0L; + List pagedMetrics = new ArrayList(); + resultNode.put("page", page); + resultNode.put("category", category); + resultNode.put("isMetrics", true); + resultNode.put("itemsPerPage", size); + resultNode.put("keywords", keywords); + + if (responseNode != null && responseNode.isContainerNode() && responseNode.has("hits")) + { + JsonNode hitsNode = responseNode.get("hits"); + if (hitsNode != null) + { + if (hitsNode.has("total")) + { + count = hitsNode.get("total").asLong(); + } + if (hitsNode.has("hits")) + { + JsonNode dataNode = hitsNode.get("hits"); + if (dataNode != null && dataNode.isArray()) + { + Iterator arrayIterator = dataNode.elements(); + if (arrayIterator != null) + { + while (arrayIterator.hasNext()) + { + JsonNode node = arrayIterator.next(); + if (node.isContainerNode() && node.has("_id")) + { + Metric metric = new Metric(); + metric.id = node.get("_id").asInt(); + if (node.has("_source")) { + JsonNode sourceNode = node.get("_source"); + if (sourceNode != null) { + if (sourceNode.has("metric_name")) { + metric.name = sourceNode.get("metric_name").asText(); + } + if (sourceNode.has("metric_description")) { + metric.description = sourceNode.get("metric_description").asText(); + } + if (sourceNode.has("dashboard_name")) { + metric.dashboardName = sourceNode.get("dashboard_name").asText(); + } + if (sourceNode.has("metric_group")) { + metric.group = sourceNode.get("metric_group").asText(); + } + if (sourceNode.has("metric_category")) { + metric.category = sourceNode.get("metric_category").asText(); + } + if (sourceNode.has("urn")) { + metric.urn = sourceNode.get("urn").asText(); + } + if (sourceNode.has("metric_source")) { + metric.source = sourceNode.get("metric_source").asText(); + if (StringUtils.isBlank(metric.source)) + { + metric.source = null; + } + } + metric.schema = sourceNode.toString(); + } + } + pagedMetrics.add(metric); + } + } + } + + } + } + + } + } + resultNode.put("count", count); + resultNode.put("totalPages", (int)Math.ceil(count/((double)size))); + resultNode.set("data", Json.toJson(pagedMetrics)); + return resultNode; + } + + public static JsonNode elasticSearchFlowByKeyword( + String category, + String keywords, + int page, + int size) + { + ObjectNode queryNode = Json.newObject(); + queryNode.put("from", (page-1)*size); + queryNode.put("size", size); + JsonNode searchOpt = null; + JsonNode responseNode = null; + ObjectNode keywordNode = null; + + try + { + keywordNode = utils.Search.generateElasticSearchQueryString(category, null, keywords); + } + catch(Exception e) + { + Logger.error("Elastic search flow input query is not JSON format. Error message :" + e.getMessage()); + } + + if (keywordNode != null) + { + queryNode.put("query", keywordNode); + F.Promise < WS.Response> responsePromise = WS.url(Play.application().configuration().getString( + SearchDAO.ELASTICSEARCH_FLOW_URL_KEY)).post(queryNode); + responseNode = responsePromise.get().asJson(); + } + + ObjectNode resultNode = Json.newObject(); + Long count = 0L; + List pagedFlowJobs = new ArrayList(); + resultNode.put("page", page); + resultNode.put("category", category); + resultNode.put("isFlowJob", true); + resultNode.put("itemsPerPage", size); + resultNode.put("keywords", keywords); + + if (responseNode != null && responseNode.isContainerNode() && responseNode.has("hits")) + { + JsonNode hitsNode = responseNode.get("hits"); + if (hitsNode != null) + { + if (hitsNode.has("total")) + { + count = hitsNode.get("total").asLong(); + } + if (hitsNode.has("hits")) + { + JsonNode dataNode = hitsNode.get("hits"); + if (dataNode != null && dataNode.isArray()) + { + Iterator arrayIterator = dataNode.elements(); + if (arrayIterator != null) + { + while (arrayIterator.hasNext()) + { + JsonNode node = arrayIterator.next(); + if (node.isContainerNode() && node.has("_id")) + { + FlowJob flowJob = new FlowJob(); + if (node.has("_source")) { + JsonNode sourceNode = node.get("_source"); + if (sourceNode != null) { + if (sourceNode.has("app_code")) { + flowJob.appCode = sourceNode.get("app_code").asText(); + } + if (sourceNode.has("app_id")) { + flowJob.appId = sourceNode.get("app_id").asInt(); + } + if (sourceNode.has("flow_id")) { + flowJob.flowId = sourceNode.get("flow_id").asLong(); + } + if (sourceNode.has("flow_name")) { + flowJob.flowName = sourceNode.get("flow_name").asText(); + flowJob.displayName = flowJob.flowName; + } + if (sourceNode.has("flow_path")) { + flowJob.flowPath = sourceNode.get("flow_path").asText(); + } + if (sourceNode.has("flow_group")) { + flowJob.flowGroup = sourceNode.get("flow_group").asText(); + } + flowJob.link = "#/flows/" + flowJob.appCode + "/" + + flowJob.flowGroup + "/" + Long.toString(flowJob.flowId) + "/page/1"; + flowJob.path = flowJob.appCode + "/" + flowJob.flowPath; + + flowJob.schema = sourceNode.toString(); + } + } + pagedFlowJobs.add(flowJob); + } + } + } + + } + } + + } + } + resultNode.put("count", count); + resultNode.put("totalPages", (int)Math.ceil(count/((double)size))); + resultNode.set("data", Json.toJson(pagedFlowJobs)); + return resultNode; + } + public static ObjectNode getPagedDatasetByKeyword(String category, String keyword, String source, int page, int size) { List pagedDatasets = new ArrayList(); diff --git a/web/app/models/FlowJob.java b/web/app/models/FlowJob.java index 6b4250d555..4e784d7d7b 100644 --- a/web/app/models/FlowJob.java +++ b/web/app/models/FlowJob.java @@ -28,4 +28,5 @@ public class FlowJob { public String path; public Integer appId; public Long flowId; + public String schema; } diff --git a/web/app/utils/Search.java b/web/app/utils/Search.java new file mode 100644 index 0000000000..9a310b29f4 --- /dev/null +++ b/web/app/utils/Search.java @@ -0,0 +1,1194 @@ +/** + * Copyright 2015 LinkedIn Corp. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + */ +package utils; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.commons.lang3.StringUtils; + +import play.Logger; +import play.cache.Cache; +import play.libs.Json; + +import java.io.BufferedReader; +import java.io.FileReader; +import java.util.*; + +public class Search +{ + + private final static String datasetMustQueryUnit = + "{\"bool\": " + + "{\"must\":[" + + "{\"bool\":" + + "{\"should\":" + + "[{\"bool\":{\"should\":[{\"wildcard\":{\"name\":{\"value\":\"*$VALUE*\",\"boost\":16}}}," + + "{\"prefix\":{\"name\":{\"value\":\"$VALUE\",\"boost\":32}}}," + + "{\"match\":{\"name\":{\"query\":\"$VALUE\",\"boost\":48}}}," + + "{\"match\":{\"name\":{\"query\":\"$VALUE\",\"type\":\"phrase\",\"boost\":64}}}," + + "{\"wildcard\":{\"urn\":{\"value\":\"*$VALUE*\",\"boost\":8}}}," + + "{\"wildcard\":{\"field\":{\"value\":\"*$VALUE*\",\"boost\":4}}}," + + "{\"wildcard\":{\"properties\":{\"value\":\"*$VALUE*\",\"boost\":2}}}," + + "{\"wildcard\":{\"schema\":{\"value\":\"*$VALUE*\",\"boost\":1}}}]}}]}}, " + + "{\"match\":{\"source\": \"$SOURCE\"}}]}}"; + + private final static String datasetShouldQueryUnit = + "{\"bool\": " + + "{\"should\": [" + + "{\"wildcard\": {\"name\": {\"value\": \"*$VALUE*\", \"boost\": 16}}}, " + + "{\"prefix\": {\"name\": {\"value\": \"$VALUE\", \"boost\": 32}}}, " + + "{\"match\": {\"name\": {\"query\": \"$VALUE\", \"boost\": 48}}}, " + + "{\"match\": {\"name\": {\"query\": \"$VALUE\", \"type\": \"phrase\", \"boost\": 64}}}, " + + "{\"wildcard\": {\"urn\": {\"value\": \"*$VALUE*\", \"boost\": 8}}}, " + + "{\"wildcard\": {\"field\": {\"value\": \"*$VALUE*\", \"boost\": 4}}}, " + + "{\"wildcard\": {\"properties\": {\"value\": \"*$VALUE*\", \"boost\": 2}}}, " + + "{\"wildcard\": {\"schema\": {\"value\": \"*$VALUE*\", \"boost\": 1}}}" + + "]}" + + "}"; + + + private final static String metricShouldQueryUnit = + "{\"bool\": " + + "{\"should\": [" + + "{\"wildcard\": {\"metric_name\": {\"value\": \"*$VALUE*\", \"boost\": 32}}}, " + + "{\"prefix\": {\"metric_name\": {\"value\": \"$VALUE\", \"boost\": 36}}}, " + + "{\"match\": {\"metric_name\": {\"query\": \"$VALUE\", \"boost\": 48}}}, " + + "{\"match\": {\"metric_name\": {\"query\": \"$VALUE\", \"type\": \"phrase\", \"boost\": 64}}}, " + + "{\"wildcard\": {\"dashboard_name\": {\"value\": \"*$VALUE*\", \"boost\": 20}}}," + + "{\"prefix\": {\"dashboard_name\": {\"value\": \"$VALUE\", \"boost\": 24}}}, " + + "{\"match\": {\"dashboard_name\": {\"query\": \"$VALUE\", \"boost\": 26}}}, " + + "{\"match\": {\"dashboard_name\": {\"query\": \"$VALUE\", \"type\": \"phrase\", \"boost\": 28}}}, " + + "{\"wildcard\": {\"metric_group\": {\"value\": \"*$VALUE*\", \"boost\": 8}}}, " + + "{\"prefix\": {\"metric_group\": {\"value\": \"$VALUE\", \"boost\": 12}}}, " + + "{\"match\": {\"metric_group\": {\"query\": \"$VALUE\", \"boost\": 14}}}, " + + "{\"match\": {\"metric_group\": {\"query\": \"$VALUE\", \"type\": \"phrase\", \"boost\": 16}}}, " + + "{\"wildcard\": {\"metric_category\": {\"value\": \"*$VALUE*\", \"boost\": 1}}}, " + + "{\"prefix\": {\"metric_category\": {\"value\": \"$VALUE\", \"boost\": 2}}}, " + + "{\"match\": {\"metric_category\": {\"query\": \"$VALUE\", \"boost\": 3}}}, " + + "{\"match\": {\"metric_category\": {\"query\": \"$VALUE\", \"type\": \"phrase\", \"boost\": 4}}}" + + "]" + + "}" + + "}"; + + private final static String commentsQuery = + "{\"bool\":" + + "{ \"should\": [" + + "{\"has_child\": {\"type\": \"comment\", \"query\": {\"match\" : {\"text\" : \"$VALUE\"}}}}, " + + "{\"has_child\": {\"type\": \"field\", \"query\": {\"match\": {\"comments\" : \"$VALUE\" }}}}" + + "]" + + "}" + + "}"; + + private final static String flowShouldQueryUnit = + "{\"bool\": " + + "{\"should\": " + + "[{\"wildcard\": {\"app_code\": {\"value\": \"*$VALUE*\", \"boost\": 1}}}, " + + "{\"prefix\": {\"app_code\": {\"value\": \"$VALUE\", \"boost\": 2}}}, " + + "{\"match\": {\"app_code\": {\"query\": \"$VALUE\", \"boost\": 3}}}, " + + "{\"match\": {\"app_code\": {\"query\": \"$VALUE\", \"type\": \"phrase\", \"boost\": 4}}}, " + + "{\"wildcard\": {\"flow_name\": {\"value\": \"*$VALUE*\", \"boost\": 8}}}, " + + "{\"prefix\": {\"flow_name\": {\"value\": \"$VALUE\", \"boost\": 16}}}, " + + "{\"match\": {\"flow_name\": {\"query\": \"$VALUE\", \"boost\": 24}}}, " + + "{\"match\": {\"flow_name\": {\"query\": \"$VALUE\", \"type\": \"phrase\", \"boost\": 32}}}, " + + "{\"wildcard\": {\"jobs.job_name\": {\"value\": \"*$VALUE*\", \"boost\": 8}}}, " + + "{\"prefix\": {\"jobs.job_name\": {\"value\": \"$VALUE\", \"boost\": 16}}}, " + + "{\"match\": {\"jobs.job_name\": {\"query\": \"$VALUE\", \"boost\": 24}}}, " + + "{\"match\": {\"jobs.job_name\": {\"query\": \"$VALUE\", \"type\": \"phrase\", \"boost\": 32}}}" + + "]" + + "}" + + "}"; + + public final static String DATASET_CATEGORY = "datasets"; + + public final static String METRIC_CATEGORY = "metrics"; + + public final static String COMMENT_CATEGORY = "comments"; + + public final static String FLOW_CATEGORY = "flows"; + + public final static String JOB_CATEGORY = "jobs"; + + public static ObjectNode generateElasticSearchQueryString(String category, String source, String keywords) + { + if (StringUtils.isBlank(keywords)) + return null; + + List shouldValueList = new ArrayList(); + + String queryTemplate = datasetShouldQueryUnit; + String[] values = keywords.trim().split(","); + if (StringUtils.isNotBlank(category)) + { + if (category.equalsIgnoreCase(METRIC_CATEGORY)) + { + queryTemplate = metricShouldQueryUnit; + } + else if (category.equalsIgnoreCase(COMMENT_CATEGORY)) + { + queryTemplate = commentsQuery; + } + else if (category.equalsIgnoreCase(FLOW_CATEGORY) || category.equalsIgnoreCase(JOB_CATEGORY)) + { + queryTemplate = flowShouldQueryUnit; + } + else if (category.equalsIgnoreCase(DATASET_CATEGORY) && StringUtils.isNotBlank(source)) + { + queryTemplate = datasetMustQueryUnit; + } + } + + for(String value : values) + { + if (StringUtils.isNotBlank(value)) { + String query= queryTemplate.replace("$VALUE", value.replace("\"", "").toLowerCase().trim()); + if (StringUtils.isNotBlank(source)) + { + query = query.replace("$SOURCE", source.toLowerCase()); + } + shouldValueList.add(Json.parse(query)); + } + } + + ObjectNode shouldNode = Json.newObject(); + shouldNode.set("should", Json.toJson(shouldValueList)); + ObjectNode queryNode = Json.newObject(); + queryNode.put("bool", shouldNode); + return queryNode; + } + + public static ObjectNode generateDatasetAdvSearchQueryString(JsonNode searchOpt) + { + List scopeInList = new ArrayList(); + List scopeNotInList = new ArrayList(); + List tableInList = new ArrayList(); + List tableNotInList = new ArrayList(); + List fieldAnyList = new ArrayList(); + List fieldAllList = new ArrayList(); + List fieldNotInList = new ArrayList(); + String comments = null; + String datasetSources = null; + + if (searchOpt != null && (searchOpt.isContainerNode())) { + if (searchOpt.has("scope")) { + JsonNode scopeNode = searchOpt.get("scope"); + if (scopeNode != null && scopeNode.isContainerNode()) { + if (scopeNode.has("in")) { + JsonNode scopeInNode = scopeNode.get("in"); + if (scopeInNode != null) { + String scopeInStr = scopeInNode.asText(); + if (StringUtils.isNotBlank(scopeInStr)) { + String[] scopeInArray = scopeInStr.split(","); + if (scopeInArray != null) { + for (String value : scopeInArray) { + if (StringUtils.isNotBlank(value)) { + scopeInList.add(value.trim()); + } + } + } + } + } + } + if (scopeNode.has("not")) { + JsonNode scopeNotInNode = scopeNode.get("not"); + if (scopeNotInNode != null) { + String scopeNotInStr = scopeNotInNode.asText(); + if (StringUtils.isNotBlank(scopeNotInStr)) { + String[] scopeNotInArray = scopeNotInStr.split(","); + if (scopeNotInArray != null) { + for (String value : scopeNotInArray) { + if (StringUtils.isNotBlank(value)) { + scopeNotInList.add(value.trim()); + } + } + } + } + } + } + } + } + + if (searchOpt.has("table")) { + JsonNode tableNode = searchOpt.get("table"); + if (tableNode != null && tableNode.isContainerNode()) { + if (tableNode.has("in")) { + JsonNode tableInNode = tableNode.get("in"); + if (tableInNode != null) { + String tableInStr = tableInNode.asText(); + if (StringUtils.isNotBlank(tableInStr)) { + String[] tableInArray = tableInStr.split(","); + if (tableInArray != null) { + for (String value : tableInArray) { + if (StringUtils.isNotBlank(value)) { + tableInList.add(value.trim()); + } + } + } + } + } + } + if (tableNode.has("not")) { + JsonNode tableNotInNode = tableNode.get("not"); + if (tableNotInNode != null) { + String tableNotInStr = tableNotInNode.asText(); + if (StringUtils.isNotBlank(tableNotInStr)) { + String[] tableNotInArray = tableNotInStr.split(","); + if (tableNotInArray != null) { + for (String value : tableNotInArray) { + if (StringUtils.isNotBlank(value)) { + tableNotInList.add(value.trim()); + } + } + } + } + } + } + } + } + + if (searchOpt.has("fields")) { + JsonNode fieldNode = searchOpt.get("fields"); + if (fieldNode != null && fieldNode.isContainerNode()) { + if (fieldNode.has("any")) { + JsonNode fieldAnyNode = fieldNode.get("any"); + if (fieldAnyNode != null) { + String fieldAnyStr = fieldAnyNode.asText(); + if (StringUtils.isNotBlank(fieldAnyStr)) { + String[] fieldAnyArray = fieldAnyStr.split(","); + if (fieldAnyArray != null) { + for (String value : fieldAnyArray) { + if (StringUtils.isNotBlank(value)) { + fieldAnyList.add(value.trim()); + } + } + } + } + } + } + if (fieldNode.has("all")) { + JsonNode fieldAllNode = fieldNode.get("all"); + if (fieldAllNode != null) { + String fieldAllStr = fieldAllNode.asText(); + if (StringUtils.isNotBlank(fieldAllStr)) { + String[] fieldAllArray = fieldAllStr.split(","); + if (fieldAllArray != null) { + for (String value : fieldAllArray) { + if (StringUtils.isNotBlank(value)) { + fieldAllList.add(value.trim()); + } + } + } + } + } + } + if (fieldNode.has("not")) { + JsonNode fieldNotInNode = fieldNode.get("not"); + if (fieldNotInNode != null) { + String fieldNotInStr = fieldNotInNode.asText(); + if (StringUtils.isNotBlank(fieldNotInStr)) { + String[] fieldNotInArray = fieldNotInStr.split(","); + if (fieldNotInArray != null) { + for (String value : fieldNotInArray) { + if (StringUtils.isNotBlank(value)) { + fieldNotInList.add(value.trim()); + } + } + } + } + } + } + } + } + if (searchOpt.has("sources")) { + JsonNode sourcesNode = searchOpt.get("sources"); + if (sourcesNode != null) + { + datasetSources = sourcesNode.asText(); + } + } + if (searchOpt.has("comments")) { + JsonNode commentsNode = searchOpt.get("comments"); + if (commentsNode != null) { + comments = commentsNode.asText(); + } + } + } + + List mustValueList = new ArrayList(); + List mustNotValueList = new ArrayList(); + List shouldValueList = new ArrayList(); + String hasChildValue = "{\"has_child\": " + + "{\"type\": \"$TYPE\", \"query\": {\"match\" : {\"$FIELD\" : \"$VALUE\"}}}}"; + String matchQueryUnit = "{\"query\": {\"match\" : {\"$FIELD\" : \"$VALUE\"}}}"; + + String shouldQueryUnit = "{\"bool\": " + + "{\"should\": [" + + "{\"wildcard\": {\"$FIELD\": {\"value\": \"*$VALUE*\", \"boost\": 1}}}, " + + "{\"prefix\": {\"$FIELD\": {\"value\": \"$VALUE\", \"boost\": 4}}}, " + + "{\"term\": {\"$FIELD\": {\"value\": \"$VALUE\", \"boost\": 16}}}" + + "]}}"; + String mustNotQueryUnit = "{\"term\" : {\"$FIELD\" : \"$VALUE\"}}"; + + if (scopeInList != null && scopeInList.size() > 0) + { + for(String scope : scopeInList) + { + if (StringUtils.isNotBlank(scope)) + { + shouldValueList.add( + Json.parse( + shouldQueryUnit.replace("$FIELD", "parent_name"). + replace("$VALUE", scope.toLowerCase().trim()))); + } + } + ObjectNode shouldNode = Json.newObject(); + ObjectNode boolNode = Json.newObject(); + shouldNode.set("should", Json.toJson(shouldValueList)); + boolNode.put("bool", shouldNode); + mustValueList.add(boolNode); + } + if (scopeNotInList != null && scopeNotInList.size() > 0) + { + shouldValueList.clear(); + for(String scope: scopeNotInList) + { + if (StringUtils.isNotBlank(scope)) + { + shouldValueList.add( + Json.parse( + mustNotQueryUnit.replace("$FIELD", "parent_name"). + replace("$VALUE", scope.toLowerCase().trim()))); + } + } + ObjectNode shouldNode = Json.newObject(); + ObjectNode boolNode = Json.newObject(); + shouldNode.set("should", Json.toJson(shouldValueList)); + boolNode.put("bool", shouldNode); + mustNotValueList.add(boolNode); + } + if (tableInList != null && tableInList.size() > 0) + { + shouldValueList.clear(); + for(String table : tableInList) + { + if (StringUtils.isNotBlank(table)) + { + shouldValueList.add( + Json.parse( + shouldQueryUnit.replace("$FIELD", "name"). + replace("$VALUE", table.toLowerCase().trim()))); + } + } + ObjectNode shouldNode = Json.newObject(); + ObjectNode boolNode = Json.newObject(); + shouldNode.set("should", Json.toJson(shouldValueList)); + boolNode.put("bool", shouldNode); + mustValueList.add(boolNode); + } + if (tableNotInList != null && tableNotInList.size() > 0) + { + shouldValueList.clear(); + for(String table : tableNotInList) + { + if (StringUtils.isNotBlank(table)) + { + shouldValueList.add( + Json.parse( + mustNotQueryUnit.replace("$FIELD", "name"). + replace("$VALUE", table.toLowerCase().trim()))); + } + } + ObjectNode shouldNode = Json.newObject(); + ObjectNode boolNode = Json.newObject(); + shouldNode.set("should", Json.toJson(shouldValueList)); + boolNode.put("bool", shouldNode); + mustNotValueList.add(boolNode); + } + if (fieldAnyList != null && fieldAnyList.size() > 0) + { + shouldValueList.clear(); + for(String field : fieldAnyList) + { + if (StringUtils.isNotBlank(field)) + { + shouldValueList.add( + Json.parse( + hasChildValue.replace("$TYPE", "field"). + replace("$FIELD", "field_name"). + replace("$VALUE", field.toLowerCase().trim()))); + } + } + ObjectNode shouldNode = Json.newObject(); + ObjectNode boolNode = Json.newObject(); + shouldNode.set("should", Json.toJson(shouldValueList)); + boolNode.put("bool", shouldNode); + mustValueList.add(boolNode); + } + if (fieldAllList != null && fieldAllList.size() > 0) + { + for(String field : fieldAllList) + { + if (StringUtils.isNotBlank(field)) + { + mustValueList.add( + Json.parse( + hasChildValue.replace("$TYPE", "field"). + replace("$FIELD", "field_name"). + replace("$VALUE", field.toLowerCase().trim()))); + } + } + } + if (fieldNotInList != null && fieldNotInList.size() > 0) + { + shouldValueList.clear(); + for(String field : fieldNotInList) + { + if (StringUtils.isNotBlank(field)) + { + shouldValueList.add( + Json.parse( + hasChildValue.replace("$TYPE", "field"). + replace("$FIELD", "field_name"). + replace("$VALUE", field.toLowerCase().trim()))); + } + } + ObjectNode shouldNode = Json.newObject(); + ObjectNode boolNode = Json.newObject(); + shouldNode.set("should", Json.toJson(shouldValueList)); + boolNode.put("bool", shouldNode); + mustNotValueList.add(boolNode); + } + if (StringUtils.isNotBlank(comments)) + { + shouldValueList.clear(); + String inputComments[] = comments.trim().split(","); + for(String comment : inputComments) + { + if (StringUtils.isNotBlank(comment)) + { + shouldValueList.add( + Json.parse( + hasChildValue.replace("$TYPE", "comment"). + replace("$FIELD", "text"). + replace("$VALUE", comment.toLowerCase().trim()))); + shouldValueList.add( + Json.parse( + hasChildValue.replace("$TYPE", "field"). + replace("$FIELD", "comments"). + replace("$VALUE", comment.toLowerCase().trim()))); + } + } + ObjectNode shouldNode = Json.newObject(); + ObjectNode boolNode = Json.newObject(); + shouldNode.set("should", Json.toJson(shouldValueList)); + boolNode.put("bool", shouldNode); + mustValueList.add(boolNode); + } + if (StringUtils.isNotBlank(datasetSources)) + { + String inputSources[] = datasetSources.trim().split(","); + shouldValueList.clear(); + for(String source : inputSources) + { + if (StringUtils.isNotBlank(source)) + { + shouldValueList.add( + Json.parse( + matchQueryUnit.replace("$FIELD", "source"). + replace("$VALUE", source.toLowerCase().trim()))); + } + } + ObjectNode shouldNode = Json.newObject(); + ObjectNode boolNode = Json.newObject(); + shouldNode.set("should", Json.toJson(shouldValueList)); + boolNode.put("bool", shouldNode); + mustValueList.add(boolNode); + } + + ObjectNode boolNode = Json.newObject(); + ObjectNode queryNode = Json.newObject(); + if (mustValueList.size() > 0 && mustNotValueList.size() > 0) + { + boolNode.set("must", Json.toJson(mustValueList)); + boolNode.set("must_not", Json.toJson(mustNotValueList)); + queryNode.put("bool", boolNode); + } + else if (mustValueList.size() > 0) + { + boolNode.set("must", Json.toJson(mustValueList)); + queryNode.put("bool", boolNode); + } + else if (mustNotValueList.size() > 0) + { + boolNode.set("must_not", Json.toJson(mustNotValueList)); + queryNode.put("bool", boolNode); + } + + return queryNode; + } + + public static ObjectNode generateMetricAdvSearchQueryString(JsonNode searchOpt) + { + List dashboardInList = new ArrayList(); + List dashboardNotInList = new ArrayList(); + List groupInList = new ArrayList(); + List groupNotInList = new ArrayList(); + List categoryInList = new ArrayList(); + List categoryNotInList = new ArrayList(); + List metricInList = new ArrayList(); + List metricNotInList = new ArrayList(); + + if (searchOpt != null && (searchOpt.isContainerNode())) { + if (searchOpt.has("dashboard")) { + JsonNode dashboardNode = searchOpt.get("dashboard"); + if (dashboardNode != null && dashboardNode.isContainerNode()) { + if (dashboardNode.has("in")) { + JsonNode dashboardInNode = dashboardNode.get("in"); + if (dashboardInNode != null) { + String dashboardInStr = dashboardInNode.asText(); + if (StringUtils.isNotBlank(dashboardInStr)) { + String[] dashboardInArray = dashboardInStr.split(","); + if (dashboardInArray != null) { + for (String value : dashboardInArray) { + if (StringUtils.isNotBlank(value)) { + dashboardInList.add(value.trim()); + } + } + } + } + } + } + if (dashboardNode.has("not")) { + JsonNode dashboardNotInNode = dashboardNode.get("not"); + if (dashboardNotInNode != null) { + String dashboardNotInStr = dashboardNotInNode.asText(); + if (StringUtils.isNotBlank(dashboardNotInStr)) { + String[] dashboardNotInArray = dashboardNotInStr.split(","); + if (dashboardNotInArray != null) { + for (String value : dashboardNotInArray) { + if (StringUtils.isNotBlank(value)) { + dashboardNotInList.add(value.trim()); + } + } + } + } + } + } + } + } + + if (searchOpt.has("group")) { + JsonNode groupNode = searchOpt.get("group"); + if (groupNode != null && groupNode.isContainerNode()) { + if (groupNode.has("in")) { + JsonNode groupInNode = groupNode.get("in"); + if (groupInNode != null) { + String groupInStr = groupInNode.asText(); + if (StringUtils.isNotBlank(groupInStr)) { + String[] groupInArray = groupInStr.split(","); + if (groupInArray != null) { + for (String value : groupInArray) { + if (StringUtils.isNotBlank(value)) { + groupInList.add(value.trim()); + } + } + } + } + } + } + if (groupNode.has("not")) { + JsonNode groupNotInNode = groupNode.get("not"); + if (groupNotInNode != null) { + String groupNotInStr = groupNotInNode.asText(); + if (StringUtils.isNotBlank(groupNotInStr)) { + String[] groupNotInArray = groupNotInStr.split(","); + if (groupNotInArray != null) { + for (String value : groupNotInArray) { + if (StringUtils.isNotBlank(value)) { + groupNotInList.add(value.trim()); + } + } + } + } + } + } + } + } + + if (searchOpt.has("cat")) { + JsonNode categoryNode = searchOpt.get("cat"); + if (categoryNode != null && categoryNode.isContainerNode()) { + if (categoryNode.has("in")) { + JsonNode categoryInNode = categoryNode.get("in"); + if (categoryInNode != null) { + String categoryInStr = categoryInNode.asText(); + if (StringUtils.isNotBlank(categoryInStr)) { + String[] categoryInArray = categoryInStr.split(","); + if (categoryInArray != null) { + for (String value : categoryInArray) { + if (StringUtils.isNotBlank(value)) { + categoryInList.add(value.trim()); + } + } + } + } + } + } + if (categoryNode.has("not")) { + JsonNode categoryNotInNode = categoryNode.get("not"); + if (categoryNotInNode != null) { + String categoryNotInStr = categoryNotInNode.asText(); + if (StringUtils.isNotBlank(categoryNotInStr)) { + String[] categoryNotInArray = categoryNotInStr.split(","); + if (categoryNotInArray != null) { + for (String value : categoryNotInArray) { + if (StringUtils.isNotBlank(value)) { + categoryNotInList.add(value.trim()); + } + } + } + } + } + } + } + } + + if (searchOpt.has("metric")) { + JsonNode metricNode = searchOpt.get("metric"); + if (metricNode != null && metricNode.isContainerNode()) { + if (metricNode.has("in")) { + JsonNode metricInNode = metricNode.get("in"); + if (metricInNode != null) { + String metricInStr = metricInNode.asText(); + if (StringUtils.isNotBlank(metricInStr)) { + String[] metricInArray = metricInStr.split(","); + if (metricInArray != null) { + for (String value : metricInArray) { + if (StringUtils.isNotBlank(value)) { + metricInList.add(value.trim()); + } + } + } + } + } + } + if (metricNode.has("not")) { + JsonNode metricNotInNode = metricNode.get("not"); + if (metricNotInNode != null) { + String metricNotInStr = metricNotInNode.asText(); + if (StringUtils.isNotBlank(metricNotInStr)) { + String[] metricNotInArray = metricNotInStr.split(","); + if (metricNotInArray != null) { + for (String value : metricNotInArray) { + if (StringUtils.isNotBlank(value)) { + metricNotInList.add(value.trim()); + } + } + } + } + } + } + } + } + } + + List mustValueList = new ArrayList(); + List mustNotValueList= new ArrayList(); + List shouldValueList = new ArrayList(); + + String shouldQueryUnit = + "{\"bool\": " + + "{\"should\": " + + "[{\"wildcard\": {\"$FIELD\": {\"value\": \"*$VALUE*\", \"boost\": 1}}}, " + + "{\"prefix\": {\"$FIELD\": {\"value\": \"$VALUE\", \"boost\": 4}}}, " + + "{\"term\": {\"$FIELD\": {\"value\": \"$VALUE\", \"boost\": 16}}}" + + "]" + + "}" + + "}"; + String shouldMatchQueryUnit = + "{\"bool\": " + + "{\"should\": " + + "[{\"wildcard\": {\"$FIELD\": {\"value\": \"*$VALUE*\", \"boost\": 1}}}, " + + "{\"prefix\": {\"$FIELD\": {\"value\": \"$VALUE\", \"boost\": 4}}}, " + + "{\"match\": {\"$FIELD\": {\"query\": \"$VALUE\", \"boost\": 16}}}" + + "]" + + "}" + + "}"; + String mustNotQueryUnit = "{\"term\" : {\"$FIELD\" : \"$VALUE\"}}"; + + if (dashboardInList != null && dashboardInList.size() > 0) + { + for(String dashboard : dashboardInList) + { + if (StringUtils.isNotBlank(dashboard)) + { + shouldValueList.add( + Json.parse( + shouldMatchQueryUnit.replace("$FIELD", "dashboard_name"). + replace("$VALUE", dashboard.toLowerCase().trim()))); + } + } + ObjectNode shouldNode = Json.newObject(); + ObjectNode boolNode = Json.newObject(); + shouldNode.set("should", Json.toJson(shouldValueList)); + boolNode.put("bool", shouldNode); + mustValueList.add(boolNode); + } + if (dashboardNotInList != null && dashboardNotInList.size() > 0) + { + shouldValueList.clear(); + for(String dashboard : dashboardNotInList) + { + if (StringUtils.isNotBlank(dashboard)) + { + shouldValueList.add( + Json.parse( + mustNotQueryUnit.replace("$FIELD", "dashboard_name"). + replace("$VALUE", dashboard.toLowerCase().trim()))); + } + } + ObjectNode shouldNode = Json.newObject(); + ObjectNode boolNode = Json.newObject(); + shouldNode.set("should", Json.toJson(shouldValueList)); + boolNode.put("bool", shouldNode); + mustNotValueList.add(boolNode); + } + if (groupInList != null && groupInList.size() > 0) + { + shouldValueList.clear(); + for(String group : groupInList) + { + if (StringUtils.isNotBlank(group)) + { + shouldValueList.add( + Json.parse( + shouldMatchQueryUnit.replace("$FIELD", "metric_group"). + replace("$VALUE", group.toLowerCase().trim()))); + } + } + ObjectNode shouldNode = Json.newObject(); + ObjectNode boolNode = Json.newObject(); + shouldNode.set("should", Json.toJson(shouldValueList)); + boolNode.put("bool", shouldNode); + mustValueList.add(boolNode); + } + if (groupNotInList != null && groupNotInList.size() > 0) + { + shouldValueList.clear(); + for(String group : groupNotInList) + { + if (StringUtils.isNotBlank(group)) + { + shouldValueList.add( + Json.parse( + mustNotQueryUnit.replace("$FIELD", "metric_group"). + replace("$VALUE", group.toLowerCase().trim()))); + } + } + ObjectNode shouldNode = Json.newObject(); + ObjectNode boolNode = Json.newObject(); + shouldNode.set("should", Json.toJson(shouldValueList)); + boolNode.put("bool", shouldNode); + mustNotValueList.add(boolNode); + } + + if (categoryInList != null && categoryInList.size() > 0) + { + shouldValueList.clear(); + for(String category : categoryInList) + { + if (StringUtils.isNotBlank(category)) + { + shouldValueList.add( + Json.parse( + shouldMatchQueryUnit.replace("$FIELD", "metric_category"). + replace("$VALUE", category.toLowerCase().trim()))); + } + } + ObjectNode shouldNode = Json.newObject(); + ObjectNode boolNode = Json.newObject(); + shouldNode.set("should", Json.toJson(shouldValueList)); + boolNode.put("bool", shouldNode); + mustValueList.add(boolNode); + } + if (categoryNotInList != null && categoryNotInList.size() > 0) + { + shouldValueList.clear(); + for(String category : categoryNotInList) + { + if (StringUtils.isNotBlank(category)) + { + shouldValueList.add( + Json.parse( + mustNotQueryUnit.replace("$FIELD", "metric_category"). + replace("$VALUE", category.toLowerCase().trim()))); + } + } + ObjectNode shouldNode = Json.newObject(); + ObjectNode boolNode = Json.newObject(); + shouldNode.set("should", Json.toJson(shouldValueList)); + boolNode.put("bool", shouldNode); + mustNotValueList.add(boolNode); + } + + if (metricInList != null && metricInList.size() > 0) + { + shouldValueList.clear(); + for(String name : metricInList) + { + if (StringUtils.isNotBlank(name)) + { + shouldValueList.add( + Json.parse( + shouldMatchQueryUnit.replace("$FIELD", "metric_name"). + replace("$VALUE", name.toLowerCase().trim()))); + } + } + ObjectNode shouldNode = Json.newObject(); + ObjectNode boolNode = Json.newObject(); + shouldNode.set("should", Json.toJson(shouldValueList)); + boolNode.put("bool", shouldNode); + mustValueList.add(boolNode); + } + if (metricNotInList != null && metricNotInList.size() > 0) + { + shouldValueList.clear(); + for(String name : metricNotInList) + { + if (StringUtils.isNotBlank(name)) + { + shouldValueList.add( + Json.parse( + mustNotQueryUnit.replace("$FIELD", "metric_name"). + replace("$VALUE", name.toLowerCase().trim()))); + } + } + ObjectNode shouldNode = Json.newObject(); + ObjectNode boolNode = Json.newObject(); + shouldNode.set("should", Json.toJson(shouldValueList)); + boolNode.put("bool", shouldNode); + mustNotValueList.add(boolNode); + } + + ObjectNode boolNode = Json.newObject(); + ObjectNode queryNode = Json.newObject(); + if (mustValueList.size() > 0 && mustNotValueList.size() > 0) + { + boolNode.set("must", Json.toJson(mustValueList)); + boolNode.set("must_not", Json.toJson(mustNotValueList)); + queryNode.put("bool", boolNode); + } + else if (mustValueList.size() > 0) + { + boolNode.set("must", Json.toJson(mustValueList)); + queryNode.put("bool", boolNode); + } + else if (mustNotValueList.size() > 0) + { + boolNode.set("must_not", Json.toJson(mustNotValueList)); + queryNode.put("bool", boolNode); + } + + return queryNode; + } + + public static ObjectNode generateFlowJobAdvSearchQueryString(JsonNode searchOpt) + { + List appcodeInList = new ArrayList(); + List appcodeNotInList = new ArrayList(); + List flowInList = new ArrayList(); + List flowNotInList = new ArrayList(); + List jobInList = new ArrayList(); + List jobNotInList = new ArrayList(); + + if (searchOpt != null && (searchOpt.isContainerNode())) { + if (searchOpt.has("appcode")) { + JsonNode appcodeNode = searchOpt.get("appcode"); + if (appcodeNode != null && appcodeNode.isContainerNode()) { + if (appcodeNode.has("in")) { + JsonNode appcodeInNode = appcodeNode.get("in"); + if (appcodeInNode != null) { + String appcodeInStr = appcodeInNode.asText(); + if (StringUtils.isNotBlank(appcodeInStr)) { + String[] appcodeInArray = appcodeInStr.split(","); + if (appcodeInArray != null) { + for (String value : appcodeInArray) { + if (StringUtils.isNotBlank(value)) { + appcodeInList.add(value.trim()); + } + } + } + } + } + } + if (appcodeNode.has("not")) { + JsonNode appcodeNotInNode = appcodeNode.get("not"); + if (appcodeNotInNode != null) { + String appcodeNotInStr = appcodeNotInNode.asText(); + if (StringUtils.isNotBlank(appcodeNotInStr)) { + String[] appcodeNotInArray = appcodeNotInStr.split(","); + if (appcodeNotInArray != null) { + for (String value : appcodeNotInArray) { + if (StringUtils.isNotBlank(value)) { + appcodeNotInList.add(value.trim()); + } + } + } + } + } + } + } + } + + if (searchOpt.has("flow")) { + JsonNode flowNode = searchOpt.get("flow"); + if (flowNode != null && flowNode.isContainerNode()) { + if (flowNode.has("in")) { + JsonNode flowInNode = flowNode.get("in"); + if (flowInNode != null) { + String flowInStr = flowInNode.asText(); + if (StringUtils.isNotBlank(flowInStr)) { + String[] flowInArray = flowInStr.split(","); + if (flowInArray != null) { + for (String value : flowInArray) { + if (StringUtils.isNotBlank(value)) { + flowInList.add(value.trim()); + } + } + } + } + } + } + if (flowNode.has("not")) { + JsonNode flowNotInNode = flowNode.get("not"); + if (flowNotInNode != null) { + String flowNotInStr = flowNotInNode.asText(); + if (StringUtils.isNotBlank(flowNotInStr)) { + String[] flowNotInArray = flowNotInStr.split(","); + if (flowNotInArray != null) { + for (String value : flowNotInArray) { + if (StringUtils.isNotBlank(value)) { + flowNotInList.add(value.trim()); + } + } + } + } + } + } + } + } + + if (searchOpt.has("job")) { + JsonNode jobNode = searchOpt.get("job"); + if (jobNode != null && jobNode.isContainerNode()) { + if (jobNode.has("in")) { + JsonNode jobInNode = jobNode.get("in"); + if (jobInNode != null) { + String jobInStr = jobInNode.asText(); + if (StringUtils.isNotBlank(jobInStr)) { + String[] jobInArray = jobInStr.split(","); + if (jobInArray != null) { + for (String value : jobInArray) { + if (StringUtils.isNotBlank(value)) { + jobInList.add(value.trim()); + } + } + } + } + } + } + if (jobNode.has("not")) { + JsonNode jobNotInNode = jobNode.get("not"); + if (jobNotInNode != null) { + String jobNotInStr = jobNotInNode.asText(); + if (StringUtils.isNotBlank(jobNotInStr)) { + String[] jobNotInArray = jobNotInStr.split(","); + if (jobNotInArray != null) { + for (String value : jobNotInArray) { + if (StringUtils.isNotBlank(value)) { + jobNotInList.add(value.trim()); + } + } + } + } + } + } + } + } + } + + List mustValueList = new ArrayList(); + List mustNotValueList= new ArrayList(); + List shouldValueList = new ArrayList(); + + String shouldMatchQueryUnit = + "{\"bool\": " + + "{\"should\": " + + "[{\"wildcard\": {\"$FIELD\": {\"value\": \"*$VALUE*\", \"boost\": 1}}}, " + + "{\"prefix\": {\"$FIELD\": {\"value\": \"$VALUE\", \"boost\": 4}}}, " + + "{\"match\": {\"$FIELD\": {\"query\": \"$VALUE\", \"boost\": 16}}}" + + "]" + + "}" + + "}"; + String mustNotQueryUnit = "{\"term\" : {\"$FIELD\" : \"$VALUE\"}}"; + + if (appcodeInList != null && appcodeInList.size() > 0) + { + for(String appCode : appcodeInList) + { + if (StringUtils.isNotBlank(appCode)) + { + shouldValueList.add( + Json.parse( + shouldMatchQueryUnit.replace("$FIELD", "app_code"). + replace("$VALUE", appCode.toLowerCase().trim()))); + } + } + ObjectNode shouldNode = Json.newObject(); + ObjectNode boolNode = Json.newObject(); + shouldNode.set("should", Json.toJson(shouldValueList)); + boolNode.put("bool", shouldNode); + mustValueList.add(boolNode); + } + if (appcodeNotInList != null && appcodeNotInList.size() > 0) + { + shouldValueList.clear(); + for(String appCode : appcodeNotInList) + { + if (StringUtils.isNotBlank(appCode)) + { + shouldValueList.add( + Json.parse( + mustNotQueryUnit.replace("$FIELD", "app_code"). + replace("$VALUE", appCode.toLowerCase().trim()))); + } + } + ObjectNode shouldNode = Json.newObject(); + ObjectNode boolNode = Json.newObject(); + shouldNode.set("should", Json.toJson(shouldValueList)); + boolNode.put("bool", shouldNode); + mustNotValueList.add(boolNode); + } + + if (flowInList != null && flowInList.size() > 0) + { + shouldValueList.clear(); + for(String flow : flowInList) + { + if (StringUtils.isNotBlank(flow)) + { + shouldValueList.add( + Json.parse( + shouldMatchQueryUnit.replace("$FIELD", "flow_name"). + replace("$VALUE", flow.toLowerCase().trim()))); + } + } + ObjectNode shouldNode = Json.newObject(); + ObjectNode boolNode = Json.newObject(); + shouldNode.set("should", Json.toJson(shouldValueList)); + boolNode.put("bool", shouldNode); + mustValueList.add(boolNode); + } + if (flowNotInList != null && flowNotInList.size() > 0) + { + shouldValueList.clear(); + for(String flow : flowNotInList) + { + if (StringUtils.isNotBlank(flow)) + { + shouldValueList.add( + Json.parse( + mustNotQueryUnit.replace("$FIELD", "flow_name"). + replace("$VALUE", flow.toLowerCase().trim()))); + } + } + ObjectNode shouldNode = Json.newObject(); + ObjectNode boolNode = Json.newObject(); + shouldNode.set("should", Json.toJson(shouldValueList)); + boolNode.put("bool", shouldNode); + mustNotValueList.add(boolNode); + } + + if (jobInList != null && jobInList.size() > 0) + { + shouldValueList.clear(); + for(String job : jobInList) + { + if (StringUtils.isNotBlank(job)) + { + shouldValueList.add( + Json.parse( + shouldMatchQueryUnit.replace("$FIELD", "jobs.job_name"). + replace("$VALUE", job.toLowerCase().trim()))); + } + } + ObjectNode nestedNode = Json.newObject(); + ObjectNode queryNode = Json.newObject(); + ObjectNode shouldNode = Json.newObject(); + ObjectNode boolNode = Json.newObject(); + shouldNode.set("should", Json.toJson(shouldValueList)); + boolNode.put("bool", shouldNode); + queryNode.put("query", boolNode); + queryNode.put("path", "jobs"); + nestedNode.put("nested", queryNode); + + mustValueList.add(nestedNode); + } + if (jobNotInList != null && jobNotInList.size() > 0) + { + shouldValueList.clear(); + for(String job : jobNotInList) + { + if (StringUtils.isNotBlank(job)) + { + shouldValueList.add( + Json.parse( + mustNotQueryUnit.replace("$FIELD", "jobs.job_name"). + replace("$VALUE", job.toLowerCase().trim()))); + } + } + ObjectNode nestedNode = Json.newObject(); + ObjectNode queryNode = Json.newObject(); + ObjectNode shouldNode = Json.newObject(); + ObjectNode boolNode = Json.newObject(); + shouldNode.set("should", Json.toJson(shouldValueList)); + boolNode.put("bool", shouldNode); + queryNode.put("query", boolNode); + queryNode.put("path", "jobs"); + nestedNode.put("nested", queryNode); + mustNotValueList.add(nestedNode); + } + + ObjectNode boolNode = Json.newObject(); + ObjectNode queryNode = Json.newObject(); + if (mustValueList.size() > 0 && mustNotValueList.size() > 0) + { + boolNode.set("must", Json.toJson(mustValueList)); + boolNode.set("must_not", Json.toJson(mustNotValueList)); + queryNode.put("bool", boolNode); + } + else if (mustValueList.size() > 0) + { + boolNode.set("must", Json.toJson(mustValueList)); + queryNode.put("bool", boolNode); + } + else if (mustNotValueList.size() > 0) + { + boolNode.set("must_not", Json.toJson(mustNotValueList)); + queryNode.put("bool", boolNode); + } + + return queryNode; + } + +} \ No newline at end of file diff --git a/web/app/views/index.scala.html b/web/app/views/index.scala.html index 2751f89bac..852677d4d5 100644 --- a/web/app/views/index.scala.html +++ b/web/app/views/index.scala.html @@ -1198,6 +1198,7 @@
+ {{flowJob.displayName}} @@ -1206,6 +1207,9 @@ {{{ flowJob.path }}}

source: {{{ flowJob.appCode }}}

+
+ {{{ flowJob.schema }}} +
@@ -1221,10 +1225,12 @@
{{#if isMetric}} + {{#link-to 'metric' dataset}} {{{dataset.name}}} {{/link-to}} {{else}} + {{#link-to 'dataset' dataset}} {{{dataset.name}}} {{/link-to}} @@ -1233,7 +1239,11 @@

{{{ dataset.urn }}}

-

source: {{{ dataset.source }}}

+ {{#if dataset.source}} +

source: {{{ dataset.source }}}

+ {{else}} +

source: Metric

+ {{/if}}
{{{ dataset.schema }}}
@@ -1302,6 +1312,7 @@
+ {{flowJob.displayName}} @@ -1310,6 +1321,9 @@ {{{ flowJob.path }}}

source: {{{ flowJob.appCode }}}

+
+ {{{ flowJob.schema }}} +
@@ -1324,6 +1338,7 @@
+ {{#link-to 'dataset' dataset}} {{{dataset.name}}} {{/link-to}} diff --git a/web/app/views/main.scala.html b/web/app/views/main.scala.html index 8526dbd1c9..35a1728814 100644 --- a/web/app/views/main.scala.html +++ b/web/app/views/main.scala.html @@ -90,20 +90,55 @@