diff --git a/wherehows-dao/build.gradle b/wherehows-dao/build.gradle index 002eac4f5b..4100d6d440 100644 --- a/wherehows-dao/build.gradle +++ b/wherehows-dao/build.gradle @@ -15,6 +15,9 @@ dependencies { runtime externalDependency.hikaricp testCompile externalDependency.testng + + compile externalDependency.play_java_ws + } findbugs { @@ -31,4 +34,4 @@ tasks.withType(FindBugs) { test { // enable TestNG support (default is JUnit) useTestNG() -} +} \ No newline at end of file diff --git a/wherehows-dao/src/main/java/wherehows/dao/table/SearchDAO.java b/wherehows-dao/src/main/java/wherehows/dao/table/SearchDAO.java new file mode 100644 index 0000000000..d1c134438c --- /dev/null +++ b/wherehows-dao/src/main/java/wherehows/dao/table/SearchDAO.java @@ -0,0 +1,344 @@ +/** + * Copyright 2015 LinkedIn Corp. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + */ + +package wherehows.dao.table; +import java.util.*; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import play.Logger; +import play.Play; +import play.libs.F.Promise; +import play.libs.Json; +import play.libs.ws.*; +import wherehows.models.table.Dataset; + + +import static wherehows.util.Search.*; + +public class SearchDAO +{ + public static String ELASTICSEARCH_DATASET_URL_KEY = "elasticsearch.dataset.url"; + public static String WHEREHOWS_SEARCH_ENGINE_KEY = "search.engine"; + + public static List getAutoCompleteList(String input, int limit) + { + List names = new ArrayList<>(); + names.addAll(getAutoCompleteListDataset(input, limit / 3)); + return names; + } + + public static List getAutoCompleteListDataset(String input, int limit) + { + String elasticSearchTypeURLKey = "elasticsearch.dataset.url"; + String fieldName = "name_suggest"; + return getAutoCompleteListbyES(elasticSearchTypeURLKey,fieldName,input,limit); + } + + public static List getAutoCompleteListbyES(String elasticSearchTypeURLKey, String fieldName, String input, + int limit) + { + // use elastic search completion suggester, ES will validate the input and limit + List completionSuggestionList = new ArrayList(); + Set completionSuggestionSet = new HashSet(); + + JsonNode responseNode = null; + ObjectNode keywordNode = null; + + try { + keywordNode = generateElasticSearchCompletionSuggesterQuery(fieldName, input, limit); + } catch (Exception e) { + Logger.error("Elastic search completion suggester error. Error message :" + e.getMessage()); + } + + Logger.info("The completion suggester query sent to Elastic Search was: " + keywordNode.toString()); + + Promise responsePromise = + WS.url(Play.application().configuration().getString(elasticSearchTypeURLKey)).post(keywordNode); + responseNode = responsePromise.get(1000).asJson(); + + if (responseNode == null || !responseNode.isContainerNode()) { + return completionSuggestionList; + } + + JsonNode suggestNode = responseNode.get("suggest"); + if (suggestNode == null || !suggestNode.has("wh-suggest")) { + Logger.error("Elastic search completion suggester response does not contain suggest node"); + return completionSuggestionList; + } + + JsonNode whSuggestNode = suggestNode.get("wh-suggest"); + if (whSuggestNode == null || !whSuggestNode.isArray()) { + Logger.error("Elastic search completion suggester response does not contain wh-suggest node"); + return completionSuggestionList; + } + + Iterator arrayIterator = whSuggestNode.elements(); + if (arrayIterator == null) { + return completionSuggestionList; + } + + while (arrayIterator.hasNext()) { + JsonNode node = arrayIterator.next(); + if (!node.isContainerNode() || !node.has("options")) { + continue; + } + + JsonNode optionsNode = node.get("options"); + if (optionsNode == null || !optionsNode.isArray()) { + continue; + } + + Iterator arrayIteratorOptions = optionsNode.elements(); + if (arrayIteratorOptions == null) { + continue; + } + + while (arrayIteratorOptions.hasNext()) { + JsonNode textNode = arrayIteratorOptions.next(); + if (textNode == null || !textNode.has("text")) { + continue; + } + String oneSuggestion = textNode.get("text").asText(); + completionSuggestionSet.add(oneSuggestion); + } + } + + completionSuggestionList.addAll(completionSuggestionSet); + Logger.info("Returned suggestion list is: " + completionSuggestionList); + return completionSuggestionList; + } + + // this is for did you mean feature + public static List getSuggestionList(String category, String searchKeyword) + { + List SuggestionList = new ArrayList(); + String elasticSearchType = "dataset"; + String elasticSearchTypeURLKey = "elasticsearch.dataset.url"; + String fieldName = "name"; + + JsonNode responseNode = null; + ObjectNode keywordNode = null; + + try { + String lCategory = category.toLowerCase(); + Logger.info("lCategory is " + category); + + // ToDO: deprecate category or reuse for entity + switch (lCategory) { + case "dataset": + elasticSearchType = "dataset"; + elasticSearchTypeURLKey = "elasticsearch.dataset.url"; + fieldName = "name"; + break; + default: + break; + } + + keywordNode = generateElasticSearchPhraseSuggesterQuery(elasticSearchType, fieldName, searchKeyword); + } catch (Exception e) { + Logger.error("Elastic search phrase suggester error. Error message :" + e.getMessage()); + } + + Logger.info("The suggest query sent to Elastic Search is: " + keywordNode.toString()); + + Promise responsePromise = + WS.url(Play.application().configuration().getString(elasticSearchTypeURLKey)).post(keywordNode); + responseNode = responsePromise.get(1000).asJson(); + + if (responseNode == null || !responseNode.isContainerNode() || !responseNode.has("hits")) { + return SuggestionList; + } + + JsonNode suggestNode = responseNode.get("suggest"); + Logger.info("suggestNode is " + suggestNode.toString()); + + if (suggestNode == null || !suggestNode.has("simple_phrase")) { + return SuggestionList; + } + + JsonNode simplePhraseNode = suggestNode.get("simple_phrase"); + if (simplePhraseNode == null || !simplePhraseNode.isArray()) { + return SuggestionList; + } + + Iterator arrayIterator = simplePhraseNode.elements(); + if (arrayIterator == null) { + return SuggestionList; + } + + while (arrayIterator.hasNext()) { + JsonNode node = arrayIterator.next(); + if (!node.isContainerNode() || !node.has("options")) { + continue; + } + + JsonNode optionsNode = node.get("options"); + if (optionsNode == null || !optionsNode.isArray()) { + continue; + } + + Iterator arrayIteratorOptions = optionsNode.elements(); + if (arrayIteratorOptions == null) { + continue; + } + + while (arrayIteratorOptions.hasNext()) { + JsonNode textNode = arrayIteratorOptions.next(); + if (textNode == null || !textNode.has("text")) { + continue; + } + String oneSuggestion = textNode.get("text").asText(); + SuggestionList.add(oneSuggestion); + } + } + + return SuggestionList; + } + + public static JsonNode elasticSearchDatasetByKeyword( + String category, + String keywords, + String source, + int page, + int size) + { + ObjectNode queryNode = Json.newObject(); + queryNode.put("from", (page-1)*size); + queryNode.put("size", size); + JsonNode responseNode = null; + ObjectNode keywordNode = null; + + try { + keywordNode = generateElasticSearchQueryString(category, source, keywords); + } catch (Exception e) { + Logger.error("Elastic search dataset input query is not JSON format. Error message :" + e.getMessage()); + } + + if (keywordNode != null) { + ObjectNode funcScoreNodes = Json.newObject(); + + ObjectNode fieldValueFactorNode = Json.newObject(); + fieldValueFactorNode.put("field", "static_boosting_score"); + fieldValueFactorNode.put("factor", 1); + fieldValueFactorNode.put("modifier", "square"); + fieldValueFactorNode.put("missing", 1); + + funcScoreNodes.put("query", keywordNode); + funcScoreNodes.put("field_value_factor", fieldValueFactorNode); + + ObjectNode funcScoreNodesWrapper = Json.newObject(); + funcScoreNodesWrapper.put("function_score", funcScoreNodes); + + queryNode.put("query", funcScoreNodesWrapper); + + ObjectNode filterNode = Json.newObject(); + try { + filterNode = generateElasticSearchFilterString(source); + } catch (Exception e) { + Logger.error("Elastic search filter query node generation failed :" + e.getMessage()); + } + + if (filterNode != null) { + queryNode.put("post_filter", filterNode); + } + + Logger.info( + " === elasticSearchDatasetByKeyword === The query sent to Elastic Search is: " + queryNode.toString()); + + Promise responsePromise = + WS.url(Play.application().configuration().getString(SearchDAO.ELASTICSEARCH_DATASET_URL_KEY)).post(queryNode); + responseNode = responsePromise.get(1000).asJson(); + + // Logger.debug("The responseNode from Elastic Search is: " + responseNode.toString()); + + } + + ObjectNode resultNode = Json.newObject(); + Long count = 0L; + List pagedDatasets = new ArrayList<>(); + resultNode.put("page", page); + resultNode.put("category", category); + resultNode.put("source", source); + resultNode.put("itemsPerPage", size); + resultNode.put("keywords", keywords); + + if (responseNode != null && responseNode.isContainerNode() && responseNode.has("hits")) + { + JsonNode hitsNode = responseNode.get("hits"); + if (hitsNode != null) + { + if (hitsNode.has("total")) + { + count = hitsNode.get("total").asLong(); + } + if (hitsNode.has("hits")) + { + JsonNode dataNode = hitsNode.get("hits"); + if (dataNode != null && dataNode.isArray()) + { + Iterator arrayIterator = dataNode.elements(); + if (arrayIterator != null) + { + while (arrayIterator.hasNext()) + { + JsonNode node = arrayIterator.next(); + if (node.isContainerNode() && node.has("_id")) + { + Dataset dataset = new Dataset(); + dataset.id = node.get("_id").asLong(); + if (node.has("_source")) + { + JsonNode sourceNode = node.get("_source"); + if (sourceNode != null) + { + if (sourceNode.has("name")) + { + dataset.name = sourceNode.get("name").asText(); + } + if (sourceNode.has("source")) + { + dataset.source = sourceNode.get("source").asText(); + } + if (sourceNode.has("urn")) + { + dataset.urn = sourceNode.get("urn").asText(); + } + if (sourceNode.has("schema")) + { + dataset.schema = sourceNode.get("schema").asText(); + } + } + } + pagedDatasets.add(dataset); + } + } + } + + } + } + + } + } + resultNode.put("count", count); + resultNode.put("totalPages", (int)Math.ceil(count/((double)size))); + resultNode.set("data", Json.toJson(pagedDatasets)); + return resultNode; + } + +} \ No newline at end of file diff --git a/wherehows-dao/src/main/java/wherehows/util/Search.java b/wherehows-dao/src/main/java/wherehows/util/Search.java new file mode 100644 index 0000000000..e533d4527c --- /dev/null +++ b/wherehows-dao/src/main/java/wherehows/util/Search.java @@ -0,0 +1,174 @@ +/** + * Copyright 2015 LinkedIn Corp. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + */ +package wherehows.util; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import java.util.ArrayList; +import java.util.List; +import java.nio.file.Paths; +import java.nio.file.Files; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; + +@Slf4j +public class Search { + public final static String DATASET_CATEGORY = "datasets"; + public final static String METRIC_CATEGORY = "metrics"; + public final static String COMMENT_CATEGORY = "comments"; + public final static String FLOW_CATEGORY = "flows"; + public final static String JOB_CATEGORY = "jobs"; + + private static final String WHZ_ELASTICSEARCH_DATASET_QUERY_FILE = System.getenv("WHZ_ELASTICSEARCH_DATASET_QUERY_TEMPLATE"); + private static final String WHZ_ELASTICSEARCH_METRIC_QUERY_FILE = System.getenv("WHZ_ELASTICSEARCH_METRIC_QUERY_TEMPLATE"); + private static final String WHZ_ELASTICSEARCH_FLOW_QUERY_FILE = System.getenv("WHZ_ELASTICSEARCH_FLOW_QUERY_TEMPLATE"); + private static final String WHZ_ELASTICSEARCH_COMMENT_QUERY_FILE = System.getenv("WHZ_ELASTICSEARCH_COMMENT_QUERY_TEMPLATE"); + private static final String WHZ_ELASTICSEARCH_SUGGESTER_QUERY_FILE = System.getenv("WHZ_ELASTICSEARCH_SUGGESTER_QUERY_TEMPLATE"); + private static final String WHZ_ELASTICSEARCH_AUTO_COMPLETION_QUERY_FILE = System.getenv("WHZ_ELASTICSEARCH_AUTO_COMPLETION_QUERY_TEMPLATE"); + private static final String WHZ_ELASTICSEARCH_FILTER_UNIT_FILE = System.getenv("WHZ_ELASTICSEARCH_FILTER_UNIT"); + + public static String readJsonQueryFile(String jsonFile) { + try { + ObjectMapper objectMapper = new ObjectMapper(); + + String contents = new String(Files.readAllBytes(Paths.get(jsonFile))); + JsonNode json = objectMapper.readTree(contents); + return json.toString(); + } catch (Exception e) { + log.error("ReadJsonQueryFile failed. Error: " + e.getMessage()); + e.printStackTrace(); + return null; + } + } + + public static ObjectNode generateElasticSearchCompletionSuggesterQuery(String field, String searchKeyword, + int limit) { + if (StringUtils.isBlank(searchKeyword)) { + return null; + } + + String queryTemplate = readJsonQueryFile(WHZ_ELASTICSEARCH_AUTO_COMPLETION_QUERY_FILE); + String query = queryTemplate.replace("$SEARCHKEYWORD", searchKeyword.toLowerCase()); + + if (StringUtils.isNotBlank(field)) { + query = query.replace("$FIELD", field.toLowerCase()); + } + + query = query.replace("$LIMIT", Integer.toString(limit)); + + ObjectNode suggestNode = new ObjectMapper().createObjectNode(); + + try { + ObjectNode textNode = (ObjectNode) new ObjectMapper().readTree(query); + suggestNode.putPOJO("suggest", textNode); + } catch (Exception e) { + log.error("suggest Exception = " + e.getMessage()); + } + + log.info("completionSuggesterQuery suggestNode is " + suggestNode.toString()); + return suggestNode; + } + + public static ObjectNode generateElasticSearchPhraseSuggesterQuery(String category, String field, + String searchKeyword) { + if (StringUtils.isBlank(searchKeyword)) { + return null; + } + + String queryTemplate = readJsonQueryFile(WHZ_ELASTICSEARCH_SUGGESTER_QUERY_FILE); + String query = queryTemplate.replace("$SEARCHKEYWORD", searchKeyword.toLowerCase()); + + if (StringUtils.isNotBlank(field)) { + query = query.replace("$FIELD", field.toLowerCase()); + } + + ObjectNode suggestNode = new ObjectMapper().createObjectNode(); + try { + ObjectNode textNode = (ObjectNode) new ObjectMapper().readTree(query); + suggestNode.putPOJO("suggest", textNode); + } catch (Exception e) { + log.error("suggest Exception = " + e.getMessage()); + } + log.info("suggestNode is " + suggestNode.toString()); + return suggestNode; + } + + public static ObjectNode generateElasticSearchQueryString(String category, String source, String keywords) throws Exception + { + if (StringUtils.isBlank(keywords)) { + return null; + } + + List shouldValueList = new ArrayList(); + + String queryTemplate = readJsonQueryFile(WHZ_ELASTICSEARCH_DATASET_QUERY_FILE); + + String[] values = keywords.trim().split(","); + if (StringUtils.isNotBlank(category)) { + if (category.equalsIgnoreCase(METRIC_CATEGORY)) { + queryTemplate = readJsonQueryFile(WHZ_ELASTICSEARCH_METRIC_QUERY_FILE); + ; + } else if (category.equalsIgnoreCase(COMMENT_CATEGORY)) { + queryTemplate = readJsonQueryFile(WHZ_ELASTICSEARCH_COMMENT_QUERY_FILE); + } else if (category.equalsIgnoreCase(FLOW_CATEGORY) || category.equalsIgnoreCase(JOB_CATEGORY)) { + queryTemplate = readJsonQueryFile(WHZ_ELASTICSEARCH_FLOW_QUERY_FILE); + } + } + + ObjectMapper objectMapper = new ObjectMapper(); + + for (String value : values) { + if (StringUtils.isNotBlank(value)) { + String query = queryTemplate.replace("$VALUE", value.replace("\"", "").toLowerCase().trim()); + shouldValueList.add(objectMapper.readTree(query)); + } + } + + ObjectNode shouldNode = new ObjectMapper().createObjectNode(); + shouldNode.putPOJO("should", shouldValueList); + ObjectNode queryNode = new ObjectMapper().createObjectNode(); + queryNode.putPOJO("bool", shouldNode); + + return queryNode; + } + + + public static ObjectNode generateElasticSearchFilterString(String sources) throws Exception{ + if (StringUtils.isBlank(sources)) { + return null; + } + + List shouldValueList = new ArrayList(); + + String queryTemplate = readJsonQueryFile(WHZ_ELASTICSEARCH_FILTER_UNIT_FILE); + String[] values = sources.trim().split(","); + + ObjectMapper objectMapper = new ObjectMapper(); + + for (String value : values) { + if (StringUtils.isNotBlank(value)) { + String query = queryTemplate.replace("$SOURCE", value.replace("\"", "").toLowerCase().trim()); + shouldValueList.add(objectMapper.readTree(query)); + } + } + + ObjectNode shouldNode = new ObjectMapper().createObjectNode(); + shouldNode.putPOJO("should", shouldValueList); + ObjectNode queryNode = new ObjectMapper().createObjectNode(); + queryNode.putPOJO("bool", shouldNode); + + return queryNode; + } +} \ No newline at end of file diff --git a/wherehows-frontend/app/controllers/api/v1/AdvSearch.java b/wherehows-frontend/app/controllers/api/v1/AdvSearch.java deleted file mode 100644 index 47c8ef0363..0000000000 --- a/wherehows-frontend/app/controllers/api/v1/AdvSearch.java +++ /dev/null @@ -1,181 +0,0 @@ -/** - * Copyright 2015 LinkedIn Corp. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - */ -package controllers.api.v1; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import dao.AdvSearchDAO; -import dao.SearchDAO; -import org.apache.commons.lang3.StringUtils; -import play.Logger; -import play.Play; -import play.libs.Json; -import play.mvc.Controller; -import play.mvc.Result; - - -public class AdvSearch extends Controller { - public static Result getDatasetSources() { - ObjectNode result = Json.newObject(); - - result.put("status", "ok"); - result.set("sources", Json.toJson(AdvSearchDAO.getDatasetSources())); - - return ok(result); - } - - public static Result getDatasetScopes() { - ObjectNode result = Json.newObject(); - - result.put("status", "ok"); - result.set("scopes", Json.toJson(AdvSearchDAO.getDatasetScopes())); - - return ok(result); - } - - public static Result getDatasetTableNames() { - ObjectNode result = Json.newObject(); - String scopes = request().getQueryString("scopes"); - result.put("status", "ok"); - result.set("tables", Json.toJson(AdvSearchDAO.getTableNames(scopes))); - - return ok(result); - } - - public static Result getDatasetFields() { - ObjectNode result = Json.newObject(); - String tables = request().getQueryString("tables"); - result.put("status", "ok"); - result.set("fields", Json.toJson(AdvSearchDAO.getFields(tables))); - - return ok(result); - } - - public static Result getFlowApplicationCodes() { - ObjectNode result = Json.newObject(); - result.put("status", "ok"); - result.set("appcodes", Json.toJson(AdvSearchDAO.getFlowApplicationCodes())); - - return ok(result); - } - - public static Result getFlowNames() { - ObjectNode result = Json.newObject(); - String apps = request().getQueryString("apps"); - result.put("status", "ok"); - result.set("flowNames", Json.toJson(AdvSearchDAO.getFlowNames(apps))); - - return ok(result); - } - - public static Result getJobNames() { - ObjectNode result = Json.newObject(); - result.put("status", "ok"); - result.set("jobNames", Json.toJson(AdvSearchDAO.getFlowJobNames())); - - return ok(result); - } - - public static Result getDashboardNames() { - ObjectNode result = Json.newObject(); - result.put("status", "ok"); - result.set("dashboardNames", Json.toJson(AdvSearchDAO.getMetricDashboardNames())); - - return ok(result); - } - - public static Result getMetricGroups() { - ObjectNode result = Json.newObject(); - result.put("status", "ok"); - result.set("metricGroups", Json.toJson(AdvSearchDAO.getMetricGroups())); - - return ok(result); - } - - public static Result getMetricCategories() { - ObjectNode result = Json.newObject(); - result.put("status", "ok"); - result.set("metricCategories", Json.toJson(AdvSearchDAO.getMetricCategories())); - - return ok(result); - } - - public static Result getMetricNames() { - ObjectNode result = Json.newObject(); - result.put("status", "ok"); - result.set("metricNames", Json.toJson(AdvSearchDAO.getMetricNames())); - - return ok(result); - } - - public static Result search() { - ObjectNode result = Json.newObject(); - String searchOptStr = request().getQueryString("searchOpts"); - JsonNode searchOpt = Json.parse(searchOptStr); - int page = 1; - int size = 10; - String pageStr = request().getQueryString("page"); - if (StringUtils.isBlank(pageStr)) { - page = 1; - } else { - try { - page = Integer.parseInt(pageStr); - } catch (NumberFormatException e) { - Logger.error("AdvSearch Controller search wrong page parameter. Error message: " + e.getMessage()); - page = 1; - } - } - - String sizeStr = request().getQueryString("size"); - if (StringUtils.isBlank(sizeStr)) { - size = 10; - } else { - try { - size = Integer.parseInt(sizeStr); - } catch (NumberFormatException e) { - Logger.error("AdvSearch Controller search wrong page parameter. Error message: " + e.getMessage()); - size = 10; - } - } - result.put("status", "ok"); - String searchEngine = Play.application().configuration().getString(SearchDAO.WHEREHOWS_SEARCH_ENGINE_KEY); - - if (searchOpt != null && searchOpt.has("category")) { - String category = searchOpt.get("category").asText(); - if (category.equalsIgnoreCase("flow")) { - if (StringUtils.isNotBlank(searchEngine) && searchEngine.equalsIgnoreCase("elasticsearch")) { - result.set("result", Json.toJson(AdvSearchDAO.elasticSearchFlowJobs(searchOpt, page, size))); - } else { - result.set("result", Json.toJson(AdvSearchDAO.searchFlows(searchOpt, page, size))); - } - return ok(result); - } else if (category.equalsIgnoreCase("metric")) { - if (StringUtils.isNotBlank(searchEngine) && searchEngine.equalsIgnoreCase("elasticsearch")) { - result.set("result", Json.toJson(AdvSearchDAO.elasticSearchMetric(searchOpt, page, size))); - } else { - result.set("result", Json.toJson(AdvSearchDAO.searchMetrics(searchOpt, page, size))); - } - return ok(result); - } - } - - if (StringUtils.isNotBlank(searchEngine) && searchEngine.equalsIgnoreCase("elasticsearch")) { - result.set("result", Json.toJson(AdvSearchDAO.elasticSearch(searchOpt, page, size))); - } else { - result.set("result", Json.toJson(AdvSearchDAO.search(searchOpt, page, size))); - } - - return ok(result); - } -} diff --git a/wherehows-frontend/app/controllers/api/v1/Search.java b/wherehows-frontend/app/controllers/api/v1/Search.java index 8b2bb4bbf6..450c3b6a64 100644 --- a/wherehows-frontend/app/controllers/api/v1/Search.java +++ b/wherehows-frontend/app/controllers/api/v1/Search.java @@ -14,7 +14,6 @@ package controllers.api.v1; import com.fasterxml.jackson.databind.node.ObjectNode; -import dao.SearchDAO; import java.util.List; import org.apache.commons.lang3.math.NumberUtils; import play.Logger; @@ -26,12 +25,12 @@ import play.mvc.Result; import static org.apache.commons.lang3.StringUtils.*; +import wherehows.dao.table.SearchDAO; + public class Search extends Controller { private static final String AUTOCOMPLETE_ALL_KEY = "autocomplete.all"; private static final String AUTOCOMPLETE_DATASET_KEY = "autocomplete.dataset"; - private static final String AUTOCOMPLETE_FLOW_KEY = "autocomplete.flow"; - private static final String AUTOCOMPLETE_METRIC_KEY = "autocomplete.metric"; private static final int DEFAULT_AUTOCOMPLETE_SIZE = 20; private static final int DEFAULT_AUTOCOMPLETE_CACHE_TIME = 3600; // cache for an hour @@ -79,49 +78,7 @@ public class Search extends Controller { return ok(result); } - public static Result getSearchAutoCompleteForMetric() { - // if not input, then get all search names (without limit). - String input = request().getQueryString("input"); - int size = 0; // size 0 means no limit - if (isNotBlank(input)) { - size = NumberUtils.toInt(request().getQueryString("size"), DEFAULT_AUTOCOMPLETE_SIZE); - } - String cacheKey = AUTOCOMPLETE_METRIC_KEY + (isNotBlank(input) ? "." + input : "-all"); - List names = (List) Cache.get(cacheKey); - if (names == null || names.size() == 0) { - names = SearchDAO.getAutoCompleteListMetric(input, size); - Cache.set(cacheKey, names, DEFAULT_AUTOCOMPLETE_CACHE_TIME); - } - - ObjectNode result = Json.newObject(); - result.put("status", "ok"); - result.put("input", input); - result.set("source", Json.toJson(names)); - return ok(result); - } - - public static Result getSearchAutoCompleteForFlow() { - // if not input, then get all search names (without limit). - String input = request().getQueryString("input"); - int size = 0; // size 0 means no limit - if (isNotBlank(input)) { - size = NumberUtils.toInt(request().getQueryString("size"), DEFAULT_AUTOCOMPLETE_SIZE); - } - - String cacheKey = AUTOCOMPLETE_FLOW_KEY + (isNotBlank(input) ? "." + input : "-all"); - List names = (List) Cache.get(cacheKey); - if (names == null || names.size() == 0) { - names = SearchDAO.getAutoCompleteListFlow(input, size); - Cache.set(cacheKey, names, DEFAULT_AUTOCOMPLETE_CACHE_TIME); - } - - ObjectNode result = Json.newObject(); - result.put("status", "ok"); - result.put("input", input); - result.set("source", Json.toJson(names)); - return ok(result); - } public static Result searchByKeyword() { ObjectNode result = Json.newObject(); @@ -165,38 +122,9 @@ public class Search extends Controller { } String searchEngine = Play.application().configuration().getString(SearchDAO.WHEREHOWS_SEARCH_ENGINE_KEY); + Logger.info("searchEngine is: " + searchEngine); // TODO: deprecated this setting - if (category.toLowerCase().equalsIgnoreCase("metrics")) { - if (isNotBlank(searchEngine) && searchEngine.equalsIgnoreCase("elasticsearch")) { - result.set("result", SearchDAO.elasticSearchMetricByKeyword(category, keyword, page, size)); - } else { - result.set("result", SearchDAO.getPagedMetricByKeyword(category, keyword, page, size)); - } - } else if (category.toLowerCase().equalsIgnoreCase("flows")) { - if (isNotBlank(searchEngine) && searchEngine.equalsIgnoreCase("elasticsearch")) { - result.set("result", SearchDAO.elasticSearchFlowByKeyword(category, keyword, page, size)); - } else { - result.set("result", SearchDAO.getPagedFlowByKeyword(category, keyword, page, size)); - } - } else if (category.toLowerCase().equalsIgnoreCase("jobs")) { - if (isNotBlank(searchEngine) && searchEngine.equalsIgnoreCase("elasticsearch")) { - result.set("result", SearchDAO.elasticSearchFlowByKeyword(category, keyword, page, size)); - } else { - result.set("result", SearchDAO.getPagedJobByKeyword(category, keyword, page, size)); - } - } else if (category.toLowerCase().equalsIgnoreCase("comments")) { - if (isNotBlank(searchEngine) && searchEngine.equalsIgnoreCase("elasticsearch")) { - result.set("result", SearchDAO.elasticSearchDatasetByKeyword(category, keyword, null, page, size)); - } else { - result.set("result", SearchDAO.getPagedCommentsByKeyword(category, keyword, page, size)); - } - } else { - if (isNotBlank(searchEngine) && searchEngine.equalsIgnoreCase("elasticsearch")) { - result.set("result", SearchDAO.elasticSearchDatasetByKeyword(category, keyword, source, page, size)); - } else { - result.set("result", SearchDAO.getPagedDatasetByKeyword(category, keyword, source, page, size)); - } - } + result.set("result", SearchDAO.elasticSearchDatasetByKeyword(category, keyword, source, page, size)); return ok(result); } diff --git a/wherehows-frontend/app/dao/AdvSearchDAO.java b/wherehows-frontend/app/dao/AdvSearchDAO.java deleted file mode 100644 index c0171ceb5a..0000000000 --- a/wherehows-frontend/app/dao/AdvSearchDAO.java +++ /dev/null @@ -1,2071 +0,0 @@ -/** - * Copyright 2015 LinkedIn Corp. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - */ -package dao; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import wherehows.models.table.Dataset; -import wherehows.models.table.FlowJob; -import wherehows.models.table.Metric; -import org.apache.commons.lang3.StringUtils; -import org.springframework.dao.EmptyResultDataAccessException; -import org.springframework.jdbc.core.JdbcTemplate; -import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; -import org.springframework.jdbc.datasource.DataSourceTransactionManager; -import org.springframework.transaction.TransactionStatus; -import org.springframework.transaction.support.TransactionCallback; -import org.springframework.transaction.support.TransactionTemplate; -import play.Logger; -import play.Play; -import play.libs.F.Promise; -import play.libs.Json; -import play.libs.ws.*; - -import java.util.*; - -public class AdvSearchDAO extends AbstractMySQLOpenSourceDAO -{ - public final static String GET_DATASET_SOURCES = "SELECT source " + - "FROM dict_dataset GROUP BY 1 ORDER BY count(*) DESC"; - - public final static String GET_FLOW_APPCODES = "SELECT DISTINCT app_code " + - "FROM cfg_application GROUP BY 1 ORDER BY 1"; - - public final static String GET_DATASET_SCOPES = "SELECT DISTINCT parent_name " + - "FROM dict_dataset WHERE parent_name is not null order by 1;"; - - public final static String GET_DATASET_TABLE_NAMES_BY_SCOPE = "SELECT DISTINCT name " + - "FROM dict_dataset WHERE parent_name in (:scopes)"; - - public final static String GET_FLOW_NAMES_BY_APP = "SELECT DISTINCT f.flow_name " + - "FROM flow f JOIN cfg_application a on f.app_id = a.app_id WHERE app in (:apps)"; - - public final static String GET_DATASET_TABLE_NAMES = "SELECT DISTINCT name FROM dict_dataset ORDER BY 1"; - - public final static String GET_FLOW_NAMES = "SELECT DISTINCT flow_name FROM flow ORDER BY 1"; - - public final static String GET_JOB_NAMES = "SELECT DISTINCT job_name " + - "FROM flow_job GROUP BY 1 ORDER BY 1"; - - public final static String GET_DASHBOARD_NAMES = "SELECT DISTINCT dashboard_name " + - "FROM dict_business_metric WHERE dashboard_name is not null and dashboard_name != '' ORDER BY 1"; - - public final static String GET_METRIC_GROUPS = "SELECT DISTINCT metric_group " + - "FROM dict_business_metric WHERE metric_group is not null and metric_group != '' ORDER BY 1"; - - public final static String GET_METRIC_CATEGORIES = "SELECT DISTINCT metric_category " + - "FROM dict_business_metric WHERE metric_category is not null and metric_category != '' ORDER BY 1"; - - public final static String GET_METRIC_NAMES = "SELECT DISTINCT metric_name " + - "FROM dict_business_metric WHERE metric_name is not null and metric_name != '' ORDER BY 1"; - - public final static String GET_DATASET_FIELDS = "SELECT DISTINCT field_name " + - "FROM dict_field_detail ORDER BY 1"; - - public final static String GET_DATASET_FIELDS_BY_TABLE_NAMES = "SELECT DISTINCT f.field_name " + - "FROM dict_field_detail f join dict_dataset d on f.dataset_id = d.id where d.name regexp"; - - public final static String SEARCH_DATASETS_BY_COMMENTS_WITH_PAGINATION = "SELECT SQL_CALC_FOUND_ROWS " + - "id, name, source, urn, `schema` FROM dict_dataset where id in ( " + - "SELECT dataset_id FROM comments WHERE MATCH(text) against ('*$keyword*' in BOOLEAN MODE) ) " + - "UNION ALL SELECT id, name, source, urn, `schema` from dict_dataset " + - "WHERE id in ( SELECT DISTINCT dataset_id FROM " + - "dict_dataset_field_comment WHERE comment_id in " + - "(SELECT id FROM field_comments where MATCH(comment) against ('*$keyword*' in BOOLEAN MODE))) " + - "ORDER BY 2 LIMIT ?, ?"; - - public final static String ADVSEARCH_RANK_CLAUSE = " ORDER BY CASE WHEN $condition1 THEN 0 " + - "WHEN $condition2 THEN 2 WHEN $condition3 THEN 3 WHEN $condition4 THEN 4 ELSE 9 END, " + - "CASE WHEN urn LIKE 'teradata://DWH_%' THEN 2 WHEN urn LIKE 'hdfs://data/tracking/%' THEN 1 " + - "WHEN urn LIKE 'teradata://DWH/%' THEN 3 WHEN urn LIKE 'hdfs://data/databases/%' THEN 4 " + - "WHEN urn LIKE 'hdfs://data/dervied/%' THEN 5 ELSE 99 END, urn"; - - public final static String DATASET_BY_COMMENT_PAGINATION_IN_CLAUSE = "SELECT SQL_CALC_FOUND_ROWS " + - "id, name, source, `schema`, urn, FROM_UNIXTIME(source_modified_time) as modified " + - "FROM dict_dataset WHERE id IN ( " + - "SELECT dataset_id FROM comments WHERE MATCH(text) " + - "AGAINST ('*$keyword*' in BOOLEAN MODE) and dataset_id in ($id_list)) " + - "UNION ALL SELECT id, name, source, `schema`, urn, FROM_UNIXTIME(source_modified_time) as modified " + - "FROM dict_dataset WHERE id IN (SELECT DISTINCT dataset_id FROM " + - "dict_dataset_field_comment WHERE comment_id in " + - "(SELECT id FROM field_comments where MATCH(comment) against ('*$keyword*' in BOOLEAN MODE)) ) " + - "ORDER BY 2 LIMIT ?, ?;"; - - public final static String ADV_SEARCH_FLOW = "SELECT SQL_CALC_FOUND_ROWS " + - "a.app_code, f.flow_id, f.flow_name, f.flow_path, f.flow_group FROM flow f " + - "JOIN cfg_application a on f.app_id = a.app_id "; - - public final static String ADV_SEARCH_JOB = "SELECT SQL_CALC_FOUND_ROWS " + - "a.app_code, f.flow_name, f.flow_path, f.flow_group, j.flow_id, j.job_id, " + - "j.job_name, j.job_path, j.job_type " + - "FROM flow_job j JOIN flow f on j.app_id = f.app_id AND j.flow_id = f.flow_id " + - "JOIN cfg_application a on j.app_id = a.app_id "; - - public final static String ADV_SEARCH_METRIC = "SELECT SQL_CALC_FOUND_ROWS metric_id, " + - "metric_name, metric_description, dashboard_name, metric_group, metric_category, " + - "metric_sub_category, metric_level, metric_source_type, metric_source, " + - "metric_source_dataset_id, metric_ref_id_type, metric_ref_id, metric_type, metric_grain, " + - "metric_display_factor, metric_display_factor_sym, metric_good_direction, " + - "metric_formula, dimensions, owners, tags, urn, metric_url, wiki_url, scm_url, 0 as watch_id " + - "FROM dict_business_metric "; - - public static List getDatasetSources() - { - return getJdbcTemplate().queryForList(GET_DATASET_SOURCES, String.class); - } - - public static List getDatasetScopes() - { - return getJdbcTemplate().queryForList(GET_DATASET_SCOPES, String.class); - } - - public static List getTableNames(String scopes) - { - List tables = null; - if (StringUtils.isNotBlank(scopes)) - { - String[] scopeArray = scopes.split(","); - List scopeList = Arrays.asList(scopeArray); - Map param = Collections.singletonMap("scopes", scopeList); - NamedParameterJdbcTemplate namedParameterJdbcTemplate = new - NamedParameterJdbcTemplate(getJdbcTemplate().getDataSource()); - tables = namedParameterJdbcTemplate.queryForList( - GET_DATASET_TABLE_NAMES_BY_SCOPE, param, String.class); - } - else - { - tables = getJdbcTemplate().queryForList(GET_DATASET_TABLE_NAMES, String.class); - } - - return tables; - } - - public static List getFields(String tables) - { - String query = null; - if (StringUtils.isNotBlank(tables)) - { - String[] tableArray = tables.split(","); - query = GET_DATASET_FIELDS_BY_TABLE_NAMES; - query += "'"; - for(int i = 0; i < tableArray.length; i++) - { - if (i == 0) - { - query += tableArray[i]; - } - else - { - query += "|" + tableArray[i]; - } - } - query += "' order by 1"; - } - else - { - query = GET_DATASET_FIELDS; - } - - return getJdbcTemplate().queryForList(query, String.class); - } - - public static List getFlowApplicationCodes() - { - return getJdbcTemplate().queryForList(GET_FLOW_APPCODES, String.class); - } - - public static List getFlowNames(String applications) - { - List flowNames = null; - if (StringUtils.isNotBlank(applications)) - { - String[] appArray = applications.split(","); - List appList = Arrays.asList(appArray); - Map param = Collections.singletonMap("apps", appList); - NamedParameterJdbcTemplate namedParameterJdbcTemplate = new - NamedParameterJdbcTemplate(getJdbcTemplate().getDataSource()); - flowNames = namedParameterJdbcTemplate.queryForList( - GET_FLOW_NAMES_BY_APP, param, String.class); - } - else - { - flowNames = getJdbcTemplate().queryForList(GET_FLOW_NAMES, String.class); - } - - return flowNames; - } - - public static List getFlowJobNames() - { - return getJdbcTemplate().queryForList(GET_JOB_NAMES, String.class); - } - - public static ObjectNode elasticSearch(JsonNode searchOpt, int page, int size) - { - ObjectNode resultNode = Json.newObject(); - Long count = 0L; - List pagedDatasets = new ArrayList<>(); - ObjectNode queryNode = Json.newObject(); - queryNode.put("from", (page-1)*size); - queryNode.put("size", size); - - JsonNode searchNode = utils.Search.generateDatasetAdvSearchQueryString(searchOpt); - - if (searchNode != null && searchNode.isContainerNode()) - { - queryNode.set("query", searchNode); - } - - Logger.info(" === AdvSearchDAO::elasticSearch === The query sent to Elastic Search is: " + queryNode.toString()); - - Promise responsePromise = WS.url( - Play.application().configuration().getString( - SearchDAO.ELASTICSEARCH_DATASET_URL_KEY)).post(queryNode); - JsonNode responseNode = responsePromise.get(1000).asJson(); - - resultNode.put("page", page); - resultNode.put("category", "Datasets"); - resultNode.put("itemsPerPage", size); - - if (responseNode != null && responseNode.isContainerNode() && responseNode.has("hits")) { - JsonNode hitsNode = responseNode.get("hits"); - if (hitsNode != null) { - if (hitsNode.has("total")) { - count = hitsNode.get("total").asLong(); - } - if (hitsNode.has("hits")) { - JsonNode dataNode = hitsNode.get("hits"); - if (dataNode != null && dataNode.isArray()) { - Iterator arrayIterator = dataNode.elements(); - if (arrayIterator != null) { - while (arrayIterator.hasNext()) { - JsonNode node = arrayIterator.next(); - if (node.isContainerNode() && node.has("_id")) { - Dataset dataset = new Dataset(); - dataset.id = node.get("_id").asLong(); - if (node.has("_source")) { - JsonNode sourceNode = node.get("_source"); - if (sourceNode != null) { - if (sourceNode.has("name")) { - dataset.name = sourceNode.get("name").asText(); - } - if (sourceNode.has("source")) { - dataset.source = sourceNode.get("source").asText(); - } - if (sourceNode.has("urn")) { - dataset.urn = sourceNode.get("urn").asText(); - } - if (sourceNode.has("schema")) { - dataset.schema = sourceNode.get("schema").asText(); - } - } - } - pagedDatasets.add(dataset); - } - } - } - } - } - } - } - resultNode.put("count", count); - resultNode.put("totalPages", (int)Math.ceil(count/((double)size))); - resultNode.set("data", Json.toJson(pagedDatasets)); - return resultNode; - } - - public static ObjectNode elasticSearchMetric(JsonNode searchOpt, int page, int size) - { - ObjectNode resultNode = Json.newObject(); - Long count = 0L; - List pagedMetrics = new ArrayList<>(); - ObjectNode queryNode = Json.newObject(); - queryNode.put("from", (page-1)*size); - queryNode.put("size", size); - - JsonNode searchNode = utils.Search.generateMetricAdvSearchQueryString(searchOpt); - - if (searchNode != null && searchNode.isContainerNode()) - { - queryNode.set("query", searchNode); - } - - Logger.info(" === AdvSearchDAO::elasticSearchMetric === The query sent to Elastic Search is: " + queryNode.toString()); - - Promise responsePromise = WS.url(Play.application().configuration().getString( - SearchDAO.ELASTICSEARCH_METRIC_URL_KEY)).post(queryNode); - JsonNode responseNode = responsePromise.get(1000).asJson(); - - resultNode.put("page", page); - resultNode.put("category", "Metrics"); - resultNode.put("isMetrics", true); - resultNode.put("itemsPerPage", size); - - if (responseNode != null && responseNode.isContainerNode() && responseNode.has("hits")) { - JsonNode hitsNode = responseNode.get("hits"); - if (hitsNode != null) { - if (hitsNode.has("total")) { - count = hitsNode.get("total").asLong(); - } - if (hitsNode.has("hits")) { - JsonNode dataNode = hitsNode.get("hits"); - if (dataNode != null && dataNode.isArray()) { - Iterator arrayIterator = dataNode.elements(); - if (arrayIterator != null) { - while (arrayIterator.hasNext()) { - JsonNode node = arrayIterator.next(); - if (node.isContainerNode() && node.has("_id")) { - Metric metric = new Metric(); - metric.id = node.get("_id").asInt(); - if (node.has("_source")) { - JsonNode sourceNode = node.get("_source"); - if (sourceNode != null) { - if (sourceNode.has("metric_name")) { - metric.name = sourceNode.get("metric_name").asText(); - } - if (sourceNode.has("metric_description")) { - metric.description = sourceNode.get("metric_description").asText(); - } - if (sourceNode.has("dashboard_name")) { - metric.dashboardName = sourceNode.get("dashboard_name").asText(); - } - if (sourceNode.has("metric_group")) { - metric.group = sourceNode.get("metric_group").asText(); - } - if (sourceNode.has("metric_category")) { - metric.category = sourceNode.get("metric_category").asText(); - } - if (sourceNode.has("urn")) { - metric.urn = sourceNode.get("urn").asText(); - } - if (sourceNode.has("metric_source")) { - metric.source = sourceNode.get("metric_source").asText(); - if (StringUtils.isBlank(metric.source)) - { - metric.source = null; - } - } - metric.schema = sourceNode.toString(); - } - } - pagedMetrics.add(metric); - } - } - } - } - } - } - } - resultNode.put("count", count); - resultNode.put("totalPages", (int)Math.ceil(count/((double)size))); - resultNode.set("data", Json.toJson(pagedMetrics)); - return resultNode; - } - - public static ObjectNode elasticSearchFlowJobs(JsonNode searchOpt, int page, int size) - { - ObjectNode resultNode = Json.newObject(); - Long count = 0L; - List pagedFlows = new ArrayList<>(); - ObjectNode queryNode = Json.newObject(); - queryNode.put("from", (page-1)*size); - queryNode.put("size", size); - - JsonNode searchNode = utils.Search.generateFlowJobAdvSearchQueryString(searchOpt); - - if (searchNode != null && searchNode.isContainerNode()) - { - queryNode.set("query", searchNode); - } - - Logger.info(" === AdvSearchDAO::elasticSearchFlowJobs === The query sent to Elastic Search is: " + queryNode.toString()); - - Promise responsePromise = WS.url(Play.application().configuration().getString( - SearchDAO.ELASTICSEARCH_FLOW_URL_KEY)).post(queryNode); - JsonNode responseNode = responsePromise.get(1000).asJson(); - - resultNode.put("page", page); - resultNode.put("category", "Flows"); - resultNode.put("isFlowJob", true); - resultNode.put("itemsPerPage", size); - - if (responseNode != null && responseNode.isContainerNode() && responseNode.has("hits")) { - JsonNode hitsNode = responseNode.get("hits"); - if (hitsNode != null) { - if (hitsNode.has("total")) { - count = hitsNode.get("total").asLong(); - } - if (hitsNode.has("hits")) { - JsonNode dataNode = hitsNode.get("hits"); - if (dataNode != null && dataNode.isArray()) { - Iterator arrayIterator = dataNode.elements(); - if (arrayIterator != null) { - while (arrayIterator.hasNext()) { - JsonNode node = arrayIterator.next(); - if (node.isContainerNode() && node.has("_id")) { - FlowJob flowJob = new FlowJob(); - if (node.has("_source")) { - JsonNode sourceNode = node.get("_source"); - if (sourceNode != null) { - if (sourceNode.has("app_code")) { - flowJob.appCode = sourceNode.get("app_code").asText(); - } - if (sourceNode.has("app_id")) { - flowJob.appId = sourceNode.get("app_id").asInt(); - } - if (sourceNode.has("flow_id")) { - flowJob.flowId = sourceNode.get("flow_id").asLong(); - } - if (sourceNode.has("flow_name")) { - flowJob.flowName = sourceNode.get("flow_name").asText(); - flowJob.displayName = flowJob.flowName; - } - if (sourceNode.has("flow_path")) { - flowJob.flowPath = sourceNode.get("flow_path").asText(); - } - if (sourceNode.has("flow_group")) { - flowJob.flowGroup = sourceNode.get("flow_group").asText(); - } - flowJob.link = "#/flows/name/" + flowJob.appCode + "/" + - Long.toString(flowJob.flowId) + "/page/1?urn=" + flowJob.flowGroup; - flowJob.path = flowJob.appCode + "/" + flowJob.flowPath; - - flowJob.schema = sourceNode.toString(); - } - } - pagedFlows.add(flowJob); - } - } - } - } - } - } - } - resultNode.put("count", count); - resultNode.put("totalPages", (int)Math.ceil(count/((double)size))); - resultNode.set("data", Json.toJson(pagedFlows)); - return resultNode; - } - public static List getMetricDashboardNames() - { - return getJdbcTemplate().queryForList(GET_DASHBOARD_NAMES, String.class); - } - - public static List getMetricGroups() - { - return getJdbcTemplate().queryForList(GET_METRIC_GROUPS, String.class); - } - - public static List getMetricCategories() - { - return getJdbcTemplate().queryForList(GET_METRIC_CATEGORIES, String.class); - } - - public static List getMetricNames() - { - return getJdbcTemplate().queryForList(GET_METRIC_NAMES, String.class); - } - - public static ObjectNode search(JsonNode searchOpt, int page, int size) - { - ObjectNode resultNode = Json.newObject(); - int count = 0; - List scopeInList = new ArrayList(); - List scopeNotInList = new ArrayList(); - List tableInList = new ArrayList(); - List tableNotInList = new ArrayList(); - List fieldAnyList = new ArrayList(); - List fieldAllList = new ArrayList(); - List fieldNotInList = new ArrayList(); - String fieldAllIDs = ""; - String comments = ""; - - if (searchOpt != null && (searchOpt.isContainerNode())) - { - if (searchOpt.has("scope")) { - JsonNode scopeNode = searchOpt.get("scope"); - if (scopeNode != null && scopeNode.isContainerNode()) - { - if (scopeNode.has("in")) - { - JsonNode scopeInNode = scopeNode.get("in"); - if (scopeInNode != null) - { - String scopeInStr = scopeInNode.asText(); - if (StringUtils.isNotBlank(scopeInStr)) - { - String[] scopeInArray = scopeInStr.split(","); - if (scopeInArray != null) - { - for(String value : scopeInArray) - { - if (StringUtils.isNotBlank(value)) - { - scopeInList.add(value.trim()); - } - } - } - } - } - } - if (scopeNode.has("not")) - { - JsonNode scopeNotInNode = scopeNode.get("not"); - if (scopeNotInNode != null) - { - String scopeNotInStr = scopeNotInNode.asText(); - if (StringUtils.isNotBlank(scopeNotInStr)) - { - String[] scopeNotInArray = scopeNotInStr.split(","); - if (scopeNotInArray != null) - { - for(String value : scopeNotInArray) - { - if (StringUtils.isNotBlank(value)) - { - scopeNotInList.add(value.trim()); - } - } - } - } - } - } - } - } - - if (searchOpt.has("table")) { - JsonNode tableNode = searchOpt.get("table"); - if (tableNode != null && tableNode.isContainerNode()) - { - if (tableNode.has("in")) - { - JsonNode tableInNode = tableNode.get("in"); - if (tableInNode != null) - { - String tableInStr = tableInNode.asText(); - if (StringUtils.isNotBlank(tableInStr)) - { - String[] tableInArray = tableInStr.split(","); - if (tableInArray != null) - { - for(String value : tableInArray) - { - if (StringUtils.isNotBlank(value)) - { - tableInList.add(value.trim()); - } - } - } - } - } - } - if (tableNode.has("not")) - { - JsonNode tableNotInNode = tableNode.get("not"); - if (tableNotInNode != null) - { - String tableNotInStr = tableNotInNode.asText(); - if (StringUtils.isNotBlank(tableNotInStr)) - { - String[] tableNotInArray = tableNotInStr.split(","); - if (tableNotInArray != null) - { - for(String value : tableNotInArray) - { - if (StringUtils.isNotBlank(value)) - { - tableNotInList.add(value.trim()); - } - } - } - } - } - } - } - } - - if (searchOpt.has("fields")) { - JsonNode fieldNode = searchOpt.get("fields"); - if (fieldNode != null && fieldNode.isContainerNode()) - { - if (fieldNode.has("any")) - { - JsonNode fieldAnyNode = fieldNode.get("any"); - if (fieldAnyNode != null) - { - String fieldAnyStr = fieldAnyNode.asText(); - if (StringUtils.isNotBlank(fieldAnyStr)) - { - String[] fieldAnyArray = fieldAnyStr.split(","); - if (fieldAnyArray != null) - { - for(String value : fieldAnyArray) - { - if (StringUtils.isNotBlank(value)) - { - fieldAnyList.add(value.trim()); - } - } - } - } - } - } - if (fieldNode.has("all")) - { - JsonNode fieldAllNode = fieldNode.get("all"); - if (fieldAllNode != null) - { - String fieldAllStr = fieldAllNode.asText(); - if (StringUtils.isNotBlank(fieldAllStr)) - { - String[] fieldAllArray = fieldAllStr.split(","); - if (fieldAllArray != null) - { - for(String value : fieldAllArray) - { - if (StringUtils.isNotBlank(value)) - { - fieldAllList.add(value.trim()); - } - } - } - } - } - } - if (fieldNode.has("not")) - { - JsonNode fieldNotInNode = fieldNode.get("not"); - if (fieldNotInNode != null) - { - String fieldNotInStr = fieldNotInNode.asText(); - if (StringUtils.isNotBlank(fieldNotInStr)) - { - String[] fieldNotInArray = fieldNotInStr.split(","); - if (fieldNotInArray != null) - { - for(String value : fieldNotInArray) - { - if (StringUtils.isNotBlank(value)) - { - fieldNotInList.add(value.trim()); - } - } - } - } - } - } - } - } - - String datasetSources = ""; - if (searchOpt.has("sources")) { - JsonNode sourcesNode = searchOpt.get("sources"); - if (sourcesNode != null) - { - datasetSources = sourcesNode.asText(); - } - } - - boolean needAndKeyword = false; - int fieldQueryIndex = 0; - if (fieldAllList.size() > 0) - { - String fieldAllQuery = "SELECT DISTINCT f1.dataset_id FROM dict_field_detail f1 "; - String fieldWhereClause = " WHERE "; - for (String field : fieldAllList) - { - fieldQueryIndex++; - if (fieldQueryIndex == 1) - { - fieldWhereClause += "f1.field_name LIKE '%" + field + "%' "; - } - else - { - fieldAllQuery += "JOIN dict_field_detail f" + fieldQueryIndex + " ON f" + - (fieldQueryIndex-1) + ".dataset_id = f" + fieldQueryIndex + ".dataset_id "; - fieldWhereClause += " and f" + fieldQueryIndex + ".field_name LIKE '%" + field + "%' "; - - } - } - fieldAllQuery += fieldWhereClause; - List> rows = getJdbcTemplate().queryForList(fieldAllQuery); - for (Map row : rows) { - - fieldAllIDs += (Long)row.get("dataset_id") + ","; - } - if (fieldAllIDs.length() > 0) - { - fieldAllIDs = fieldAllIDs.substring(0, fieldAllIDs.length()-1); - } - if (StringUtils.isBlank(fieldAllIDs)) - { - fieldAllIDs = Integer.toString(0); - - } - } - - List pagedDatasets = new ArrayList(); - final JdbcTemplate jdbcTemplate = getJdbcTemplate(); - javax.sql.DataSource ds = jdbcTemplate.getDataSource(); - DataSourceTransactionManager tm = new DataSourceTransactionManager(ds); - - TransactionTemplate txTemplate = new TransactionTemplate(tm); - - ObjectNode result; - - if (searchOpt.has("comments")) - { - JsonNode commentsNode = searchOpt.get("comments"); - if (commentsNode != null) - { - comments = commentsNode.asText(); - if (StringUtils.isNotBlank(comments)) - { - if (scopeInList.size() == 0 && scopeNotInList.size() == 0 - && tableInList.size() == 0 && tableNotInList.size() == 0 - && fieldAllList.size() == 0 && fieldAnyList.size() == 0 && fieldNotInList.size() == 0) - { - final String commentsQueryStr = - SEARCH_DATASETS_BY_COMMENTS_WITH_PAGINATION.replace("$keyword", comments); - - result = txTemplate.execute(new TransactionCallback() - { - public ObjectNode doInTransaction(TransactionStatus status) - { - List> rows = null; - rows = jdbcTemplate.queryForList(commentsQueryStr, (page-1)*size, size); - - for (Map row : rows) { - - Dataset ds = new Dataset(); - ds.id = (Long)row.get("id"); - ds.name = (String)row.get("name"); - ds.source = (String)row.get("source"); - ds.urn = (String)row.get("urn"); - ds.schema = (String)row.get("schema"); - pagedDatasets.add(ds); - } - long count = 0; - try { - count = jdbcTemplate.queryForObject( - "SELECT FOUND_ROWS()", - Long.class); - } - catch(EmptyResultDataAccessException e) - { - Logger.error("Exception = " + e.getMessage()); - } - - ObjectNode resultNode = Json.newObject(); - resultNode.put("count", count); - resultNode.put("page", page); - resultNode.put("itemsPerPage", size); - resultNode.put("totalPages", (int)Math.ceil(count/((double)size))); - resultNode.set("data", Json.toJson(pagedDatasets)); - - return resultNode; - } - }); - return result; - } - } - } - } - - String query = ""; - if (StringUtils.isNotBlank(comments)) - { - query = "SELECT DISTINCT d.id FROM dict_dataset d"; - } - else - { - query = "SELECT SQL_CALC_FOUND_ROWS " + - "DISTINCT d.id, d.name, d.schema, d.source, d.urn, " + - "FROM_UNIXTIME(d.source_modified_time) as modified FROM dict_dataset d"; - } - if (fieldAllList.size() > 0 || fieldAnyList.size() > 0 || fieldNotInList.size() > 0) - { - String fieldQuery = "SELECT DISTINCT dataset_id FROM dict_field_detail f WHERE ("; - query += " WHERE d.id IN ( "; - query += fieldQuery; - String whereClause = ""; - boolean fieldNeedAndKeyword = false; - if (fieldAnyList.size() > 0) - { - whereClause = " ("; - int indexForAnyList = 0; - for (String field : fieldAnyList) - { - if (indexForAnyList == 0) - { - whereClause += "f.field_name LIKE '%" + field + "%'"; - } - else - { - whereClause += " or f.field_name LIKE '%" + field + "%'"; - } - indexForAnyList++; - } - whereClause += " ) "; - fieldNeedAndKeyword = true; - query += whereClause; - } - if (fieldAllList.size() > 0 && StringUtils.isNotBlank(fieldAllIDs)) - { - if (fieldNeedAndKeyword) - { - whereClause = " and ("; - } - else - { - whereClause = " ("; - } - whereClause += "f.dataset_id IN (" + fieldAllIDs + ")"; - whereClause += " ) "; - query += whereClause; - fieldNeedAndKeyword = true; - } - if (fieldNotInList.size() > 0) - { - if (fieldNeedAndKeyword) - { - whereClause = " and ( f.dataset_id not in (select dataset_id from dict_field_detail where"; - } - else - { - whereClause = " ( f.dataset_id not in (select dataset_id from dict_field_detail where"; - } - int indexForNotInList = 0; - for (String field : fieldNotInList) - { - if (indexForNotInList == 0) - { - whereClause += " field_name LIKE '%" + field + "%'"; - } - else - { - whereClause += " or field_name LIKE '%" + field + "%'"; - } - indexForNotInList++; - } - whereClause += " )) "; - query += whereClause; - fieldNeedAndKeyword = true; - } - needAndKeyword = true; - query += ") )"; - } - - if (scopeInList.size() > 0 || scopeNotInList.size() > 0) - { - if (needAndKeyword) - { - query += " and"; - } - else - { - query += " where"; - } - boolean scopeNeedAndKeyword = false; - if (scopeInList.size() > 0) - { - query += " d.parent_name in ("; - scopeNeedAndKeyword = true; - int indexForScopeInList = 0; - for (String scope : scopeInList) - { - if (indexForScopeInList == 0) - { - query += "'" + scope + "'"; - } - else - { - query += ", '" + scope + "'"; - } - indexForScopeInList++; - } - query += ") "; - } - if (scopeNotInList.size() > 0) - { - if (scopeNeedAndKeyword) - { - query += " and d.parent_name not in ("; - } - else - { - query += " d.parent_name not in ("; - } - int indexForScopeNotInList = 0; - for (String scope : scopeNotInList) - { - if (indexForScopeNotInList == 0) - { - query += "'" + scope + "'"; - } - else - { - query += ", '" + scope + "'"; - } - indexForScopeNotInList++; - } - query += ") "; - } - needAndKeyword = true; - } - String condition1 = ""; - String condition2 = ""; - String condition3 = ""; - String condition4 = ""; - - if (tableInList.size() > 0 || tableNotInList.size() > 0) - { - if (needAndKeyword) - { - query += " and"; - } - else - { - query += " where"; - } - boolean tableNeedAndKeyword = false; - if (tableInList.size() > 0) - { - query += " ("; - int indexForTableInList = 0; - for (String table : tableInList) - { - if (indexForTableInList == 0) - { - query += "d.name LIKE '%" + table + "%'"; - } - else - { - condition1 += " or "; - condition2 += " or "; - condition3 += " or "; - condition4 += " or "; - query += " or d.name LIKE '%" + table + "%'"; - } - condition1 += "name = '" + table + "'"; - condition2 += "name LIKE '" + table + "%'"; - condition3 += "name LIKE '%" + table + "'"; - condition4 += "name LIKE '%" + table + "%'"; - indexForTableInList++; - } - query += " ) "; - tableNeedAndKeyword = true; - } - if (tableNotInList.size() > 0) - { - if (tableNeedAndKeyword) - { - query += " and ("; - } - else - { - query += " ("; - } - int indexForTableNotInList = 0; - for (String table : tableNotInList) - { - if (indexForTableNotInList == 0) - { - query += "d.name NOT LIKE '%" + table + "%'"; - } - else - { - query += " and d.name NOT LIKE '%" + table + "%'"; - } - indexForTableNotInList++; - } - query += " ) "; - } - needAndKeyword = true; - } - - if (StringUtils.isNotBlank(datasetSources)) - { - if (needAndKeyword) - { - query += " and"; - } - else - { - query += " WHERE"; - } - query += " d.source in ("; - String[] dataestSourceArray = datasetSources.split(","); - for(int i = 0; i < dataestSourceArray.length; i++) - { - query += "'" + dataestSourceArray[i] + "'"; - if (i != (dataestSourceArray.length -1)) - { - query += ","; - } - } - query += ")"; - } - if ((tableInList.size() > 0 || tableNotInList.size() > 0) && - StringUtils.isNotBlank(condition1) && - StringUtils.isNotBlank(condition2) && - StringUtils.isNotBlank(condition3) && - StringUtils.isNotBlank(condition4)) - { - query += ADVSEARCH_RANK_CLAUSE.replace("$condition1", condition1) - .replace("$condition2", condition2) - .replace("$condition3", condition3) - .replace("$condition4", condition4); - } - else - { - query += " ORDER BY CASE WHEN urn LIKE 'teradata://DWH_%' THEN 2 " + - "WHEN urn LIKE 'hdfs://data/tracking/%' THEN 1 " + - "WHEN urn LIKE 'teradata://DWH/%' THEN 3 " + - "WHEN urn LIKE 'hdfs://data/databases/%' THEN 4 " + - "WHEN urn LIKE 'hdfs://data/dervied/%' THEN 5 ELSE 99 end, urn"; - } - if (StringUtils.isBlank(comments)) - { - query += " LIMIT " + (page-1)*size + ", " + size; - final String queryString = query; - - result = txTemplate.execute(new TransactionCallback() - { - public ObjectNode doInTransaction(TransactionStatus status) - { - List> rows = null; - rows = jdbcTemplate.queryForList(queryString); - - for (Map row : rows) { - - Dataset ds = new Dataset(); - ds.id = (Long)row.get("id"); - ds.name = (String)row.get("name"); - ds.source = (String)row.get("source"); - ds.urn = (String)row.get("urn"); - ds.schema = (String)row.get("schema"); - pagedDatasets.add(ds); - } - long count = 0; - try { - count = jdbcTemplate.queryForObject( - "SELECT FOUND_ROWS()", - Long.class); - } - catch(EmptyResultDataAccessException e) - { - Logger.error("Exception = " + e.getMessage()); - } - - ObjectNode resultNode = Json.newObject(); - resultNode.put("count", count); - resultNode.put("page", page); - resultNode.put("itemsPerPage", size); - resultNode.put("totalPages", (int)Math.ceil(count/((double)size))); - resultNode.set("data", Json.toJson(pagedDatasets)); - - return resultNode; - } - }); - return result; - } - else - { - String datasetIDStr = ""; - final String queryString = query; - - datasetIDStr = txTemplate.execute(new TransactionCallback() - { - public String doInTransaction(TransactionStatus status) - { - List> rows = null; - rows = jdbcTemplate.queryForList(queryString); - String idsString = ""; - - for (Map row : rows) { - - Long id = (Long)row.get("id"); - idsString += Long.toString(id) + ","; - } - if (StringUtils.isNotBlank(idsString)) - { - idsString = idsString.substring(0, idsString.length()-1); - } - return idsString; - } - }); - if (StringUtils.isBlank(datasetIDStr)) - { - resultNode.put("count", 0); - resultNode.put("page", page); - resultNode.put("itemsPerPage", size); - resultNode.put("totalPages", 0); - resultNode.set("data", Json.toJson("")); - return resultNode; - } - final String commentsQueryWithConditionStr = DATASET_BY_COMMENT_PAGINATION_IN_CLAUSE.replace("$keyword", comments). - replace("$id_list", datasetIDStr); - result = txTemplate.execute(new TransactionCallback() - { - public ObjectNode doInTransaction(TransactionStatus status) - { - List> rows = null; - rows = jdbcTemplate.queryForList(commentsQueryWithConditionStr, (page-1)*size, size); - - for (Map row : rows) { - - Dataset ds = new Dataset(); - ds.id = (Long)row.get("id"); - ds.name = (String)row.get("name"); - ds.source = (String)row.get("source"); - ds.urn = (String)row.get("urn"); - ds.schema = (String)row.get("schema"); - pagedDatasets.add(ds); - } - long count = 0; - try { - count = jdbcTemplate.queryForObject( - "SELECT FOUND_ROWS()", - Long.class); - } - catch(EmptyResultDataAccessException e) - { - Logger.error("Exception = " + e.getMessage()); - } - - ObjectNode resultNode = Json.newObject(); - resultNode.put("count", count); - resultNode.put("page", page); - resultNode.put("itemsPerPage", size); - resultNode.put("totalPages", (int)Math.ceil(count/((double)size))); - resultNode.set("data", Json.toJson(pagedDatasets)); - - return resultNode; - } - }); - return result; - } - } - resultNode.put("count", 0); - resultNode.put("page", page); - resultNode.put("itemsPerPage", size); - resultNode.put("totalPages", 0); - resultNode.set("data", Json.toJson("")); - return resultNode; - } - - public static ObjectNode searchFlows(JsonNode searchOpt, int page, int size) - { - ObjectNode resultNode = Json.newObject(); - int count = 0; - List appcodeInList = new ArrayList(); - List appcodeNotInList = new ArrayList(); - List flowInList = new ArrayList(); - List flowNotInList = new ArrayList(); - List jobInList = new ArrayList(); - List jobNotInList = new ArrayList(); - - if (searchOpt != null && (searchOpt.isContainerNode())) - { - if (searchOpt.has("appcode")) { - JsonNode appcodeNode = searchOpt.get("appcode"); - if (appcodeNode != null && appcodeNode.isContainerNode()) - { - if (appcodeNode.has("in")) - { - JsonNode appcodeInNode = appcodeNode.get("in"); - if (appcodeInNode != null) - { - String appcodeInStr = appcodeInNode.asText(); - if (StringUtils.isNotBlank(appcodeInStr)) - { - String[] appcodeInArray = appcodeInStr.split(","); - if (appcodeInArray != null) - { - for(String value : appcodeInArray) - { - if (StringUtils.isNotBlank(value)) - { - appcodeInList.add(value.trim()); - } - } - } - } - } - } - if (appcodeNode.has("not")) - { - JsonNode appcodeNotInNode = appcodeNode.get("not"); - if (appcodeNotInNode != null) - { - String appcodeNotInStr = appcodeNotInNode.asText(); - if (StringUtils.isNotBlank(appcodeNotInStr)) - { - String[] appcodeNotInArray = appcodeNotInStr.split(","); - if (appcodeNotInArray != null) - { - for(String value : appcodeNotInArray) - { - if (StringUtils.isNotBlank(value)) - { - appcodeNotInList.add(value.trim()); - } - } - } - } - } - } - } - } - - if (searchOpt.has("flow")) { - JsonNode flowNode = searchOpt.get("flow"); - if (flowNode != null && flowNode.isContainerNode()) - { - if (flowNode.has("in")) - { - JsonNode flowInNode = flowNode.get("in"); - if (flowInNode != null) - { - String flowInStr = flowInNode.asText(); - if (StringUtils.isNotBlank(flowInStr)) - { - String[] flowInArray = flowInStr.split(","); - if (flowInArray != null) - { - for(String value : flowInArray) - { - if (StringUtils.isNotBlank(value)) - { - flowInList.add(value.trim()); - } - } - } - } - } - } - if (flowNode.has("not")) - { - JsonNode flowNotInNode = flowNode.get("not"); - if (flowNotInNode != null) - { - String flowNotInStr = flowNotInNode.asText(); - if (StringUtils.isNotBlank(flowNotInStr)) - { - String[] flowNotInArray = flowNotInStr.split(","); - if (flowNotInArray != null) - { - for(String value : flowNotInArray) - { - if (StringUtils.isNotBlank(value)) - { - flowNotInList.add(value.trim()); - } - } - } - } - } - } - } - } - - if (searchOpt.has("job")) { - JsonNode jobNode = searchOpt.get("job"); - if (jobNode != null && jobNode.isContainerNode()) - { - if (jobNode.has("in")) - { - JsonNode jobInNode = jobNode.get("in"); - if (jobInNode != null) - { - String jobInStr = jobInNode.asText(); - if (StringUtils.isNotBlank(jobInStr)) - { - String[] jobInArray = jobInStr.split(","); - if (jobInArray != null) - { - for(String value : jobInArray) - { - if (StringUtils.isNotBlank(value)) - { - jobInList.add(value.trim()); - } - } - } - } - } - } - if (jobNode.has("not")) - { - JsonNode jobNotInNode = jobNode.get("not"); - if (jobNotInNode != null) - { - String jobNotInStr = jobNotInNode.asText(); - if (StringUtils.isNotBlank(jobNotInStr)) - { - String[] jobNotInArray = jobNotInStr.split(","); - if (jobNotInArray != null) - { - for(String value : jobNotInArray) - { - if (StringUtils.isNotBlank(value)) - { - jobNotInList.add(value.trim()); - } - } - } - } - } - } - } - } - - boolean needAndKeyword = false; - - final List pagedFlows = new ArrayList(); - final JdbcTemplate jdbcTemplate = getJdbcTemplate(); - javax.sql.DataSource ds = jdbcTemplate.getDataSource(); - DataSourceTransactionManager tm = new DataSourceTransactionManager(ds); - - TransactionTemplate txTemplate = new TransactionTemplate(tm); - - ObjectNode result; - String query = null; - if (jobInList.size() > 0 || jobNotInList.size() > 0) - { - query = ADV_SEARCH_JOB; - } - else - { - query = ADV_SEARCH_FLOW; - } - - if (appcodeInList.size() > 0 || appcodeNotInList.size() > 0) - { - boolean appcodeNeedAndKeyword = false; - if (appcodeInList.size() > 0) - { - int indexForAppcodeInList = 0; - for (String appcode : appcodeInList) - { - if (indexForAppcodeInList == 0) - { - query += "WHERE a.app_code in ('" + appcode + "'"; - } - else - { - query += ", '" + appcode + "'"; - } - indexForAppcodeInList++; - } - query += ") "; - appcodeNeedAndKeyword = true; - } - if (appcodeNotInList.size() > 0) - { - if (appcodeNeedAndKeyword) - { - query += " AND "; - } - else - { - query += " WHERE "; - } - int indexForAppcodeNotInList = 0; - for (String appcode : appcodeNotInList) - { - if (indexForAppcodeNotInList == 0) - { - query += "a.app_code not in ('" + appcode + "'"; - } - else - { - query += ", '" + appcode + "'"; - } - indexForAppcodeNotInList++; - } - query += ") "; - } - needAndKeyword = true; - } - - if (flowInList.size() > 0 || flowNotInList.size() > 0) - { - if (needAndKeyword) - { - query += " AND "; - } - else - { - query += " WHERE "; - } - boolean flowNeedAndKeyword = false; - if (flowInList.size() > 0) - { - query += "( "; - int indexForFlowInList = 0; - for (String flow : flowInList) - { - if (indexForFlowInList == 0) - { - query += "f.flow_name LIKE '%" + flow + "%'"; - } - else - { - query += " or f.flow_name LIKE '%" + flow + "%'"; - } - indexForFlowInList++; - } - query += ") "; - flowNeedAndKeyword = true; - } - if (flowNotInList.size() > 0) - { - if (flowNeedAndKeyword) - { - query += " AND "; - } - query += "( "; - int indexForFlowNotInList = 0; - for (String flow : flowNotInList) - { - if (indexForFlowNotInList == 0) - { - query += "f.flow_name NOT LIKE '%" + flow + "%'"; - } - else - { - query += " and f.flow_name NOT LIKE '%" + flow + "%'"; - } - indexForFlowNotInList++; - } - query += ") "; - } - needAndKeyword = true; - } - - if (jobInList.size() > 0 || jobNotInList.size() > 0) - { - if (needAndKeyword) - { - query += " AND "; - } - else - { - query += " WHERE "; - } - query += "( "; - boolean jobNeedAndKeyword = false; - if (jobInList.size() > 0) - { - query += "( "; - int indexForJobInList = 0; - for (String job : jobInList) - { - if (indexForJobInList == 0) - { - query += "j.job_name LIKE '%" + job + "%'"; - } - else - { - query += " or j.job_name LIKE '%" + job + "%'"; - } - indexForJobInList++; - } - query += ") "; - jobNeedAndKeyword = true; - } - if (jobNotInList.size() > 0) - { - if (jobNeedAndKeyword) - { - query += " AND "; - } - query += "( "; - int indexForJobNotInList = 0; - for (String job : jobNotInList) - { - if (indexForJobNotInList == 0) - { - query += "j.job_name NOT LIKE '%" + job + "%'"; - } - else - { - query += " and j.job_name NOT LIKE '%" + job + "%'"; - } - indexForJobNotInList++; - } - query += ") "; - } - query += " ) "; - } - - query += " LIMIT " + (page-1)*size + ", " + size; - final String queryString = query; - - result = txTemplate.execute(new TransactionCallback() - { - public ObjectNode doInTransaction(TransactionStatus status) - { - List> rows = null; - rows = jdbcTemplate.queryForList(queryString); - - for (Map row : rows) { - - FlowJob flow = new FlowJob(); - flow.appCode = (String)row.get("app_code"); - flow.flowName = (String)row.get("flow_name"); - flow.flowPath = (String)row.get("flow_path"); - flow.flowGroup = (String)row.get("flow_group"); - flow.jobName = (String)row.get("job_name"); - flow.jobPath = (String)row.get("job_path"); - flow.flowId = (Long)row.get("flow_id"); - if (StringUtils.isNotBlank(flow.jobName)) - { - flow.displayName = flow.jobName; - } - else - { - flow.displayName = flow.flowName; - } - flow.link = "#/flows/name/" + flow.appCode + "/" + - Long.toString(flow.flowId) + "/page/1?urn=" + flow.flowGroup; - flow.path = flow.appCode + "/" + flow.flowPath; - pagedFlows.add(flow); - } - long count = 0; - try { - count = jdbcTemplate.queryForObject( - "SELECT FOUND_ROWS()", - Long.class); - } - catch(EmptyResultDataAccessException e) - { - Logger.error("Exception = " + e.getMessage()); - } - - ObjectNode resultNode = Json.newObject(); - resultNode.put("count", count); - resultNode.put("page", page); - resultNode.put("isFlowJob", true); - resultNode.put("itemsPerPage", size); - resultNode.put("totalPages", (int)Math.ceil(count/((double)size))); - resultNode.set("data", Json.toJson(pagedFlows)); - - return resultNode; - } - }); - return result; - } - resultNode.put("count", 0); - resultNode.put("page", page); - resultNode.put("itemsPerPage", size); - resultNode.put("totalPages", 0); - resultNode.set("data", Json.toJson("")); - return resultNode; - } - - public static ObjectNode searchMetrics(JsonNode searchOpt, int page, int size) - { - ObjectNode resultNode = Json.newObject(); - int count = 0; - List dashboardInList = new ArrayList(); - List dashboardNotInList = new ArrayList(); - List groupInList = new ArrayList(); - List groupNotInList = new ArrayList(); - List categoryInList = new ArrayList(); - List categoryNotInList = new ArrayList(); - List metricInList = new ArrayList(); - List metricNotInList = new ArrayList(); - - if (searchOpt != null && (searchOpt.isContainerNode())) - { - if (searchOpt.has("dashboard")) { - JsonNode dashboardNode = searchOpt.get("dashboard"); - if (dashboardNode != null && dashboardNode.isContainerNode()) - { - if (dashboardNode.has("in")) - { - JsonNode dashboardInNode = dashboardNode.get("in"); - if (dashboardInNode != null) - { - String dashboardInStr = dashboardInNode.asText(); - if (StringUtils.isNotBlank(dashboardInStr)) - { - String[] dashboardInArray = dashboardInStr.split(","); - if (dashboardInArray != null) - { - for(String value : dashboardInArray) - { - if (StringUtils.isNotBlank(value)) - { - dashboardInList.add(value.trim()); - } - } - } - } - } - } - if (dashboardNode.has("not")) - { - JsonNode dashboardNotInNode = dashboardNode.get("not"); - if (dashboardNotInNode != null) - { - String dashboardNotInStr = dashboardNotInNode.asText(); - if (StringUtils.isNotBlank(dashboardNotInStr)) - { - String[] dashboardNotInArray = dashboardNotInStr.split(","); - if (dashboardNotInArray != null) - { - for(String value : dashboardNotInArray) - { - if (StringUtils.isNotBlank(value)) - { - dashboardNotInList.add(value.trim()); - } - } - } - } - } - } - } - } - - if (searchOpt.has("group")) { - JsonNode groupNode = searchOpt.get("group"); - if (groupNode != null && groupNode.isContainerNode()) - { - if (groupNode.has("in")) - { - JsonNode groupInNode = groupNode.get("in"); - if (groupInNode != null) - { - String groupInStr = groupInNode.asText(); - if (StringUtils.isNotBlank(groupInStr)) - { - String[] groupInArray = groupInStr.split(","); - if (groupInArray != null) - { - for(String value : groupInArray) - { - if (StringUtils.isNotBlank(value)) - { - groupInList.add(value.trim()); - } - } - } - } - } - } - if (groupNode.has("not")) - { - JsonNode groupNotInNode = groupNode.get("not"); - if (groupNotInNode != null) - { - String groupNotInStr = groupNotInNode.asText(); - if (StringUtils.isNotBlank(groupNotInStr)) - { - String[] groupNotInArray = groupNotInStr.split(","); - if (groupNotInArray != null) - { - for(String value : groupNotInArray) - { - if (StringUtils.isNotBlank(value)) - { - groupNotInList.add(value.trim()); - } - } - } - } - } - } - } - } - - if (searchOpt.has("cat")) { - JsonNode categoryNode = searchOpt.get("cat"); - if (categoryNode != null && categoryNode.isContainerNode()) - { - if (categoryNode.has("in")) - { - JsonNode categoryInNode = categoryNode.get("in"); - if (categoryInNode != null) - { - String categoryInStr = categoryInNode.asText(); - if (StringUtils.isNotBlank(categoryInStr)) - { - String[] categoryInArray = categoryInStr.split(","); - if (categoryInArray != null) - { - for(String value : categoryInArray) - { - if (StringUtils.isNotBlank(value)) - { - categoryInList.add(value.trim()); - } - } - } - } - } - } - if (categoryNode.has("not")) - { - JsonNode categoryNotInNode = categoryNode.get("not"); - if (categoryNotInNode != null) - { - String categoryNotInStr = categoryNotInNode.asText(); - if (StringUtils.isNotBlank(categoryNotInStr)) - { - String[] categoryNotInArray = categoryNotInStr.split(","); - if (categoryNotInArray != null) - { - for(String value : categoryNotInArray) - { - if (StringUtils.isNotBlank(value)) - { - categoryNotInList.add(value.trim()); - } - } - } - } - } - } - } - } - - if (searchOpt.has("metric")) { - JsonNode metricNode = searchOpt.get("metric"); - if (metricNode != null && metricNode.isContainerNode()) - { - if (metricNode.has("in")) - { - JsonNode metricInNode = metricNode.get("in"); - if (metricInNode != null) - { - String metricInStr = metricInNode.asText(); - if (StringUtils.isNotBlank(metricInStr)) - { - String[] metricInArray = metricInStr.split(","); - if (metricInArray != null) - { - for(String value : metricInArray) - { - if (StringUtils.isNotBlank(value)) - { - metricInList.add(value.trim()); - } - } - } - } - } - } - if (metricNode.has("not")) - { - JsonNode metricNotInNode = metricNode.get("not"); - if (metricNotInNode != null) - { - String metricNotInStr = metricNotInNode.asText(); - if (StringUtils.isNotBlank(metricNotInStr)) - { - String[] metricNotInArray = metricNotInStr.split(","); - if (metricNotInArray != null) - { - for(String value : metricNotInArray) - { - if (StringUtils.isNotBlank(value)) - { - metricNotInList.add(value.trim()); - } - } - } - } - } - } - } - } - - boolean needAndKeyword = false; - - final List pagedMetrics = new ArrayList(); - final JdbcTemplate jdbcTemplate = getJdbcTemplate(); - javax.sql.DataSource ds = jdbcTemplate.getDataSource(); - DataSourceTransactionManager tm = new DataSourceTransactionManager(ds); - - TransactionTemplate txTemplate = new TransactionTemplate(tm); - - ObjectNode result; - String query = ADV_SEARCH_METRIC; - - if (dashboardInList.size() > 0 || dashboardNotInList.size() > 0) - { - boolean dashboardNeedAndKeyword = false; - if (dashboardInList.size() > 0) - { - int indexForDashboardInList = 0; - for (String dashboard : dashboardInList) - { - if (indexForDashboardInList == 0) - { - query += "WHERE dashboard_name in ('" + dashboard + "'"; - } - else - { - query += ", '" + dashboard + "'"; - } - indexForDashboardInList++; - } - query += ") "; - dashboardNeedAndKeyword = true; - } - if (dashboardNotInList.size() > 0) - { - if (dashboardNeedAndKeyword) - { - query += " AND "; - } - else - { - query += " WHERE "; - } - int indexForDashboardNotInList = 0; - for (String dashboard : dashboardNotInList) - { - if (indexForDashboardNotInList == 0) - { - query += "dashboard_name not in ('" + dashboard + "'"; - } - else - { - query += ", '" + dashboard + "'"; - } - indexForDashboardNotInList++; - } - query += ") "; - } - needAndKeyword = true; - } - - if (groupInList.size() > 0 || groupNotInList.size() > 0) - { - if (needAndKeyword) - { - query += " AND "; - } - else - { - query += " WHERE "; - } - query += "( "; - boolean groupNeedAndKeyword = false; - if (groupInList.size() > 0) - { - query += "( "; - int indexForGroupInList = 0; - for (String group : groupInList) - { - if (indexForGroupInList == 0) - { - query += "metric_group LIKE '%" + group + "%'"; - } - else - { - query += " or metric_group LIKE '%" + group + "%'"; - } - indexForGroupInList++; - } - query += ") "; - groupNeedAndKeyword = true; - } - if (groupNotInList.size() > 0) - { - if (groupNeedAndKeyword) - { - query += " AND "; - } - query += "( "; - int indexForGroupNotInList = 0; - for (String group : groupNotInList) - { - if (indexForGroupNotInList == 0) - { - query += "metric_group NOT LIKE '%" + group + "%'"; - } - else - { - query += " and metric_group NOT LIKE '%" + group + "%'"; - } - indexForGroupNotInList++; - } - query += ") "; - } - query += ") "; - needAndKeyword = true; - } - - if (categoryInList.size() > 0 || categoryNotInList.size() > 0) - { - if (needAndKeyword) - { - query += " AND "; - } - else - { - query += " WHERE "; - } - query += "( "; - boolean categoryNeedAndKeyword = false; - if (categoryInList.size() > 0) - { - int indexForCategoryInList = 0; - query += "( "; - for (String category : categoryInList) - { - if (indexForCategoryInList == 0) - { - query += "metric_category LIKE '%" + category + "%'"; - } - else - { - query += " or metric_category LIKE '%" + category + "%'"; - } - indexForCategoryInList++; - } - query += ") "; - categoryNeedAndKeyword = true; - } - if (categoryNotInList.size() > 0) - { - if (categoryNeedAndKeyword) - { - query += " AND "; - } - query += "( "; - int indexForCategoryNotInList = 0; - for (String category : categoryNotInList) - { - if (indexForCategoryNotInList == 0) - { - query += "metric_category NOT LIKE '%" + category + "%'"; - } - else - { - query += " and metric_category NOT LIKE '%" + category + "%'"; - } - indexForCategoryNotInList++; - } - query += ") "; - } - query += ") "; - needAndKeyword = true; - } - - if (metricInList.size() > 0 || metricNotInList.size() > 0) - { - if (needAndKeyword) - { - query += " AND "; - } - else - { - query += " WHERE "; - } - query += "( "; - boolean metricNeedAndKeyword = false; - if (metricInList.size() > 0) - { - int indexForMetricInList = 0; - query += " ( "; - for (String metric : metricInList) - { - if (indexForMetricInList == 0) - { - query += "metric_name LIKE '%" + metric + "%'"; - } - else - { - query += " or metric_name LIKE '%" + metric + "%'"; - } - indexForMetricInList++; - } - query += ") "; - metricNeedAndKeyword = true; - } - if (metricNotInList.size() > 0) - { - if (metricNeedAndKeyword) - { - query += " AND "; - } - query += "( "; - int indexForMetricNotInList = 0; - for (String metric : metricNotInList) - { - if (indexForMetricNotInList == 0) - { - query += "metric_name NOT LIKE '%" + metric + "%'"; - } - else - { - query += " and metric_name NOT LIKE '%" + metric + "%'"; - } - indexForMetricNotInList++; - } - query += ") "; - } - query += " )"; - } - - query += " LIMIT " + (page-1)*size + ", " + size; - final String queryString = query; - - result = txTemplate.execute(new TransactionCallback() - { - public ObjectNode doInTransaction(TransactionStatus status) - { - List pagedMetrics = jdbcTemplate.query(queryString, new MetricRowMapper()); - - long count = 0; - try { - count = jdbcTemplate.queryForObject( - "SELECT FOUND_ROWS()", - Long.class); - } - catch(EmptyResultDataAccessException e) - { - Logger.error("Exception = " + e.getMessage()); - } - - ObjectNode resultNode = Json.newObject(); - resultNode.put("count", count); - resultNode.put("page", page); - resultNode.put("isMetrics", true); - resultNode.put("itemsPerPage", size); - resultNode.put("totalPages", (int)Math.ceil(count/((double)size))); - resultNode.set("data", Json.toJson(pagedMetrics)); - - return resultNode; - } - }); - return result; - } - resultNode.put("count", 0); - resultNode.put("page", page); - resultNode.put("itemsPerPage", size); - resultNode.put("totalPages", 0); - resultNode.set("data", Json.toJson("")); - return resultNode; - } - -} diff --git a/wherehows-frontend/app/dao/SearchDAO.java b/wherehows-frontend/app/dao/SearchDAO.java deleted file mode 100644 index be6cd6c01e..0000000000 --- a/wherehows-frontend/app/dao/SearchDAO.java +++ /dev/null @@ -1,1052 +0,0 @@ -/** - * Copyright 2015 LinkedIn Corp. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - */ -package dao; - -import java.util.*; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import org.apache.commons.lang3.StringUtils; -import org.springframework.dao.EmptyResultDataAccessException; -import org.springframework.jdbc.core.JdbcTemplate; -import org.springframework.jdbc.datasource.DataSourceTransactionManager; -import org.springframework.transaction.TransactionStatus; -import org.springframework.transaction.support.TransactionCallback; -import org.springframework.transaction.support.TransactionTemplate; -import play.Logger; -import play.Play; -import play.libs.F.Promise; -import play.libs.Json; -import play.libs.ws.*; -import wherehows.models.table.Dataset; -import wherehows.models.table.FlowJob; -import wherehows.models.table.Metric; - - -public class SearchDAO extends AbstractMySQLOpenSourceDAO -{ - public static String ELASTICSEARCH_DATASET_URL_KEY = "elasticsearch.dataset.url"; - - public static String ELASTICSEARCH_METRIC_URL_KEY = "elasticsearch.metric.url"; - - public static String ELASTICSEARCH_FLOW_URL_KEY = "elasticsearch.flow.url"; - - public static String WHEREHOWS_SEARCH_ENGINE_KEY = "search.engine"; - - private final static String SEARCH_DATASET_WITH_PAGINATION = "SELECT SQL_CALC_FOUND_ROWS " + - "id, `name`, `schema`, `source`, `urn`, FROM_UNIXTIME(source_modified_time) as modified, " + - "rank_01 + rank_02 + rank_03 + rank_04 + rank_05 + rank_06 + rank_07 + rank_08 + rank_09 as rank " + - "FROM (SELECT id, `name`, `schema`, `source`, `urn`, source_modified_time, " + - "CASE WHEN `name` = '$keyword' THEN 3000 ELSE 0 END rank_01, " + - "CASE WHEN `name` like '$keyword%' THEN 2000 ELSE 0 END rank_02, " + - "CASE WHEN `name` like '%$keyword%' THEN 1000 ELSE 0 END rank_03, " + - "CASE WHEN `urn` = '$keyword' THEN 300 ELSE 0 END rank_04, " + - "CASE WHEN `urn` like '$keyword%' THEN 200 ELSE 0 END rank_05, " + - "CASE WHEN `urn` like '%$keyword%' THEN 100 ELSE 0 END rank_06, " + - "CASE WHEN `schema` = '$keyword' THEN 30 ELSE 0 END rank_07, " + - "CASE WHEN `schema` like '$keyword%' THEN 20 ELSE 0 END rank_08, " + - "CASE WHEN `schema` like '%$keyword%' THEN 10 ELSE 0 END rank_09 " + - "FROM dict_dataset WHERE MATCH(`name`, `schema`, `properties`, `urn`)" + - " AGAINST ('*$keyword* *v_$keyword*' IN BOOLEAN MODE) ) t " + - "ORDER BY rank DESC, `name`, `urn` LIMIT ?, ?;"; - - private final static String SEARCH_DATASET_BY_SOURCE_WITH_PAGINATION = "SELECT SQL_CALC_FOUND_ROWS " + - "id, `name`, `schema`, `source`, `urn`, FROM_UNIXTIME(source_modified_time) as modified, " + - "rank_01 + rank_02 + rank_03 + rank_04 + rank_05 + rank_06 + rank_07 + rank_08 + rank_09 as rank " + - "FROM (SELECT id, `name`, `schema`, `source`, `urn`, source_modified_time, " + - "CASE WHEN `name` = '$keyword' THEN 3000 ELSE 0 END rank_01, " + - "CASE WHEN `name` like '$keyword%' THEN 2000 ELSE 0 END rank_02, " + - "CASE WHEN `name` like '%$keyword%' THEN 1000 ELSE 0 END rank_03, " + - "CASE WHEN `urn` = '$keyword' THEN 300 ELSE 0 END rank_04, " + - "CASE WHEN `urn` like '$keyword%' THEN 200 ELSE 0 END rank_05, " + - "CASE WHEN `urn` like '%$keyword%' THEN 100 ELSE 0 END rank_06, " + - "CASE WHEN `schema` = '$keyword' THEN 30 ELSE 0 END rank_07, " + - "CASE WHEN `schema` like '$keyword%' THEN 20 ELSE 0 END rank_08, " + - "CASE WHEN `schema` like '%$keyword%' THEN 10 ELSE 0 END rank_09 " + - "FROM dict_dataset WHERE MATCH(`name`, `schema`, `properties`, `urn`)" + - " AGAINST ('*$keyword* *v_$keyword*' IN BOOLEAN MODE) and source = ? ) t " + - "ORDER BY rank desc, `name`, `urn` LIMIT ?, ?;"; - - private final static String SEARCH_FLOW_WITH_PAGINATION = "SELECT SQL_CALC_FOUND_ROWS " + - "a.app_code, f.app_id, f.flow_id, f.flow_name, f.flow_group, f.flow_path, f.flow_level, " + - "rank_01 + rank_02 + rank_03 + rank_04 as rank " + - "FROM (SELECT app_id, flow_id, flow_name, flow_group, flow_path, flow_level, " + - "CASE WHEN flow_name = '$keyword' THEN 3000 ELSE 0 END rank_01, " + - "CASE WHEN flow_name like '%$keyword' THEN 2000 ELSE 0 END rank_02, " + - "CASE WHEN flow_name like '$keyword%' THEN 1000 ELSE 0 END rank_03, " + - "CASE WHEN flow_name like '%$keyword%' THEN 100 ELSE 0 END rank_04 " + - "FROM flow WHERE flow_name like '%$keyword%' ) f " + - "JOIN cfg_application a on f.app_id = a.app_id ORDER BY " + - "rank DESC, flow_name, app_id, flow_id, flow_group, flow_path LIMIT ?, ?"; - - private final static String SEARCH_JOB_WITH_PAGINATION = "SELECT SQL_CALC_FOUND_ROWS " + - "a.app_code, f.flow_name, f.flow_group, f.flow_path, f.flow_level, " + - "j.app_id, j.flow_id, j.job_id, j.job_name, j.job_path, j.job_type, " + - "rank_01+rank_02+rank_03+rank_04 as rank " + - "FROM (SELECT app_id, flow_id, job_id, job_name, job_path, job_type, " + - "CASE WHEN job_name = '$keyword' THEN 3000 ELSE 0 END rank_01, " + - "CASE WHEN job_name like '%$keyword' THEN 2000 ELSE 0 END rank_02, " + - "CASE WHEN job_name like '$keyword%' THEN 1000 ELSE 0 END rank_03, " + - "CASE WHEN job_name like '%$keyword%' THEN 100 ELSE 0 END rank_04 " + - "FROM flow_job WHERE job_name like '%$keyword%' ) j " + - "JOIN cfg_application a on a.app_id = j.app_id " + - "JOIN flow f on f.app_id = j.app_id AND f.flow_id = j.flow_id " + - "ORDER BY rank DESC, j.job_name, j.app_id, j.flow_id, j.job_id, j.job_path LIMIT ?, ?"; - - private final static String SEARCH_METRIC_WITH_PAGINATION = "SELECT SQL_CALC_FOUND_ROWS " + - "metric_id, `metric_name`, `metric_description`, `dashboard_name`, `metric_group`, `metric_category`, " + - "`metric_sub_category`, `metric_level`, `metric_source_type`, `metric_source`, `metric_source_dataset_id`, " + - "`metric_ref_id_type`, `metric_ref_id`, `metric_type`, `metric_grain`, `metric_display_factor`, " + - "`metric_display_factor_sym`, `metric_good_direction`, `metric_formula`, `dimensions`, " + - "`owners`, `tags`, `urn`, `metric_url`, `wiki_url`, `scm_url`, " + - "rank_01 + rank_02 + rank_03 + rank_04 + rank_05 + rank_06 + rank_07 + rank_08 + rank_09 + rank_10 + " + - "rank_11 + rank_12 + rank_13 + rank_14 + rank_15 + rank_16 + rank_17 + rank_18 + rank_19 + rank_20 as rank, " + - "0 as watch_id " + - "FROM (SELECT metric_id, `metric_name`, `metric_description`, `dashboard_name`, `metric_group`, " + - "`metric_category`, `metric_sub_category`, `metric_level`, `metric_source_type`, " + - "`metric_source`, `metric_source_dataset_id`, `metric_ref_id_type`, `metric_ref_id`, `metric_type`, " + - "`metric_grain`, `metric_display_factor`, `metric_display_factor_sym`, `metric_good_direction`, " + - "`metric_formula`, `dimensions`, `owners`, `tags`, `urn`, `metric_url`, `wiki_url`, `scm_url`, " + - "CASE WHEN `metric_name` = '$keyword' THEN 90000 ELSE 0 END rank_01, " + - "CASE WHEN `metric_name` like '%$keyword' THEN 30000 ELSE 0 END rank_02, " + - "CASE WHEN `metric_name` like '$keyword%'THEN 20000 ELSE 0 END rank_03, " + - "CASE WHEN `metric_name` like '%$keyword%'THEN 10000 ELSE 0 END rank_04, " + - "CASE WHEN `metric_description` = '$keyword' THEN 9000 ELSE 0 END rank_05, " + - "CASE WHEN `metric_description` like '%$keyword' THEN 3000 ELSE 0 END rank_06, " + - "CASE WHEN `metric_description` like '$keyword%' THEN 2000 ELSE 0 END rank_07, " + - "CASE WHEN `metric_description` like '%$keyword%' THEN 1000 ELSE 0 END rank_08, " + - "CASE WHEN `metric_category` = '$keyword' THEN 900 ELSE 0 END rank_09, " + - "CASE WHEN `metric_category` like '%$keyword' THEN 300 ELSE 0 END rank_10, " + - "CASE WHEN `metric_category` like '$keyword%' THEN 200 ELSE 0 END rank_11, " + - "CASE WHEN `metric_category` like '%$keyword%' THEN 100 ELSE 0 END rank_12, " + - "CASE WHEN `metric_group` = '$keyword' THEN 90 ELSE 0 END rank_13, " + - "CASE WHEN `metric_group` like '%$keyword' THEN 30 ELSE 0 END rank_14, " + - "CASE WHEN `metric_group` like '$keyword%' THEN 20 ELSE 0 END rank_15, " + - "CASE WHEN `metric_group` like '%$keyword%' THEN 10 ELSE 0 END rank_16, " + - "CASE WHEN `dashboard_name` = '$keyword' THEN 9 ELSE 0 END rank_17, " + - "CASE WHEN `dashboard_name` like '%$keyword' THEN 3 ELSE 0 END rank_18, " + - "CASE WHEN `dashboard_name` like '$keyword%' THEN 2 ELSE 0 END rank_19, " + - "CASE WHEN `dashboard_name` like '%$keyword%' THEN 1 ELSE 0 END rank_20 " + - "FROM dict_business_metric WHERE " + - "`metric_name` like '%$keyword%' or `dashboard_name` like '%$keyword%' or `metric_group` like '%$keyword%' " + - "or `metric_category` like '%$keyword%' or `metric_description` like '%$keyword%' ) m ORDER BY " + - "rank DESC, `metric_name`, `metric_category`, `metric_group`, `dashboard_name` LIMIT ?, ?;"; - - private final static String SEARCH_DATASET_BY_COMMENTS_WITH_PAGINATION = "SELECT SQL_CALC_FOUND_ROWS " + - "id, name, source, urn, `schema` FROM dict_dataset WHERE id in " + - "(SELECT dataset_id FROM comments WHERE MATCH(text) against ('*$keyword*' in BOOLEAN MODE) ) " + - "UNION ALL SELECT id, name, source, urn, `schema` FROM dict_dataset WHERE id in " + - "(SELECT fd.dataset_id FROM ( " + - "SELECT id FROM field_comments fc WHERE " + - "MATCH(comment) AGAINST ('*$keyword*' IN BOOLEAN MODE) ) c JOIN dict_field_detail fd " + - "ON ( find_in_set(c.id, fd.comment_ids) or c.id = fd.default_comment_id )) " + - "ORDER BY 2 LIMIT ?, ?;"; - - public static List getAutoCompleteList(String input, int limit) - { - List names = new ArrayList<>(); - names.addAll(getAutoCompleteListDataset(input, limit / 3)); - names.addAll(getAutoCompleteListMetric(input, limit / 3)); - names.addAll(getAutoCompleteListFlow(input, limit / 3)); - return names; - } - - public static List getAutoCompleteListDataset(String input, int limit) - { - String elasticSearchTypeURLKey = "elasticsearch.dataset.url"; - String fieldName = "name_suggest"; - return getAutoCompleteListbyES(elasticSearchTypeURLKey,fieldName,input,limit); - } - - public static List getAutoCompleteListMetric(String input, int limit) - { - String elasticSearchTypeURLKey = "elasticsearch.metric.url"; - String fieldName = "metric_name_suggest"; - return getAutoCompleteListbyES(elasticSearchTypeURLKey,fieldName,input,limit); - } - - public static List getAutoCompleteListFlow(String input, int limit) - { - String elasticSearchTypeURLKey = "elasticsearch.flow.url"; - String fieldName = "flow_name_suggest"; - return getAutoCompleteListbyES(elasticSearchTypeURLKey,fieldName,input,limit); - } - - public static List getAutoCompleteListbyES(String elasticSearchTypeURLKey, String fieldName, String input, - int limit) - { - // use elastic search completion suggester, ES will validate the input and limit - List completionSuggestionList = new ArrayList(); - Set completionSuggestionSet = new HashSet(); - - JsonNode responseNode = null; - ObjectNode keywordNode = null; - - try { - keywordNode = utils.Search.generateElasticSearchCompletionSuggesterQuery(fieldName, input, limit); - } catch (Exception e) { - Logger.error("Elastic search completion suggester error. Error message :" + e.getMessage()); - } - - Logger.info("The completion suggester query sent to Elastic Search was: " + keywordNode.toString()); - - Promise responsePromise = - WS.url(Play.application().configuration().getString(elasticSearchTypeURLKey)).post(keywordNode); - responseNode = responsePromise.get(1000).asJson(); - - if (responseNode == null || !responseNode.isContainerNode()) { - return completionSuggestionList; - } - - JsonNode suggestNode = responseNode.get("suggest"); - if (suggestNode == null || !suggestNode.has("wh-suggest")) { - Logger.error("Elastic search completion suggester response does not contain suggest node"); - return completionSuggestionList; - } - - JsonNode whSuggestNode = suggestNode.get("wh-suggest"); - if (whSuggestNode == null || !whSuggestNode.isArray()) { - Logger.error("Elastic search completion suggester response does not contain wh-suggest node"); - return completionSuggestionList; - } - - Iterator arrayIterator = whSuggestNode.elements(); - if (arrayIterator == null) { - return completionSuggestionList; - } - - while (arrayIterator.hasNext()) { - JsonNode node = arrayIterator.next(); - if (!node.isContainerNode() || !node.has("options")) { - continue; - } - - JsonNode optionsNode = node.get("options"); - if (optionsNode == null || !optionsNode.isArray()) { - continue; - } - - Iterator arrayIteratorOptions = optionsNode.elements(); - if (arrayIteratorOptions == null) { - continue; - } - - while (arrayIteratorOptions.hasNext()) { - JsonNode textNode = arrayIteratorOptions.next(); - if (textNode == null || !textNode.has("text")) { - continue; - } - String oneSuggestion = textNode.get("text").asText(); - completionSuggestionSet.add(oneSuggestion); - } - } - - completionSuggestionList.addAll(completionSuggestionSet); - Logger.info("Returned suggestion list is: " + completionSuggestionList); - return completionSuggestionList; - } - - // this is for did you mean feature - public static List getSuggestionList(String category, String searchKeyword) - { - List SuggestionList = new ArrayList(); - String elasticSearchType = "dataset"; - String elasticSearchTypeURLKey = "elasticsearch.dataset.url"; - String fieldName = "name"; - - JsonNode responseNode = null; - ObjectNode keywordNode = null; - - try { - String lCategory = category.toLowerCase(); - Logger.info("lCategory is " + category); - - switch (lCategory) { - case "dataset": - elasticSearchType = "dataset"; - elasticSearchTypeURLKey = "elasticsearch.dataset.url"; - fieldName = "name"; - break; - case "metric": - elasticSearchType = "metric"; - elasticSearchTypeURLKey = "elasticsearch.metric.url"; - fieldName = "metric_name"; - break; - case "flow": - elasticSearchType = "flow"; - elasticSearchTypeURLKey = "elasticsearch.flow.url"; - fieldName = "flow_name"; - break; - default: - break; - } - - keywordNode = utils.Search.generateElasticSearchPhraseSuggesterQuery(elasticSearchType, fieldName, searchKeyword); - } catch (Exception e) { - Logger.error("Elastic search phrase suggester error. Error message :" + e.getMessage()); - } - - Logger.info("The suggest query sent to Elastic Search is: " + keywordNode.toString()); - - Promise responsePromise = - WS.url(Play.application().configuration().getString(elasticSearchTypeURLKey)).post(keywordNode); - responseNode = responsePromise.get(1000).asJson(); - - if (responseNode == null || !responseNode.isContainerNode() || !responseNode.has("hits")) { - return SuggestionList; - } - - JsonNode suggestNode = responseNode.get("suggest"); - Logger.info("suggestNode is " + suggestNode.toString()); - - if (suggestNode == null || !suggestNode.has("simple_phrase")) { - return SuggestionList; - } - - JsonNode simplePhraseNode = suggestNode.get("simple_phrase"); - if (simplePhraseNode == null || !simplePhraseNode.isArray()) { - return SuggestionList; - } - - Iterator arrayIterator = simplePhraseNode.elements(); - if (arrayIterator == null) { - return SuggestionList; - } - - while (arrayIterator.hasNext()) { - JsonNode node = arrayIterator.next(); - if (!node.isContainerNode() || !node.has("options")) { - continue; - } - - JsonNode optionsNode = node.get("options"); - if (optionsNode == null || !optionsNode.isArray()) { - continue; - } - - Iterator arrayIteratorOptions = optionsNode.elements(); - if (arrayIteratorOptions == null) { - continue; - } - - while (arrayIteratorOptions.hasNext()) { - JsonNode textNode = arrayIteratorOptions.next(); - if (textNode == null || !textNode.has("text")) { - continue; - } - String oneSuggestion = textNode.get("text").asText(); - SuggestionList.add(oneSuggestion); - } - } - - return SuggestionList; - } - - public static JsonNode elasticSearchDatasetByKeyword( - String category, - String keywords, - String source, - int page, - int size) - { - ObjectNode queryNode = Json.newObject(); - queryNode.put("from", (page-1)*size); - queryNode.put("size", size); - JsonNode responseNode = null; - ObjectNode keywordNode = null; - - try { - keywordNode = utils.Search.generateElasticSearchQueryString(category, source, keywords); - } catch (Exception e) { - Logger.error("Elastic search dataset input query is not JSON format. Error message :" + e.getMessage()); - } - - if (keywordNode != null) { - ObjectNode funcScoreNodes = Json.newObject(); - - ObjectNode fieldValueFactorNode = Json.newObject(); - fieldValueFactorNode.put("field", "static_boosting_score"); - fieldValueFactorNode.put("factor", 1); - fieldValueFactorNode.put("modifier", "square"); - fieldValueFactorNode.put("missing", 1); - - funcScoreNodes.put("query", keywordNode); - funcScoreNodes.put("field_value_factor", fieldValueFactorNode); - - ObjectNode funcScoreNodesWrapper = Json.newObject(); - funcScoreNodesWrapper.put("function_score", funcScoreNodes); - - queryNode.put("query", funcScoreNodesWrapper); - - ObjectNode filterNode = Json.newObject(); - try { - filterNode = utils.Search.generateElasticSearchFilterString(source); - } catch (Exception e) { - Logger.error("Elastic search filter query node generation failed :" + e.getMessage()); - } - - if (filterNode != null) { - queryNode.put("post_filter", filterNode); - } - - Logger.info( - " === elasticSearchDatasetByKeyword === The query sent to Elastic Search is: " + queryNode.toString()); - - Promise responsePromise = - WS.url(Play.application().configuration().getString(SearchDAO.ELASTICSEARCH_DATASET_URL_KEY)).post(queryNode); - responseNode = responsePromise.get(1000).asJson(); - - // Logger.debug("The responseNode from Elastic Search is: " + responseNode.toString()); - - } - - ObjectNode resultNode = Json.newObject(); - Long count = 0L; - List pagedDatasets = new ArrayList<>(); - resultNode.put("page", page); - resultNode.put("category", category); - resultNode.put("source", source); - resultNode.put("itemsPerPage", size); - resultNode.put("keywords", keywords); - - if (responseNode != null && responseNode.isContainerNode() && responseNode.has("hits")) - { - JsonNode hitsNode = responseNode.get("hits"); - if (hitsNode != null) - { - if (hitsNode.has("total")) - { - count = hitsNode.get("total").asLong(); - } - if (hitsNode.has("hits")) - { - JsonNode dataNode = hitsNode.get("hits"); - if (dataNode != null && dataNode.isArray()) - { - Iterator arrayIterator = dataNode.elements(); - if (arrayIterator != null) - { - while (arrayIterator.hasNext()) - { - JsonNode node = arrayIterator.next(); - if (node.isContainerNode() && node.has("_id")) - { - Dataset dataset = new Dataset(); - dataset.id = node.get("_id").asLong(); - if (node.has("_source")) - { - JsonNode sourceNode = node.get("_source"); - if (sourceNode != null) - { - if (sourceNode.has("name")) - { - dataset.name = sourceNode.get("name").asText(); - } - if (sourceNode.has("source")) - { - dataset.source = sourceNode.get("source").asText(); - } - if (sourceNode.has("urn")) - { - dataset.urn = sourceNode.get("urn").asText(); - } - if (sourceNode.has("schema")) - { - dataset.schema = sourceNode.get("schema").asText(); - } - } - } - pagedDatasets.add(dataset); - } - } - } - - } - } - - } - } - resultNode.put("count", count); - resultNode.put("totalPages", (int)Math.ceil(count/((double)size))); - resultNode.set("data", Json.toJson(pagedDatasets)); - return resultNode; - } - - public static JsonNode elasticSearchMetricByKeyword( - String category, - String keywords, - int page, - int size) - { - ObjectNode queryNode = Json.newObject(); - queryNode.put("from", (page-1)*size); - queryNode.put("size", size); - JsonNode responseNode = null; - ObjectNode keywordNode = null; - - try - { - keywordNode = utils.Search.generateElasticSearchQueryString(category, null, keywords); - } - catch(Exception e) - { - Logger.error("Elastic search metric input query is not JSON format. Error message :" + e.getMessage()); - } - - if (keywordNode != null) - { - queryNode.set("query", keywordNode); - - Logger.info(" === elasticSearchMetricByKeyword === The query sent to Elastic Search is: " + queryNode.toString()); - - Promise responsePromise = WS.url(Play.application().configuration().getString( - SearchDAO.ELASTICSEARCH_METRIC_URL_KEY)).post(queryNode); - responseNode = responsePromise.get(1000).asJson(); - } - - ObjectNode resultNode = Json.newObject(); - Long count = 0L; - List pagedMetrics = new ArrayList<>(); - resultNode.put("page", page); - resultNode.put("category", category); - resultNode.put("isMetrics", true); - resultNode.put("itemsPerPage", size); - resultNode.put("keywords", keywords); - - if (responseNode != null && responseNode.isContainerNode() && responseNode.has("hits")) - { - JsonNode hitsNode = responseNode.get("hits"); - if (hitsNode != null) - { - if (hitsNode.has("total")) - { - count = hitsNode.get("total").asLong(); - } - if (hitsNode.has("hits")) - { - JsonNode dataNode = hitsNode.get("hits"); - if (dataNode != null && dataNode.isArray()) - { - Iterator arrayIterator = dataNode.elements(); - if (arrayIterator != null) - { - while (arrayIterator.hasNext()) - { - JsonNode node = arrayIterator.next(); - if (node.isContainerNode() && node.has("_id")) - { - Metric metric = new Metric(); - metric.id = node.get("_id").asInt(); - if (node.has("_source")) { - JsonNode sourceNode = node.get("_source"); - if (sourceNode != null) { - if (sourceNode.has("metric_name")) { - metric.name = sourceNode.get("metric_name").asText(); - } - if (sourceNode.has("metric_description")) { - metric.description = sourceNode.get("metric_description").asText(); - } - if (sourceNode.has("dashboard_name")) { - metric.dashboardName = sourceNode.get("dashboard_name").asText(); - } - if (sourceNode.has("metric_group")) { - metric.group = sourceNode.get("metric_group").asText(); - } - if (sourceNode.has("metric_category")) { - metric.category = sourceNode.get("metric_category").asText(); - } - if (sourceNode.has("urn")) { - metric.urn = sourceNode.get("urn").asText(); - } - if (sourceNode.has("metric_source")) { - metric.source = sourceNode.get("metric_source").asText(); - if (StringUtils.isBlank(metric.source)) - { - metric.source = null; - } - } - metric.schema = sourceNode.toString(); - } - } - pagedMetrics.add(metric); - } - } - } - - } - } - - } - } - resultNode.put("count", count); - resultNode.put("totalPages", (int)Math.ceil(count/((double)size))); - resultNode.set("data", Json.toJson(pagedMetrics)); - return resultNode; - } - - public static JsonNode elasticSearchFlowByKeyword( - String category, - String keywords, - int page, - int size) - { - ObjectNode queryNode = Json.newObject(); - queryNode.put("from", (page-1)*size); - queryNode.put("size", size); - JsonNode searchOpt = null; - JsonNode responseNode = null; - ObjectNode keywordNode = null; - - try - { - keywordNode = utils.Search.generateElasticSearchQueryString(category, null, keywords); - } - catch(Exception e) - { - Logger.error("Elastic search flow input query is not JSON format. Error message :" + e.getMessage()); - } - - if (keywordNode != null) - { - queryNode.set("query", keywordNode); - - Logger.info(" === elasticSearchFlowByKeyword === The query sent to Elastic Search is: " + queryNode.toString()); - - Promise responsePromise = WS.url(Play.application().configuration().getString( - SearchDAO.ELASTICSEARCH_FLOW_URL_KEY)).post(queryNode); - responseNode = responsePromise.get(1000).asJson(); - } - - ObjectNode resultNode = Json.newObject(); - Long count = 0L; - List pagedFlowJobs = new ArrayList<>(); - resultNode.put("page", page); - resultNode.put("category", category); - resultNode.put("isFlowJob", true); - resultNode.put("itemsPerPage", size); - resultNode.put("keywords", keywords); - - if (responseNode != null && responseNode.isContainerNode() && responseNode.has("hits")) - { - JsonNode hitsNode = responseNode.get("hits"); - if (hitsNode != null) - { - if (hitsNode.has("total")) - { - count = hitsNode.get("total").asLong(); - } - if (hitsNode.has("hits")) - { - JsonNode dataNode = hitsNode.get("hits"); - if (dataNode != null && dataNode.isArray()) - { - Iterator arrayIterator = dataNode.elements(); - if (arrayIterator != null) - { - while (arrayIterator.hasNext()) - { - JsonNode node = arrayIterator.next(); - if (node.isContainerNode() && node.has("_id")) - { - FlowJob flowJob = new FlowJob(); - if (node.has("_source")) { - JsonNode sourceNode = node.get("_source"); - if (sourceNode != null) { - if (sourceNode.has("app_code")) { - flowJob.appCode = sourceNode.get("app_code").asText(); - } - if (sourceNode.has("app_id")) { - flowJob.appId = sourceNode.get("app_id").asInt(); - } - if (sourceNode.has("flow_id")) { - flowJob.flowId = sourceNode.get("flow_id").asLong(); - } - if (sourceNode.has("flow_name")) { - flowJob.flowName = sourceNode.get("flow_name").asText(); - flowJob.displayName = flowJob.flowName; - } - if (sourceNode.has("flow_path")) { - flowJob.flowPath = sourceNode.get("flow_path").asText(); - } - if (sourceNode.has("flow_group")) { - flowJob.flowGroup = sourceNode.get("flow_group").asText(); - } - flowJob.link = "#/flows/name/" + flowJob.appCode + "/" + - Long.toString(flowJob.flowId) + "/page/1?urn=" + flowJob.flowGroup; - flowJob.path = flowJob.appCode + "/" + flowJob.flowPath; - - flowJob.schema = sourceNode.toString(); - } - } - pagedFlowJobs.add(flowJob); - } - } - } - - } - } - - } - } - resultNode.put("count", count); - resultNode.put("totalPages", (int)Math.ceil(count/((double)size))); - resultNode.set("data", Json.toJson(pagedFlowJobs)); - return resultNode; - } - - public static ObjectNode getPagedDatasetByKeyword(String category, String keyword, String source, int page, int size) - { - List pagedDatasets = new ArrayList(); - final JdbcTemplate jdbcTemplate = getJdbcTemplate(); - javax.sql.DataSource ds = jdbcTemplate.getDataSource(); - DataSourceTransactionManager tm = new DataSourceTransactionManager(ds); - - TransactionTemplate txTemplate = new TransactionTemplate(tm); - - ObjectNode result; - result = txTemplate.execute(new TransactionCallback() - { - public ObjectNode doInTransaction(TransactionStatus status) - { - List> rows = null; - try{ - if (StringUtils.isBlank(source) || source.toLowerCase().equalsIgnoreCase("all")) - { - String query = SEARCH_DATASET_WITH_PAGINATION.replace("$keyword", keyword); - rows = jdbcTemplate.queryForList(query, (page-1)*size, size); - } - else - { - String query = SEARCH_DATASET_BY_SOURCE_WITH_PAGINATION.replace("$keyword", keyword); - rows = jdbcTemplate.queryForList(query, source, (page-1)*size, size); - } - } - catch (Exception e) - { - Logger.error(e.getMessage()); - } - - if (rows != null) - { - for (Map row : rows) { - - Dataset ds = new Dataset(); - ds.id = (Long)row.get(DatasetRowMapper.DATASET_ID_COLUMN); - ds.name = (String)row.get(DatasetRowMapper.DATASET_NAME_COLUMN); - ds.source = (String)row.get(DatasetRowMapper.DATASET_SOURCE_COLUMN); - ds.urn = (String)row.get(DatasetRowMapper.DATASET_URN_COLUMN); - ds.schema = (String)row.get(DatasetRowMapper.DATASET_SCHEMA_COLUMN); - pagedDatasets.add(ds); - } - } - - long count = 0; - try { - count = jdbcTemplate.queryForObject( - "SELECT FOUND_ROWS()", - Long.class); - } - catch(EmptyResultDataAccessException e) - { - Logger.error("Exception = " + e.getMessage()); - } - - ObjectNode resultNode = Json.newObject(); - resultNode.put("count", count); - resultNode.put("page", page); - resultNode.put("category", category); - resultNode.put("source", source); - resultNode.put("itemsPerPage", size); - resultNode.put("totalPages", (int)Math.ceil(count/((double)size))); - resultNode.set("data", Json.toJson(pagedDatasets)); - - return resultNode; - } - }); - - return result; - } - - public static ObjectNode getPagedMetricByKeyword(final String category, String keyword, int page, int size) - { - final JdbcTemplate jdbcTemplate = getJdbcTemplate(); - javax.sql.DataSource ds = jdbcTemplate.getDataSource(); - DataSourceTransactionManager tm = new DataSourceTransactionManager(ds); - - TransactionTemplate txTemplate = new TransactionTemplate(tm); - - ObjectNode result; - result = txTemplate.execute(new TransactionCallback() - { - public ObjectNode doInTransaction(TransactionStatus status) - { - String query = SEARCH_METRIC_WITH_PAGINATION.replace("$keyword", keyword); - List pagedMetrics = new ArrayList(); - try { - - pagedMetrics = jdbcTemplate.query( - query, - new MetricRowMapper(), - (page - 1) * size, size); - } - catch(Exception e) - { - Logger.error(e.getMessage()); - } - - long count = 0; - try { - count = jdbcTemplate.queryForObject( - "SELECT FOUND_ROWS()", - Long.class); - } - catch(EmptyResultDataAccessException e) - { - Logger.error("Exception = " + e.getMessage()); - } - - ObjectNode resultNode = Json.newObject(); - resultNode.put("count", count); - resultNode.put("page", page); - resultNode.put("isMetrics", true); - resultNode.put("category", category); - resultNode.put("itemsPerPage", size); - resultNode.put("totalPages", (int)Math.ceil(count/((double)size))); - resultNode.set("data", Json.toJson(pagedMetrics)); - - return resultNode; - } - }); - - return result; - } - - public static ObjectNode getPagedFlowByKeyword(String category, String keyword, int page, int size) - { - final List pagedFlows = new ArrayList(); - final JdbcTemplate jdbcTemplate = getJdbcTemplate(); - javax.sql.DataSource ds = jdbcTemplate.getDataSource(); - DataSourceTransactionManager tm = new DataSourceTransactionManager(ds); - - TransactionTemplate txTemplate = new TransactionTemplate(tm); - - ObjectNode result; - result = txTemplate.execute(new TransactionCallback() - { - public ObjectNode doInTransaction(TransactionStatus status) - { - String query = SEARCH_FLOW_WITH_PAGINATION.replace("$keyword", keyword); - List> rows = null; - try - { - rows = jdbcTemplate.queryForList(query, (page-1)*size, size); - } - catch (Exception e) - { - Logger.error(e.getMessage()); - } - - if (rows != null) - { - for (Map row : rows) { - - FlowJob flow = new FlowJob(); - flow.flowId = (Long)row.get(FlowRowMapper.FLOW_ID_COLUMN); - flow.flowName = (String)row.get(FlowRowMapper.FLOW_NAME_COLUMN); - flow.flowPath = (String)row.get(FlowRowMapper.FLOW_PATH_COLUMN); - flow.flowGroup = (String)row.get(FlowRowMapper.FLOW_GROUP_COLUMN); - flow.appCode = (String)row.get(FlowRowMapper.APP_CODE_COLUMN); - flow.appId = (Integer)row.get(FlowRowMapper.APP_ID_COLUMN); - flow.displayName = flow.flowName; - flow.link = "#/flows/" + flow.appCode + "/" + - flow.flowGroup + "/" + Long.toString(flow.flowId) + "/page/1"; - flow.path = flow.appCode + "/" + flow.flowPath; - pagedFlows.add(flow); - } - } - - long count = 0; - try { - count = jdbcTemplate.queryForObject( - "SELECT FOUND_ROWS()", - Long.class); - } - catch(EmptyResultDataAccessException e) - { - Logger.error("Exception = " + e.getMessage()); - } - - ObjectNode resultNode = Json.newObject(); - resultNode.put("count", count); - resultNode.put("isFlowJob", true); - resultNode.put("page", page); - resultNode.put("category", category); - resultNode.put("itemsPerPage", size); - resultNode.put("totalPages", (int)Math.ceil(count/((double)size))); - resultNode.set("data", Json.toJson(pagedFlows)); - - return resultNode; - } - }); - - return result; - } - - public static ObjectNode getPagedJobByKeyword(String category, String keyword, int page, int size) - { - final List pagedFlowJobs = new ArrayList(); - final JdbcTemplate jdbcTemplate = getJdbcTemplate(); - javax.sql.DataSource ds = jdbcTemplate.getDataSource(); - DataSourceTransactionManager tm = new DataSourceTransactionManager(ds); - - TransactionTemplate txTemplate = new TransactionTemplate(tm); - - ObjectNode result; - result = txTemplate.execute(new TransactionCallback() - { - public ObjectNode doInTransaction(TransactionStatus status) - { - String query = SEARCH_JOB_WITH_PAGINATION.replace("$keyword", keyword); - List> rows = null; - - try - { - rows = jdbcTemplate.queryForList(query, (page-1)*size, size); - } - catch (Exception e) - { - Logger.error(e.getMessage()); - } - - if (rows != null) - { - for (Map row : rows) { - - FlowJob flowJob = new FlowJob(); - flowJob.flowId = (Long)row.get(FlowRowMapper.FLOW_ID_COLUMN); - flowJob.jobId = (Long)row.get(FlowRowMapper.JOB_ID_COLUMN); - flowJob.jobName = (String)row.get(FlowRowMapper.JOB_NAME_COLUMN); - flowJob.jobPath = (String)row.get(FlowRowMapper.JOB_PATH_COLUMN); - flowJob.jobType = (String)row.get(FlowRowMapper.JOB_TYPE_COLUMN); - flowJob.flowName = (String)row.get(FlowRowMapper.FLOW_NAME_COLUMN); - flowJob.flowPath = (String)row.get(FlowRowMapper.FLOW_PATH_COLUMN); - flowJob.flowGroup = (String)row.get(FlowRowMapper.FLOW_GROUP_COLUMN); - flowJob.appCode = (String)row.get(FlowRowMapper.APP_CODE_COLUMN); - flowJob.appId = (Integer)row.get(FlowRowMapper.APP_ID_COLUMN); - flowJob.displayName = flowJob.jobName; - flowJob.link = "#/flows/" + flowJob.appCode + "/" + - flowJob.flowGroup + "/" + Long.toString(flowJob.flowId) + "/page/1"; - flowJob.path = flowJob.appCode + "/" + flowJob.jobPath; - - pagedFlowJobs.add(flowJob); - } - } - - long count = 0; - try { - count = jdbcTemplate.queryForObject( - "SELECT FOUND_ROWS()", - Long.class); - } - catch(EmptyResultDataAccessException e) - { - Logger.error("Exception = " + e.getMessage()); - } - - ObjectNode resultNode = Json.newObject(); - resultNode.put("count", count); - resultNode.put("isFlowJob", true); - resultNode.put("page", page); - resultNode.put("category", category); - resultNode.put("itemsPerPage", size); - resultNode.put("totalPages", (int)Math.ceil(count/((double)size))); - resultNode.set("data", Json.toJson(pagedFlowJobs)); - - return resultNode; - } - }); - - return result; - } - - public static ObjectNode getPagedCommentsByKeyword(String category, String keyword, int page, int size) - { - final JdbcTemplate jdbcTemplate = getJdbcTemplate(); - javax.sql.DataSource ds = jdbcTemplate.getDataSource(); - DataSourceTransactionManager tm = new DataSourceTransactionManager(ds); - - TransactionTemplate txTemplate = new TransactionTemplate(tm); - - ObjectNode result; - result = txTemplate.execute(new TransactionCallback() - { - public ObjectNode doInTransaction(TransactionStatus status) - { - List> rows = null; - List pagedDatasets = new ArrayList(); - String query = SEARCH_DATASET_BY_COMMENTS_WITH_PAGINATION.replace("$keyword", keyword); - try{ - rows = jdbcTemplate.queryForList(query, (page-1)*size, size); - } - catch (Exception e) - { - Logger.error(e.getMessage()); - } - - if (rows != null) - { - for (Map row : rows) { - - Dataset ds = new Dataset(); - ds.id = (long)row.get(DatasetRowMapper.DATASET_ID_COLUMN); - ds.name = (String)row.get(DatasetRowMapper.DATASET_NAME_COLUMN); - ds.source = (String)row.get(DatasetRowMapper.DATASET_SOURCE_COLUMN); - ds.urn = (String)row.get(DatasetRowMapper.DATASET_URN_COLUMN); - ds.schema = (String)row.get(DatasetRowMapper.DATASET_SCHEMA_COLUMN); - pagedDatasets.add(ds); - } - } - - long count = 0; - try { - count = jdbcTemplate.queryForObject( - "SELECT FOUND_ROWS()", - Long.class); - } - catch(EmptyResultDataAccessException e) - { - Logger.error("Exception = " + e.getMessage()); - } - - ObjectNode resultNode = Json.newObject(); - resultNode.put("count", count); - resultNode.put("page", page); - resultNode.put("category", category); - resultNode.put("itemsPerPage", size); - resultNode.put("totalPages", (int)Math.ceil(count/((double)size))); - resultNode.set("data", Json.toJson(pagedDatasets)); - - return resultNode; - } - }); - - return result; - } - -} diff --git a/wherehows-frontend/app/utils/Search.java b/wherehows-frontend/app/utils/Search.java deleted file mode 100644 index a32f2a16f5..0000000000 --- a/wherehows-frontend/app/utils/Search.java +++ /dev/null @@ -1,1040 +0,0 @@ -/** - * Copyright 2015 LinkedIn Corp. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - */ -package utils; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; -import java.util.ArrayList; -import java.util.List; -import java.nio.file.Paths; -import java.nio.file.Files; -import org.apache.commons.lang3.StringUtils; -import play.Logger; -import play.libs.Json; - - -public class Search { - public final static String DATASET_CATEGORY = "datasets"; - public final static String METRIC_CATEGORY = "metrics"; - public final static String COMMENT_CATEGORY = "comments"; - public final static String FLOW_CATEGORY = "flows"; - public final static String JOB_CATEGORY = "jobs"; - - private static final String WHZ_ELASTICSEARCH_DATASET_QUERY_FILE = System.getenv("WHZ_ELASTICSEARCH_DATASET_QUERY_TEMPLATE"); - private static final String WHZ_ELASTICSEARCH_METRIC_QUERY_FILE = System.getenv("WHZ_ELASTICSEARCH_METRIC_QUERY_TEMPLATE"); - private static final String WHZ_ELASTICSEARCH_FLOW_QUERY_FILE = System.getenv("WHZ_ELASTICSEARCH_FLOW_QUERY_TEMPLATE"); - private static final String WHZ_ELASTICSEARCH_COMMENT_QUERY_FILE = System.getenv("WHZ_ELASTICSEARCH_COMMENT_QUERY_TEMPLATE"); - private static final String WHZ_ELASTICSEARCH_SUGGESTER_QUERY_FILE = System.getenv("WHZ_ELASTICSEARCH_SUGGESTER_QUERY_TEMPLATE"); - private static final String WHZ_ELASTICSEARCH_AUTO_COMPLETION_QUERY_FILE = System.getenv("WHZ_ELASTICSEARCH_AUTO_COMPLETION_QUERY_TEMPLATE"); - private static final String WHZ_ELASTICSEARCH_FILTER_UNIT_FILE = System.getenv("WHZ_ELASTICSEARCH_FILTER_UNIT"); - - public static String readJsonQueryFile(String jsonFile) { - try { - ObjectMapper objectMapper = new ObjectMapper(); - - String contents = new String(Files.readAllBytes(Paths.get(jsonFile))); - JsonNode json = objectMapper.readTree(contents); - return json.toString(); - - } catch (Exception e) { - Logger.error("ReadJsonQueryFile failed. Error: " + e.getMessage()); - e.printStackTrace(); - return null; - } - } - - public static ObjectNode generateElasticSearchCompletionSuggesterQuery(String field, String searchKeyword, - int limit) { - if (StringUtils.isBlank(searchKeyword)) { - return null; - } - - String queryTemplate = readJsonQueryFile(WHZ_ELASTICSEARCH_AUTO_COMPLETION_QUERY_FILE); - String query = queryTemplate.replace("$SEARCHKEYWORD", searchKeyword.toLowerCase()); - - if (StringUtils.isNotBlank(field)) { - query = query.replace("$FIELD", field.toLowerCase()); - } - - query = query.replace("$LIMIT", Integer.toString(limit)); - - ObjectNode suggestNode = Json.newObject(); - ObjectNode textNode = Json.newObject(); - try { - textNode = (ObjectNode) new ObjectMapper().readTree(query); - suggestNode.put("suggest", textNode); - } catch (Exception e) { - Logger.error("suggest Exception = " + e.getMessage()); - } - - Logger.info("completionSuggesterQuery suggestNode is " + suggestNode.toString()); - return suggestNode; - } - - public static ObjectNode generateElasticSearchPhraseSuggesterQuery(String category, String field, - String searchKeyword) { - if (StringUtils.isBlank(searchKeyword)) { - return null; - } - - String queryTemplate = readJsonQueryFile(WHZ_ELASTICSEARCH_SUGGESTER_QUERY_FILE); - String query = queryTemplate.replace("$SEARCHKEYWORD", searchKeyword.toLowerCase()); - - if (StringUtils.isNotBlank(field)) { - query = query.replace("$FIELD", field.toLowerCase()); - } - - ObjectNode suggestNode = Json.newObject(); - ObjectNode textNode = Json.newObject(); - try { - textNode = (ObjectNode) new ObjectMapper().readTree(query); - suggestNode.put("suggest", textNode); - } catch (Exception e) { - Logger.error("suggest Exception = " + e.getMessage()); - } - Logger.info("suggestNode is " + suggestNode.toString()); - return suggestNode; - } - - public static ObjectNode generateElasticSearchQueryString(String category, String source, String keywords) { - if (StringUtils.isBlank(keywords)) { - return null; - } - - List shouldValueList = new ArrayList(); - - String queryTemplate = readJsonQueryFile(WHZ_ELASTICSEARCH_DATASET_QUERY_FILE); - - String[] values = keywords.trim().split(","); - if (StringUtils.isNotBlank(category)) { - if (category.equalsIgnoreCase(METRIC_CATEGORY)) { - queryTemplate = readJsonQueryFile(WHZ_ELASTICSEARCH_METRIC_QUERY_FILE);; - } else if (category.equalsIgnoreCase(COMMENT_CATEGORY)) { - queryTemplate = readJsonQueryFile(WHZ_ELASTICSEARCH_COMMENT_QUERY_FILE); - } else if (category.equalsIgnoreCase(FLOW_CATEGORY) || category.equalsIgnoreCase(JOB_CATEGORY)) { - queryTemplate = readJsonQueryFile(WHZ_ELASTICSEARCH_FLOW_QUERY_FILE); - } - } - - for (String value : values) { - if (StringUtils.isNotBlank(value)) { - String query = queryTemplate.replace("$VALUE", value.replace("\"", "").toLowerCase().trim()); - shouldValueList.add(Json.parse(query)); - } - } - - ObjectNode shouldNode = Json.newObject(); - shouldNode.set("should", Json.toJson(shouldValueList)); - ObjectNode queryNode = Json.newObject(); - queryNode.put("bool", shouldNode); - - return queryNode; - } - - public static ObjectNode generateDatasetAdvSearchQueryString(JsonNode searchOpt) { - List scopeInList = new ArrayList(); - List scopeNotInList = new ArrayList(); - List tableInList = new ArrayList(); - List tableNotInList = new ArrayList(); - List fieldAnyList = new ArrayList(); - List fieldAllList = new ArrayList(); - List fieldNotInList = new ArrayList(); - String comments = null; - String datasetSources = null; - - if (searchOpt != null && (searchOpt.isContainerNode())) { - if (searchOpt.has("scope")) { - JsonNode scopeNode = searchOpt.get("scope"); - if (scopeNode != null && scopeNode.isContainerNode()) { - if (scopeNode.has("in")) { - JsonNode scopeInNode = scopeNode.get("in"); - if (scopeInNode != null) { - String scopeInStr = scopeInNode.asText(); - if (StringUtils.isNotBlank(scopeInStr)) { - String[] scopeInArray = scopeInStr.split(","); - if (scopeInArray != null) { - for (String value : scopeInArray) { - if (StringUtils.isNotBlank(value)) { - scopeInList.add(value.trim()); - } - } - } - } - } - } - if (scopeNode.has("not")) { - JsonNode scopeNotInNode = scopeNode.get("not"); - if (scopeNotInNode != null) { - String scopeNotInStr = scopeNotInNode.asText(); - if (StringUtils.isNotBlank(scopeNotInStr)) { - String[] scopeNotInArray = scopeNotInStr.split(","); - if (scopeNotInArray != null) { - for (String value : scopeNotInArray) { - if (StringUtils.isNotBlank(value)) { - scopeNotInList.add(value.trim()); - } - } - } - } - } - } - } - } - - if (searchOpt.has("table")) { - JsonNode tableNode = searchOpt.get("table"); - if (tableNode != null && tableNode.isContainerNode()) { - if (tableNode.has("in")) { - JsonNode tableInNode = tableNode.get("in"); - if (tableInNode != null) { - String tableInStr = tableInNode.asText(); - if (StringUtils.isNotBlank(tableInStr)) { - String[] tableInArray = tableInStr.split(","); - if (tableInArray != null) { - for (String value : tableInArray) { - if (StringUtils.isNotBlank(value)) { - tableInList.add(value.trim()); - } - } - } - } - } - } - if (tableNode.has("not")) { - JsonNode tableNotInNode = tableNode.get("not"); - if (tableNotInNode != null) { - String tableNotInStr = tableNotInNode.asText(); - if (StringUtils.isNotBlank(tableNotInStr)) { - String[] tableNotInArray = tableNotInStr.split(","); - if (tableNotInArray != null) { - for (String value : tableNotInArray) { - if (StringUtils.isNotBlank(value)) { - tableNotInList.add(value.trim()); - } - } - } - } - } - } - } - } - - if (searchOpt.has("fields")) { - JsonNode fieldNode = searchOpt.get("fields"); - if (fieldNode != null && fieldNode.isContainerNode()) { - if (fieldNode.has("any")) { - JsonNode fieldAnyNode = fieldNode.get("any"); - if (fieldAnyNode != null) { - String fieldAnyStr = fieldAnyNode.asText(); - if (StringUtils.isNotBlank(fieldAnyStr)) { - String[] fieldAnyArray = fieldAnyStr.split(","); - if (fieldAnyArray != null) { - for (String value : fieldAnyArray) { - if (StringUtils.isNotBlank(value)) { - fieldAnyList.add(value.trim()); - } - } - } - } - } - } - if (fieldNode.has("all")) { - JsonNode fieldAllNode = fieldNode.get("all"); - if (fieldAllNode != null) { - String fieldAllStr = fieldAllNode.asText(); - if (StringUtils.isNotBlank(fieldAllStr)) { - String[] fieldAllArray = fieldAllStr.split(","); - if (fieldAllArray != null) { - for (String value : fieldAllArray) { - if (StringUtils.isNotBlank(value)) { - fieldAllList.add(value.trim()); - } - } - } - } - } - } - if (fieldNode.has("not")) { - JsonNode fieldNotInNode = fieldNode.get("not"); - if (fieldNotInNode != null) { - String fieldNotInStr = fieldNotInNode.asText(); - if (StringUtils.isNotBlank(fieldNotInStr)) { - String[] fieldNotInArray = fieldNotInStr.split(","); - if (fieldNotInArray != null) { - for (String value : fieldNotInArray) { - if (StringUtils.isNotBlank(value)) { - fieldNotInList.add(value.trim()); - } - } - } - } - } - } - } - } - if (searchOpt.has("sources")) { - JsonNode sourcesNode = searchOpt.get("sources"); - if (sourcesNode != null) { - datasetSources = sourcesNode.asText(); - } - } - if (searchOpt.has("comments")) { - JsonNode commentsNode = searchOpt.get("comments"); - if (commentsNode != null) { - comments = commentsNode.asText(); - } - } - } - - List mustValueList = new ArrayList(); - List mustNotValueList = new ArrayList(); - List shouldValueList = new ArrayList(); - String hasChildValue = - "{\"has_child\": " + "{\"type\": \"$TYPE\", \"query\": {\"match\" : {\"$FIELD\" : \"$VALUE\"}}}}"; - String matchQueryUnit = "{\"query\": {\"match\" : {\"$FIELD\" : \"$VALUE\"}}}"; - - String shouldQueryUnit = - "{\"bool\": " + "{\"should\": [" + "{\"wildcard\": {\"$FIELD\": {\"value\": \"*$VALUE*\", \"boost\": 1}}}, " - + "{\"prefix\": {\"$FIELD\": {\"value\": \"$VALUE\", \"boost\": 4}}}, " - + "{\"term\": {\"$FIELD\": {\"value\": \"$VALUE\", \"boost\": 16}}}" + "]}}"; - String mustNotQueryUnit = "{\"term\" : {\"$FIELD\" : \"$VALUE\"}}"; - - if (scopeInList != null && scopeInList.size() > 0) { - for (String scope : scopeInList) { - if (StringUtils.isNotBlank(scope)) { - shouldValueList.add(Json.parse(shouldQueryUnit.replace("$FIELD", "parent_name"). - replace("$VALUE", scope.toLowerCase().trim()))); - } - } - ObjectNode shouldNode = Json.newObject(); - ObjectNode boolNode = Json.newObject(); - shouldNode.set("should", Json.toJson(shouldValueList)); - boolNode.put("bool", shouldNode); - mustValueList.add(boolNode); - } - if (scopeNotInList != null && scopeNotInList.size() > 0) { - shouldValueList.clear(); - for (String scope : scopeNotInList) { - if (StringUtils.isNotBlank(scope)) { - shouldValueList.add(Json.parse(mustNotQueryUnit.replace("$FIELD", "parent_name"). - replace("$VALUE", scope.toLowerCase().trim()))); - } - } - ObjectNode shouldNode = Json.newObject(); - ObjectNode boolNode = Json.newObject(); - shouldNode.set("should", Json.toJson(shouldValueList)); - boolNode.put("bool", shouldNode); - mustNotValueList.add(boolNode); - } - if (tableInList != null && tableInList.size() > 0) { - shouldValueList.clear(); - for (String table : tableInList) { - if (StringUtils.isNotBlank(table)) { - shouldValueList.add(Json.parse(shouldQueryUnit.replace("$FIELD", "name"). - replace("$VALUE", table.toLowerCase().trim()))); - } - } - ObjectNode shouldNode = Json.newObject(); - ObjectNode boolNode = Json.newObject(); - shouldNode.set("should", Json.toJson(shouldValueList)); - boolNode.put("bool", shouldNode); - mustValueList.add(boolNode); - } - if (tableNotInList != null && tableNotInList.size() > 0) { - shouldValueList.clear(); - for (String table : tableNotInList) { - if (StringUtils.isNotBlank(table)) { - shouldValueList.add(Json.parse(mustNotQueryUnit.replace("$FIELD", "name"). - replace("$VALUE", table.toLowerCase().trim()))); - } - } - ObjectNode shouldNode = Json.newObject(); - ObjectNode boolNode = Json.newObject(); - shouldNode.set("should", Json.toJson(shouldValueList)); - boolNode.put("bool", shouldNode); - mustNotValueList.add(boolNode); - } - if (fieldAnyList != null && fieldAnyList.size() > 0) { - shouldValueList.clear(); - for (String field : fieldAnyList) { - if (StringUtils.isNotBlank(field)) { - shouldValueList.add(Json.parse(hasChildValue.replace("$TYPE", "field"). - replace("$FIELD", "field_name"). - replace("$VALUE", field.toLowerCase().trim()))); - } - } - ObjectNode shouldNode = Json.newObject(); - ObjectNode boolNode = Json.newObject(); - shouldNode.set("should", Json.toJson(shouldValueList)); - boolNode.put("bool", shouldNode); - mustValueList.add(boolNode); - } - if (fieldAllList != null && fieldAllList.size() > 0) { - for (String field : fieldAllList) { - if (StringUtils.isNotBlank(field)) { - mustValueList.add(Json.parse(hasChildValue.replace("$TYPE", "field"). - replace("$FIELD", "field_name"). - replace("$VALUE", field.toLowerCase().trim()))); - } - } - } - if (fieldNotInList != null && fieldNotInList.size() > 0) { - shouldValueList.clear(); - for (String field : fieldNotInList) { - if (StringUtils.isNotBlank(field)) { - shouldValueList.add(Json.parse(hasChildValue.replace("$TYPE", "field"). - replace("$FIELD", "field_name"). - replace("$VALUE", field.toLowerCase().trim()))); - } - } - ObjectNode shouldNode = Json.newObject(); - ObjectNode boolNode = Json.newObject(); - shouldNode.set("should", Json.toJson(shouldValueList)); - boolNode.put("bool", shouldNode); - mustNotValueList.add(boolNode); - } - if (StringUtils.isNotBlank(comments)) { - shouldValueList.clear(); - String inputComments[] = comments.trim().split(","); - for (String comment : inputComments) { - if (StringUtils.isNotBlank(comment)) { - shouldValueList.add(Json.parse(hasChildValue.replace("$TYPE", "comment"). - replace("$FIELD", "text"). - replace("$VALUE", comment.toLowerCase().trim()))); - shouldValueList.add(Json.parse(hasChildValue.replace("$TYPE", "field"). - replace("$FIELD", "comments"). - replace("$VALUE", comment.toLowerCase().trim()))); - } - } - ObjectNode shouldNode = Json.newObject(); - ObjectNode boolNode = Json.newObject(); - shouldNode.set("should", Json.toJson(shouldValueList)); - boolNode.put("bool", shouldNode); - mustValueList.add(boolNode); - } - if (StringUtils.isNotBlank(datasetSources)) { - String inputSources[] = datasetSources.trim().split(","); - shouldValueList.clear(); - for (String source : inputSources) { - if (StringUtils.isNotBlank(source)) { - shouldValueList.add(Json.parse(matchQueryUnit.replace("$FIELD", "source"). - replace("$VALUE", source.toLowerCase().trim()))); - } - } - ObjectNode shouldNode = Json.newObject(); - ObjectNode boolNode = Json.newObject(); - shouldNode.set("should", Json.toJson(shouldValueList)); - boolNode.put("bool", shouldNode); - mustValueList.add(boolNode); - } - - ObjectNode boolNode = Json.newObject(); - ObjectNode queryNode = Json.newObject(); - if (mustValueList.size() > 0 && mustNotValueList.size() > 0) { - boolNode.set("must", Json.toJson(mustValueList)); - boolNode.set("must_not", Json.toJson(mustNotValueList)); - queryNode.put("bool", boolNode); - } else if (mustValueList.size() > 0) { - boolNode.set("must", Json.toJson(mustValueList)); - queryNode.put("bool", boolNode); - } else if (mustNotValueList.size() > 0) { - boolNode.set("must_not", Json.toJson(mustNotValueList)); - queryNode.put("bool", boolNode); - } - - return queryNode; - } - - public static ObjectNode generateMetricAdvSearchQueryString(JsonNode searchOpt) { - List dashboardInList = new ArrayList(); - List dashboardNotInList = new ArrayList(); - List groupInList = new ArrayList(); - List groupNotInList = new ArrayList(); - List categoryInList = new ArrayList(); - List categoryNotInList = new ArrayList(); - List metricInList = new ArrayList(); - List metricNotInList = new ArrayList(); - - if (searchOpt != null && (searchOpt.isContainerNode())) { - if (searchOpt.has("dashboard")) { - JsonNode dashboardNode = searchOpt.get("dashboard"); - if (dashboardNode != null && dashboardNode.isContainerNode()) { - if (dashboardNode.has("in")) { - JsonNode dashboardInNode = dashboardNode.get("in"); - if (dashboardInNode != null) { - String dashboardInStr = dashboardInNode.asText(); - if (StringUtils.isNotBlank(dashboardInStr)) { - String[] dashboardInArray = dashboardInStr.split(","); - if (dashboardInArray != null) { - for (String value : dashboardInArray) { - if (StringUtils.isNotBlank(value)) { - dashboardInList.add(value.trim()); - } - } - } - } - } - } - if (dashboardNode.has("not")) { - JsonNode dashboardNotInNode = dashboardNode.get("not"); - if (dashboardNotInNode != null) { - String dashboardNotInStr = dashboardNotInNode.asText(); - if (StringUtils.isNotBlank(dashboardNotInStr)) { - String[] dashboardNotInArray = dashboardNotInStr.split(","); - if (dashboardNotInArray != null) { - for (String value : dashboardNotInArray) { - if (StringUtils.isNotBlank(value)) { - dashboardNotInList.add(value.trim()); - } - } - } - } - } - } - } - } - - if (searchOpt.has("group")) { - JsonNode groupNode = searchOpt.get("group"); - if (groupNode != null && groupNode.isContainerNode()) { - if (groupNode.has("in")) { - JsonNode groupInNode = groupNode.get("in"); - if (groupInNode != null) { - String groupInStr = groupInNode.asText(); - if (StringUtils.isNotBlank(groupInStr)) { - String[] groupInArray = groupInStr.split(","); - if (groupInArray != null) { - for (String value : groupInArray) { - if (StringUtils.isNotBlank(value)) { - groupInList.add(value.trim()); - } - } - } - } - } - } - if (groupNode.has("not")) { - JsonNode groupNotInNode = groupNode.get("not"); - if (groupNotInNode != null) { - String groupNotInStr = groupNotInNode.asText(); - if (StringUtils.isNotBlank(groupNotInStr)) { - String[] groupNotInArray = groupNotInStr.split(","); - if (groupNotInArray != null) { - for (String value : groupNotInArray) { - if (StringUtils.isNotBlank(value)) { - groupNotInList.add(value.trim()); - } - } - } - } - } - } - } - } - - if (searchOpt.has("cat")) { - JsonNode categoryNode = searchOpt.get("cat"); - if (categoryNode != null && categoryNode.isContainerNode()) { - if (categoryNode.has("in")) { - JsonNode categoryInNode = categoryNode.get("in"); - if (categoryInNode != null) { - String categoryInStr = categoryInNode.asText(); - if (StringUtils.isNotBlank(categoryInStr)) { - String[] categoryInArray = categoryInStr.split(","); - if (categoryInArray != null) { - for (String value : categoryInArray) { - if (StringUtils.isNotBlank(value)) { - categoryInList.add(value.trim()); - } - } - } - } - } - } - if (categoryNode.has("not")) { - JsonNode categoryNotInNode = categoryNode.get("not"); - if (categoryNotInNode != null) { - String categoryNotInStr = categoryNotInNode.asText(); - if (StringUtils.isNotBlank(categoryNotInStr)) { - String[] categoryNotInArray = categoryNotInStr.split(","); - if (categoryNotInArray != null) { - for (String value : categoryNotInArray) { - if (StringUtils.isNotBlank(value)) { - categoryNotInList.add(value.trim()); - } - } - } - } - } - } - } - } - - if (searchOpt.has("metric")) { - JsonNode metricNode = searchOpt.get("metric"); - if (metricNode != null && metricNode.isContainerNode()) { - if (metricNode.has("in")) { - JsonNode metricInNode = metricNode.get("in"); - if (metricInNode != null) { - String metricInStr = metricInNode.asText(); - if (StringUtils.isNotBlank(metricInStr)) { - String[] metricInArray = metricInStr.split(","); - if (metricInArray != null) { - for (String value : metricInArray) { - if (StringUtils.isNotBlank(value)) { - metricInList.add(value.trim()); - } - } - } - } - } - } - if (metricNode.has("not")) { - JsonNode metricNotInNode = metricNode.get("not"); - if (metricNotInNode != null) { - String metricNotInStr = metricNotInNode.asText(); - if (StringUtils.isNotBlank(metricNotInStr)) { - String[] metricNotInArray = metricNotInStr.split(","); - if (metricNotInArray != null) { - for (String value : metricNotInArray) { - if (StringUtils.isNotBlank(value)) { - metricNotInList.add(value.trim()); - } - } - } - } - } - } - } - } - } - - List mustValueList = new ArrayList(); - List mustNotValueList = new ArrayList(); - List shouldValueList = new ArrayList(); - - String shouldQueryUnit = - "{\"bool\": " + "{\"should\": " + "[{\"wildcard\": {\"$FIELD\": {\"value\": \"*$VALUE*\", \"boost\": 1}}}, " - + "{\"prefix\": {\"$FIELD\": {\"value\": \"$VALUE\", \"boost\": 4}}}, " - + "{\"term\": {\"$FIELD\": {\"value\": \"$VALUE\", \"boost\": 16}}}" + "]" + "}" + "}"; - String shouldMatchQueryUnit = - "{\"bool\": " + "{\"should\": " + "[{\"wildcard\": {\"$FIELD\": {\"value\": \"*$VALUE*\", \"boost\": 1}}}, " - + "{\"prefix\": {\"$FIELD\": {\"value\": \"$VALUE\", \"boost\": 4}}}, " - + "{\"match\": {\"$FIELD\": {\"query\": \"$VALUE\", \"boost\": 16}}}" + "]" + "}" + "}"; - String mustNotQueryUnit = "{\"term\" : {\"$FIELD\" : \"$VALUE\"}}"; - - if (dashboardInList != null && dashboardInList.size() > 0) { - for (String dashboard : dashboardInList) { - if (StringUtils.isNotBlank(dashboard)) { - shouldValueList.add(Json.parse(shouldMatchQueryUnit.replace("$FIELD", "dashboard_name"). - replace("$VALUE", dashboard.toLowerCase().trim()))); - } - } - ObjectNode shouldNode = Json.newObject(); - ObjectNode boolNode = Json.newObject(); - shouldNode.set("should", Json.toJson(shouldValueList)); - boolNode.put("bool", shouldNode); - mustValueList.add(boolNode); - } - if (dashboardNotInList != null && dashboardNotInList.size() > 0) { - shouldValueList.clear(); - for (String dashboard : dashboardNotInList) { - if (StringUtils.isNotBlank(dashboard)) { - shouldValueList.add(Json.parse(mustNotQueryUnit.replace("$FIELD", "dashboard_name"). - replace("$VALUE", dashboard.toLowerCase().trim()))); - } - } - ObjectNode shouldNode = Json.newObject(); - ObjectNode boolNode = Json.newObject(); - shouldNode.set("should", Json.toJson(shouldValueList)); - boolNode.put("bool", shouldNode); - mustNotValueList.add(boolNode); - } - if (groupInList != null && groupInList.size() > 0) { - shouldValueList.clear(); - for (String group : groupInList) { - if (StringUtils.isNotBlank(group)) { - shouldValueList.add(Json.parse(shouldMatchQueryUnit.replace("$FIELD", "metric_group"). - replace("$VALUE", group.toLowerCase().trim()))); - } - } - ObjectNode shouldNode = Json.newObject(); - ObjectNode boolNode = Json.newObject(); - shouldNode.set("should", Json.toJson(shouldValueList)); - boolNode.put("bool", shouldNode); - mustValueList.add(boolNode); - } - if (groupNotInList != null && groupNotInList.size() > 0) { - shouldValueList.clear(); - for (String group : groupNotInList) { - if (StringUtils.isNotBlank(group)) { - shouldValueList.add(Json.parse(mustNotQueryUnit.replace("$FIELD", "metric_group"). - replace("$VALUE", group.toLowerCase().trim()))); - } - } - ObjectNode shouldNode = Json.newObject(); - ObjectNode boolNode = Json.newObject(); - shouldNode.set("should", Json.toJson(shouldValueList)); - boolNode.put("bool", shouldNode); - mustNotValueList.add(boolNode); - } - - if (categoryInList != null && categoryInList.size() > 0) { - shouldValueList.clear(); - for (String category : categoryInList) { - if (StringUtils.isNotBlank(category)) { - shouldValueList.add(Json.parse(shouldMatchQueryUnit.replace("$FIELD", "metric_category"). - replace("$VALUE", category.toLowerCase().trim()))); - } - } - ObjectNode shouldNode = Json.newObject(); - ObjectNode boolNode = Json.newObject(); - shouldNode.set("should", Json.toJson(shouldValueList)); - boolNode.put("bool", shouldNode); - mustValueList.add(boolNode); - } - if (categoryNotInList != null && categoryNotInList.size() > 0) { - shouldValueList.clear(); - for (String category : categoryNotInList) { - if (StringUtils.isNotBlank(category)) { - shouldValueList.add(Json.parse(mustNotQueryUnit.replace("$FIELD", "metric_category"). - replace("$VALUE", category.toLowerCase().trim()))); - } - } - ObjectNode shouldNode = Json.newObject(); - ObjectNode boolNode = Json.newObject(); - shouldNode.set("should", Json.toJson(shouldValueList)); - boolNode.put("bool", shouldNode); - mustNotValueList.add(boolNode); - } - - if (metricInList != null && metricInList.size() > 0) { - shouldValueList.clear(); - for (String name : metricInList) { - if (StringUtils.isNotBlank(name)) { - shouldValueList.add(Json.parse(shouldMatchQueryUnit.replace("$FIELD", "metric_name"). - replace("$VALUE", name.toLowerCase().trim()))); - } - } - ObjectNode shouldNode = Json.newObject(); - ObjectNode boolNode = Json.newObject(); - shouldNode.set("should", Json.toJson(shouldValueList)); - boolNode.put("bool", shouldNode); - mustValueList.add(boolNode); - } - if (metricNotInList != null && metricNotInList.size() > 0) { - shouldValueList.clear(); - for (String name : metricNotInList) { - if (StringUtils.isNotBlank(name)) { - shouldValueList.add(Json.parse(mustNotQueryUnit.replace("$FIELD", "metric_name"). - replace("$VALUE", name.toLowerCase().trim()))); - } - } - ObjectNode shouldNode = Json.newObject(); - ObjectNode boolNode = Json.newObject(); - shouldNode.set("should", Json.toJson(shouldValueList)); - boolNode.put("bool", shouldNode); - mustNotValueList.add(boolNode); - } - - ObjectNode boolNode = Json.newObject(); - ObjectNode queryNode = Json.newObject(); - if (mustValueList.size() > 0 && mustNotValueList.size() > 0) { - boolNode.set("must", Json.toJson(mustValueList)); - boolNode.set("must_not", Json.toJson(mustNotValueList)); - queryNode.put("bool", boolNode); - } else if (mustValueList.size() > 0) { - boolNode.set("must", Json.toJson(mustValueList)); - queryNode.put("bool", boolNode); - } else if (mustNotValueList.size() > 0) { - boolNode.set("must_not", Json.toJson(mustNotValueList)); - queryNode.put("bool", boolNode); - } - - return queryNode; - } - - public static ObjectNode generateFlowJobAdvSearchQueryString(JsonNode searchOpt) { - List appcodeInList = new ArrayList(); - List appcodeNotInList = new ArrayList(); - List flowInList = new ArrayList(); - List flowNotInList = new ArrayList(); - List jobInList = new ArrayList(); - List jobNotInList = new ArrayList(); - - if (searchOpt != null && (searchOpt.isContainerNode())) { - if (searchOpt.has("appcode")) { - JsonNode appcodeNode = searchOpt.get("appcode"); - if (appcodeNode != null && appcodeNode.isContainerNode()) { - if (appcodeNode.has("in")) { - JsonNode appcodeInNode = appcodeNode.get("in"); - if (appcodeInNode != null) { - String appcodeInStr = appcodeInNode.asText(); - if (StringUtils.isNotBlank(appcodeInStr)) { - String[] appcodeInArray = appcodeInStr.split(","); - if (appcodeInArray != null) { - for (String value : appcodeInArray) { - if (StringUtils.isNotBlank(value)) { - appcodeInList.add(value.trim()); - } - } - } - } - } - } - if (appcodeNode.has("not")) { - JsonNode appcodeNotInNode = appcodeNode.get("not"); - if (appcodeNotInNode != null) { - String appcodeNotInStr = appcodeNotInNode.asText(); - if (StringUtils.isNotBlank(appcodeNotInStr)) { - String[] appcodeNotInArray = appcodeNotInStr.split(","); - if (appcodeNotInArray != null) { - for (String value : appcodeNotInArray) { - if (StringUtils.isNotBlank(value)) { - appcodeNotInList.add(value.trim()); - } - } - } - } - } - } - } - } - - if (searchOpt.has("flow")) { - JsonNode flowNode = searchOpt.get("flow"); - if (flowNode != null && flowNode.isContainerNode()) { - if (flowNode.has("in")) { - JsonNode flowInNode = flowNode.get("in"); - if (flowInNode != null) { - String flowInStr = flowInNode.asText(); - if (StringUtils.isNotBlank(flowInStr)) { - String[] flowInArray = flowInStr.split(","); - if (flowInArray != null) { - for (String value : flowInArray) { - if (StringUtils.isNotBlank(value)) { - flowInList.add(value.trim()); - } - } - } - } - } - } - if (flowNode.has("not")) { - JsonNode flowNotInNode = flowNode.get("not"); - if (flowNotInNode != null) { - String flowNotInStr = flowNotInNode.asText(); - if (StringUtils.isNotBlank(flowNotInStr)) { - String[] flowNotInArray = flowNotInStr.split(","); - if (flowNotInArray != null) { - for (String value : flowNotInArray) { - if (StringUtils.isNotBlank(value)) { - flowNotInList.add(value.trim()); - } - } - } - } - } - } - } - } - - if (searchOpt.has("job")) { - JsonNode jobNode = searchOpt.get("job"); - if (jobNode != null && jobNode.isContainerNode()) { - if (jobNode.has("in")) { - JsonNode jobInNode = jobNode.get("in"); - if (jobInNode != null) { - String jobInStr = jobInNode.asText(); - if (StringUtils.isNotBlank(jobInStr)) { - String[] jobInArray = jobInStr.split(","); - if (jobInArray != null) { - for (String value : jobInArray) { - if (StringUtils.isNotBlank(value)) { - jobInList.add(value.trim()); - } - } - } - } - } - } - if (jobNode.has("not")) { - JsonNode jobNotInNode = jobNode.get("not"); - if (jobNotInNode != null) { - String jobNotInStr = jobNotInNode.asText(); - if (StringUtils.isNotBlank(jobNotInStr)) { - String[] jobNotInArray = jobNotInStr.split(","); - if (jobNotInArray != null) { - for (String value : jobNotInArray) { - if (StringUtils.isNotBlank(value)) { - jobNotInList.add(value.trim()); - } - } - } - } - } - } - } - } - } - - List mustValueList = new ArrayList(); - List mustNotValueList = new ArrayList(); - List shouldValueList = new ArrayList(); - - String shouldMatchQueryUnit = - "{\"bool\": " + "{\"should\": " + "[{\"wildcard\": {\"$FIELD\": {\"value\": \"*$VALUE*\", \"boost\": 1}}}, " - + "{\"prefix\": {\"$FIELD\": {\"value\": \"$VALUE\", \"boost\": 4}}}, " - + "{\"match\": {\"$FIELD\": {\"query\": \"$VALUE\", \"boost\": 16}}}" + "]" + "}" + "}"; - String mustNotQueryUnit = "{\"term\" : {\"$FIELD\" : \"$VALUE\"}}"; - - if (appcodeInList != null && appcodeInList.size() > 0) { - for (String appCode : appcodeInList) { - if (StringUtils.isNotBlank(appCode)) { - shouldValueList.add(Json.parse(shouldMatchQueryUnit.replace("$FIELD", "app_code"). - replace("$VALUE", appCode.toLowerCase().trim()))); - } - } - ObjectNode shouldNode = Json.newObject(); - ObjectNode boolNode = Json.newObject(); - shouldNode.set("should", Json.toJson(shouldValueList)); - boolNode.put("bool", shouldNode); - mustValueList.add(boolNode); - } - if (appcodeNotInList != null && appcodeNotInList.size() > 0) { - shouldValueList.clear(); - for (String appCode : appcodeNotInList) { - if (StringUtils.isNotBlank(appCode)) { - shouldValueList.add(Json.parse(mustNotQueryUnit.replace("$FIELD", "app_code"). - replace("$VALUE", appCode.toLowerCase().trim()))); - } - } - ObjectNode shouldNode = Json.newObject(); - ObjectNode boolNode = Json.newObject(); - shouldNode.set("should", Json.toJson(shouldValueList)); - boolNode.put("bool", shouldNode); - mustNotValueList.add(boolNode); - } - - if (flowInList != null && flowInList.size() > 0) { - shouldValueList.clear(); - for (String flow : flowInList) { - if (StringUtils.isNotBlank(flow)) { - shouldValueList.add(Json.parse(shouldMatchQueryUnit.replace("$FIELD", "flow_name"). - replace("$VALUE", flow.toLowerCase().trim()))); - } - } - ObjectNode shouldNode = Json.newObject(); - ObjectNode boolNode = Json.newObject(); - shouldNode.set("should", Json.toJson(shouldValueList)); - boolNode.put("bool", shouldNode); - mustValueList.add(boolNode); - } - if (flowNotInList != null && flowNotInList.size() > 0) { - shouldValueList.clear(); - for (String flow : flowNotInList) { - if (StringUtils.isNotBlank(flow)) { - shouldValueList.add(Json.parse(mustNotQueryUnit.replace("$FIELD", "flow_name"). - replace("$VALUE", flow.toLowerCase().trim()))); - } - } - ObjectNode shouldNode = Json.newObject(); - ObjectNode boolNode = Json.newObject(); - shouldNode.set("should", Json.toJson(shouldValueList)); - boolNode.put("bool", shouldNode); - mustNotValueList.add(boolNode); - } - - if (jobInList != null && jobInList.size() > 0) { - shouldValueList.clear(); - for (String job : jobInList) { - if (StringUtils.isNotBlank(job)) { - shouldValueList.add(Json.parse(shouldMatchQueryUnit.replace("$FIELD", "jobs.job_name"). - replace("$VALUE", job.toLowerCase().trim()))); - } - } - ObjectNode nestedNode = Json.newObject(); - ObjectNode queryNode = Json.newObject(); - ObjectNode shouldNode = Json.newObject(); - ObjectNode boolNode = Json.newObject(); - shouldNode.set("should", Json.toJson(shouldValueList)); - boolNode.put("bool", shouldNode); - queryNode.put("query", boolNode); - queryNode.put("path", "jobs"); - nestedNode.put("nested", queryNode); - - mustValueList.add(nestedNode); - } - if (jobNotInList != null && jobNotInList.size() > 0) { - shouldValueList.clear(); - for (String job : jobNotInList) { - if (StringUtils.isNotBlank(job)) { - shouldValueList.add(Json.parse(mustNotQueryUnit.replace("$FIELD", "jobs.job_name"). - replace("$VALUE", job.toLowerCase().trim()))); - } - } - ObjectNode nestedNode = Json.newObject(); - ObjectNode queryNode = Json.newObject(); - ObjectNode shouldNode = Json.newObject(); - ObjectNode boolNode = Json.newObject(); - shouldNode.set("should", Json.toJson(shouldValueList)); - boolNode.put("bool", shouldNode); - queryNode.put("query", boolNode); - queryNode.put("path", "jobs"); - nestedNode.put("nested", queryNode); - mustNotValueList.add(nestedNode); - } - - ObjectNode boolNode = Json.newObject(); - ObjectNode queryNode = Json.newObject(); - if (mustValueList.size() > 0 && mustNotValueList.size() > 0) { - boolNode.set("must", Json.toJson(mustValueList)); - boolNode.set("must_not", Json.toJson(mustNotValueList)); - queryNode.put("bool", boolNode); - } else if (mustValueList.size() > 0) { - boolNode.set("must", Json.toJson(mustValueList)); - queryNode.put("bool", boolNode); - } else if (mustNotValueList.size() > 0) { - boolNode.set("must_not", Json.toJson(mustNotValueList)); - queryNode.put("bool", boolNode); - } - - return queryNode; - } - - public static ObjectNode generateElasticSearchFilterString(String sources) { - if (StringUtils.isBlank(sources)) { - return null; - } - - List shouldValueList = new ArrayList(); - - String queryTemplate = readJsonQueryFile(WHZ_ELASTICSEARCH_FILTER_UNIT_FILE); - String[] values = sources.trim().split(","); - - for (String value : values) { - if (StringUtils.isNotBlank(value)) { - String query = queryTemplate.replace("$SOURCE", value.replace("\"", "").toLowerCase().trim()); - shouldValueList.add(Json.parse(query)); - } - } - - ObjectNode shouldNode = Json.newObject(); - shouldNode.set("should", Json.toJson(shouldValueList)); - ObjectNode queryNode = Json.newObject(); - queryNode.put("bool", shouldNode); - return queryNode; - } -} \ No newline at end of file diff --git a/wherehows-frontend/conf/routes b/wherehows-frontend/conf/routes index e6fe111139..70d5075512 100644 --- a/wherehows-frontend/conf/routes +++ b/wherehows-frontend/conf/routes @@ -54,10 +54,6 @@ GET /api/v1/autocomplete/search GET /api/v1/autocomplete/datasets controllers.api.v1.Search.getSearchAutoCompleteForDataset() -GET /api/v1/autocomplete/metrics controllers.api.v1.Search.getSearchAutoCompleteForMetric() - -GET /api/v1/autocomplete/flows controllers.api.v1.Search.getSearchAutoCompleteForFlow() - GET /api/v1/list/datasets controllers.api.v1.Dataset.getDatasetListNodes() GET /api/v1/list/metrics controllers.api.v1.Metric.getMetricListViewDashboards() @@ -190,30 +186,6 @@ GET /api/v1/flows/:application/:project GET /api/v1/flow/:application/:flowId controllers.api.v1.Flow.getPagedJobs(application: String, flowId: Long) -GET /api/v1/advsearch/sources controllers.api.v1.AdvSearch.getDatasetSources() - -GET /api/v1/advsearch/scopes controllers.api.v1.AdvSearch.getDatasetScopes() - -GET /api/v1/advsearch/tables controllers.api.v1.AdvSearch.getDatasetTableNames() - -GET /api/v1/advsearch/fields controllers.api.v1.AdvSearch.getDatasetFields() - -GET /api/v1/advsearch/appcodes controllers.api.v1.AdvSearch.getFlowApplicationCodes() - -GET /api/v1/advsearch/flowNames controllers.api.v1.AdvSearch.getFlowNames() - -GET /api/v1/advsearch/jobNames controllers.api.v1.AdvSearch.getJobNames() - -GET /api/v1/advsearch/dashboards controllers.api.v1.AdvSearch.getDashboardNames() - -GET /api/v1/advsearch/metricGroups controllers.api.v1.AdvSearch.getMetricGroups() - -GET /api/v1/advsearch/metricCategories controllers.api.v1.AdvSearch.getMetricCategories() - -GET /api/v1/advsearch/metricNames controllers.api.v1.AdvSearch.getMetricNames() - -GET /api/v1/advsearch/search controllers.api.v1.AdvSearch.search() - GET /api/v1/lineage/dataset/:id controllers.api.v1.Lineage.getDatasetLineageGraphData(id:Int) GET /api/v1/lineage/flow/:application/:project/:flowId controllers.api.v1.Lineage.getFlowLineageGraphData(application: String, project: String, flowId: Long)