diff --git a/wherehows-dao/build.gradle b/wherehows-dao/build.gradle index 4100d6d440..1509772d6c 100644 --- a/wherehows-dao/build.gradle +++ b/wherehows-dao/build.gradle @@ -6,6 +6,7 @@ dependencies { compile externalDependency.commons_lang3 compile externalDependency.slf4j_api compile externalDependency.spring_jdbc + compile externalDependency.http_client compile externalDependency.jackson_databind compile externalDependency.jsr305 compile externalDependency.guava @@ -15,9 +16,6 @@ dependencies { runtime externalDependency.hikaricp testCompile externalDependency.testng - - compile externalDependency.play_java_ws - } findbugs { diff --git a/wherehows-dao/src/main/java/wherehows/dao/table/SearchDao.java b/wherehows-dao/src/main/java/wherehows/dao/table/SearchDao.java index 85a6226a5d..0a4dd4ac72 100644 --- a/wherehows-dao/src/main/java/wherehows/dao/table/SearchDao.java +++ b/wherehows-dao/src/main/java/wherehows/dao/table/SearchDao.java @@ -11,334 +11,267 @@ * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */ - package wherehows.dao.table; -import java.util.*; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; +import java.io.IOException; import java.util.ArrayList; import java.util.HashSet; import java.util.Iterator; import java.util.List; -import play.Logger; -import play.Play; -import play.libs.F.Promise; -import play.libs.Json; -import play.libs.ws.*; +import java.util.Set; +import lombok.extern.slf4j.Slf4j; import wherehows.models.table.Dataset; - +import wherehows.util.HttpUtil; import static wherehows.util.Search.*; -public class SearchDao -{ - public static String ELASTICSEARCH_DATASET_URL_KEY = "elasticsearch.dataset.url"; - public static String WHEREHOWS_SEARCH_ENGINE_KEY = "search.engine"; - public List getAutoCompleteList(String input, int limit) - { - List names = new ArrayList<>(); - names.addAll(getAutoCompleteListDataset(input, limit / 3)); - return names; - } +@Slf4j +public class SearchDao { - public List getAutoCompleteListDataset(String input, int limit) - { - String elasticSearchTypeURLKey = "elasticsearch.dataset.url"; - String fieldName = "name_suggest"; - return getAutoCompleteListbyES(elasticSearchTypeURLKey,fieldName,input,limit); - } + private static final ObjectMapper _OM = new ObjectMapper(); - public List getAutoCompleteListbyES(String elasticSearchTypeURLKey, String fieldName, String input, - int limit) - { - // use elastic search completion suggester, ES will validate the input and limit - List completionSuggestionList = new ArrayList(); - Set completionSuggestionSet = new HashSet(); + public List getAutoCompleteList(String elasticSearchUrl, String input, int limit) { + return getAutoCompleteListDataset(elasticSearchUrl, input, limit / 3); + } - JsonNode responseNode = null; - ObjectNode keywordNode = null; + public List getAutoCompleteListDataset(String elasticSearchUrl, String input, int limit) { + String fieldName = "name_suggest"; + return getAutoCompleteListbyES(elasticSearchUrl, fieldName, input, limit); + } - try { - keywordNode = generateElasticSearchCompletionSuggesterQuery(fieldName, input, limit); - } catch (Exception e) { - Logger.error("Elastic search completion suggester error. Error message :" + e.getMessage()); - } + public List getAutoCompleteListbyES(String elasticSearchUrl, String fieldName, String input, int limit) { + // use elastic search completion suggester, ES will validate the input and limit + List completionSuggestionList = new ArrayList<>(); + Set completionSuggestionSet = new HashSet<>(); - Logger.info("The completion suggester query sent to Elastic Search was: " + keywordNode.toString()); + ObjectNode keywordNode = generateElasticSearchCompletionSuggesterQuery(fieldName, input, limit); + log.info("The completion suggester query sent to Elastic Search was: " + keywordNode); - Promise responsePromise = - WS.url(Play.application().configuration().getString(elasticSearchTypeURLKey)).post(keywordNode); - responseNode = responsePromise.get(1000).asJson(); + JsonNode responseNode = null; + try { + responseNode = HttpUtil.httpPostRequest(elasticSearchUrl, keywordNode); + } catch (IOException ex) { + log.error("ES suggetion list query error: {}" + ex.toString()); + } + if (responseNode == null || !responseNode.isContainerNode()) { + return completionSuggestionList; + } - if (responseNode == null || !responseNode.isContainerNode()) { - return completionSuggestionList; - } + JsonNode suggestNode = responseNode.get("suggest"); + if (suggestNode == null || !suggestNode.has("wh-suggest")) { + log.error("Elastic search completion suggester response does not contain suggest node"); + return completionSuggestionList; + } - JsonNode suggestNode = responseNode.get("suggest"); - if (suggestNode == null || !suggestNode.has("wh-suggest")) { - Logger.error("Elastic search completion suggester response does not contain suggest node"); - return completionSuggestionList; - } + JsonNode whSuggestNode = suggestNode.get("wh-suggest"); + if (whSuggestNode == null || !whSuggestNode.isArray()) { + log.error("Elastic search completion suggester response does not contain wh-suggest node"); + return completionSuggestionList; + } - JsonNode whSuggestNode = suggestNode.get("wh-suggest"); - if (whSuggestNode == null || !whSuggestNode.isArray()) { - Logger.error("Elastic search completion suggester response does not contain wh-suggest node"); - return completionSuggestionList; - } + Iterator arrayIterator = whSuggestNode.elements(); + if (arrayIterator == null) { + return completionSuggestionList; + } - Iterator arrayIterator = whSuggestNode.elements(); - if (arrayIterator == null) { - return completionSuggestionList; - } + while (arrayIterator.hasNext()) { + JsonNode node = arrayIterator.next(); + if (!node.isContainerNode() || !node.has("options")) { + continue; + } - while (arrayIterator.hasNext()) { - JsonNode node = arrayIterator.next(); - if (!node.isContainerNode() || !node.has("options")) { - continue; - } + JsonNode optionsNode = node.get("options"); + if (optionsNode == null || !optionsNode.isArray()) { + continue; + } - JsonNode optionsNode = node.get("options"); - if (optionsNode == null || !optionsNode.isArray()) { - continue; - } + Iterator arrayIteratorOptions = optionsNode.elements(); + if (arrayIteratorOptions == null) { + continue; + } - Iterator arrayIteratorOptions = optionsNode.elements(); - if (arrayIteratorOptions == null) { - continue; - } + while (arrayIteratorOptions.hasNext()) { + JsonNode textNode = arrayIteratorOptions.next(); + if (textNode == null || !textNode.has("text")) { + continue; + } + String oneSuggestion = textNode.get("text").asText(); + completionSuggestionSet.add(oneSuggestion); + } + } - while (arrayIteratorOptions.hasNext()) { - JsonNode textNode = arrayIteratorOptions.next(); - if (textNode == null || !textNode.has("text")) { - continue; - } - String oneSuggestion = textNode.get("text").asText(); - completionSuggestionSet.add(oneSuggestion); - } - } + completionSuggestionList.addAll(completionSuggestionSet); + log.info("Returned suggestion list is: " + completionSuggestionList); + return completionSuggestionList; + } - completionSuggestionList.addAll(completionSuggestionSet); - Logger.info("Returned suggestion list is: " + completionSuggestionList); - return completionSuggestionList; - } + // this is for did you mean feature + public List getSuggestionList(String elasticSearchUrl, String category, String searchKeyword) { + List suggestionList = new ArrayList<>(); + String fieldName = "name"; - // this is for did you mean feature - public static List getSuggestionList(String category, String searchKeyword) - { - List SuggestionList = new ArrayList(); - String elasticSearchType = "dataset"; - String elasticSearchTypeURLKey = "elasticsearch.dataset.url"; - String fieldName = "name"; + ObjectNode keywordNode = generateElasticSearchPhraseSuggesterQuery(fieldName, searchKeyword); - JsonNode responseNode = null; - ObjectNode keywordNode = null; + log.info("The suggest query sent to Elastic Search is: " + keywordNode); - try { - String lCategory = category.toLowerCase(); - Logger.info("lCategory is " + category); + JsonNode responseNode = null; + try { + responseNode = HttpUtil.httpPostRequest(elasticSearchUrl, keywordNode); + } catch (IOException ex) { + log.error("ES suggetion list query error: {}" + ex.toString()); + } - // ToDO: deprecate category or reuse for entity - switch (lCategory) { - case "dataset": - elasticSearchType = "dataset"; - elasticSearchTypeURLKey = "elasticsearch.dataset.url"; - fieldName = "name"; - break; - default: - break; - } + if (responseNode == null || !responseNode.isContainerNode() || !responseNode.has("hits")) { + return suggestionList; + } - keywordNode = generateElasticSearchPhraseSuggesterQuery(elasticSearchType, fieldName, searchKeyword); - } catch (Exception e) { - Logger.error("Elastic search phrase suggester error. Error message :" + e.getMessage()); - } + JsonNode suggestNode = responseNode.get("suggest"); + log.info("suggestNode is " + suggestNode); - Logger.info("The suggest query sent to Elastic Search is: " + keywordNode.toString()); + if (suggestNode == null || !suggestNode.has("simple_phrase")) { + return suggestionList; + } - Promise responsePromise = - WS.url(Play.application().configuration().getString(elasticSearchTypeURLKey)).post(keywordNode); - responseNode = responsePromise.get(1000).asJson(); + JsonNode simplePhraseNode = suggestNode.get("simple_phrase"); + if (simplePhraseNode == null || !simplePhraseNode.isArray()) { + return suggestionList; + } - if (responseNode == null || !responseNode.isContainerNode() || !responseNode.has("hits")) { - return SuggestionList; - } + Iterator arrayIterator = simplePhraseNode.elements(); + if (arrayIterator == null) { + return suggestionList; + } - JsonNode suggestNode = responseNode.get("suggest"); - Logger.info("suggestNode is " + suggestNode.toString()); + while (arrayIterator.hasNext()) { + JsonNode node = arrayIterator.next(); + if (!node.isContainerNode() || !node.has("options")) { + continue; + } - if (suggestNode == null || !suggestNode.has("simple_phrase")) { - return SuggestionList; - } + JsonNode optionsNode = node.get("options"); + if (optionsNode == null || !optionsNode.isArray()) { + continue; + } - JsonNode simplePhraseNode = suggestNode.get("simple_phrase"); - if (simplePhraseNode == null || !simplePhraseNode.isArray()) { - return SuggestionList; - } + Iterator arrayIteratorOptions = optionsNode.elements(); + if (arrayIteratorOptions == null) { + continue; + } - Iterator arrayIterator = simplePhraseNode.elements(); - if (arrayIterator == null) { - return SuggestionList; - } + while (arrayIteratorOptions.hasNext()) { + JsonNode textNode = arrayIteratorOptions.next(); + if (textNode == null || !textNode.has("text")) { + continue; + } + String oneSuggestion = textNode.get("text").asText(); + suggestionList.add(oneSuggestion); + } + } - while (arrayIterator.hasNext()) { - JsonNode node = arrayIterator.next(); - if (!node.isContainerNode() || !node.has("options")) { - continue; - } + return suggestionList; + } - JsonNode optionsNode = node.get("options"); - if (optionsNode == null || !optionsNode.isArray()) { - continue; - } + public JsonNode elasticSearchDatasetByKeyword(String elasticSearchUrl, String category, String keywords, + String source, int page, int size) { + ObjectNode queryNode = _OM.createObjectNode(); + queryNode.put("from", (page - 1) * size); + queryNode.put("size", size); + JsonNode responseNode = null; - Iterator arrayIteratorOptions = optionsNode.elements(); - if (arrayIteratorOptions == null) { - continue; - } + try { + ObjectNode keywordNode = generateElasticSearchQueryString(category, source, keywords); - while (arrayIteratorOptions.hasNext()) { - JsonNode textNode = arrayIteratorOptions.next(); - if (textNode == null || !textNode.has("text")) { - continue; - } - String oneSuggestion = textNode.get("text").asText(); - SuggestionList.add(oneSuggestion); - } - } + ObjectNode fieldValueFactorNode = _OM.createObjectNode(); + fieldValueFactorNode.put("field", "static_boosting_score"); + fieldValueFactorNode.put("factor", 1); + fieldValueFactorNode.put("modifier", "square"); + fieldValueFactorNode.put("missing", 1); - return SuggestionList; - } + ObjectNode funcScoreNodes = _OM.createObjectNode(); + funcScoreNodes.put("query", keywordNode); + funcScoreNodes.put("field_value_factor", fieldValueFactorNode); - public JsonNode elasticSearchDatasetByKeyword( - String category, - String keywords, - String source, - int page, - int size) - { - ObjectNode queryNode = Json.newObject(); - queryNode.put("from", (page-1)*size); - queryNode.put("size", size); - JsonNode responseNode = null; - ObjectNode keywordNode = null; + ObjectNode funcScoreNodesWrapper = _OM.createObjectNode(); + funcScoreNodesWrapper.put("function_score", funcScoreNodes); - try { - keywordNode = generateElasticSearchQueryString(category, source, keywords); - } catch (Exception e) { - Logger.error("Elastic search dataset input query is not JSON format. Error message :" + e.getMessage()); - } + queryNode.put("query", funcScoreNodesWrapper); - if (keywordNode != null) { - ObjectNode funcScoreNodes = Json.newObject(); + ObjectNode filterNode = _OM.createObjectNode(); + try { + filterNode = generateElasticSearchFilterString(source); + } catch (Exception e) { + log.error("Elastic search filter query node generation failed :" + e.getMessage()); + } - ObjectNode fieldValueFactorNode = Json.newObject(); - fieldValueFactorNode.put("field", "static_boosting_score"); - fieldValueFactorNode.put("factor", 1); - fieldValueFactorNode.put("modifier", "square"); - fieldValueFactorNode.put("missing", 1); + if (filterNode != null) { + queryNode.put("post_filter", filterNode); + } - funcScoreNodes.put("query", keywordNode); - funcScoreNodes.put("field_value_factor", fieldValueFactorNode); + log.info(" === elasticSearchDatasetByKeyword === The query sent to Elastic Search is: " + queryNode.toString()); - ObjectNode funcScoreNodesWrapper = Json.newObject(); - funcScoreNodesWrapper.put("function_score", funcScoreNodes); + responseNode = HttpUtil.httpPostRequest(elasticSearchUrl, queryNode); + // Logger.debug("The responseNode from Elastic Search is: " + responseNode.toString()); + } catch (IOException e) { + log.error("Elastic search dataset query error: {}", e.toString()); + } - queryNode.put("query", funcScoreNodesWrapper); + ObjectNode resultNode = _OM.createObjectNode(); + long count = 0L; + List pagedDatasets = new ArrayList<>(); + resultNode.put("page", page); + resultNode.put("category", category); + resultNode.put("source", source); + resultNode.put("itemsPerPage", size); + resultNode.put("keywords", keywords); - ObjectNode filterNode = Json.newObject(); - try { - filterNode = generateElasticSearchFilterString(source); - } catch (Exception e) { - Logger.error("Elastic search filter query node generation failed :" + e.getMessage()); - } - - if (filterNode != null) { - queryNode.put("post_filter", filterNode); - } - - Logger.info( - " === elasticSearchDatasetByKeyword === The query sent to Elastic Search is: " + queryNode.toString()); - - Promise responsePromise = - WS.url(Play.application().configuration().getString(SearchDao.ELASTICSEARCH_DATASET_URL_KEY)).post(queryNode); - responseNode = responsePromise.get(1000).asJson(); - - // Logger.debug("The responseNode from Elastic Search is: " + responseNode.toString()); - - } - - ObjectNode resultNode = Json.newObject(); - Long count = 0L; - List pagedDatasets = new ArrayList<>(); - resultNode.put("page", page); - resultNode.put("category", category); - resultNode.put("source", source); - resultNode.put("itemsPerPage", size); - resultNode.put("keywords", keywords); - - if (responseNode != null && responseNode.isContainerNode() && responseNode.has("hits")) - { - JsonNode hitsNode = responseNode.get("hits"); - if (hitsNode != null) - { - if (hitsNode.has("total")) - { - count = hitsNode.get("total").asLong(); - } - if (hitsNode.has("hits")) - { - JsonNode dataNode = hitsNode.get("hits"); - if (dataNode != null && dataNode.isArray()) - { - Iterator arrayIterator = dataNode.elements(); - if (arrayIterator != null) - { - while (arrayIterator.hasNext()) - { - JsonNode node = arrayIterator.next(); - if (node.isContainerNode() && node.has("_id")) - { - Dataset dataset = new Dataset(); - dataset.id = node.get("_id").asLong(); - if (node.has("_source")) - { - JsonNode sourceNode = node.get("_source"); - if (sourceNode != null) - { - if (sourceNode.has("name")) - { - dataset.name = sourceNode.get("name").asText(); - } - if (sourceNode.has("source")) - { - dataset.source = sourceNode.get("source").asText(); - } - if (sourceNode.has("urn")) - { - dataset.urn = sourceNode.get("urn").asText(); - } - if (sourceNode.has("schema")) - { - dataset.schema = sourceNode.get("schema").asText(); - } - } - } - pagedDatasets.add(dataset); - } - } - } - - } - } - - } - } - resultNode.put("count", count); - resultNode.put("totalPages", (int)Math.ceil(count/((double)size))); - resultNode.set("data", Json.toJson(pagedDatasets)); - return resultNode; - } - -} \ No newline at end of file + if (responseNode != null && responseNode.isContainerNode() && responseNode.has("hits")) { + JsonNode hitsNode = responseNode.get("hits"); + if (hitsNode != null) { + if (hitsNode.has("total")) { + count = hitsNode.get("total").asLong(); + } + if (hitsNode.has("hits")) { + JsonNode dataNode = hitsNode.get("hits"); + if (dataNode != null && dataNode.isArray()) { + Iterator arrayIterator = dataNode.elements(); + if (arrayIterator != null) { + while (arrayIterator.hasNext()) { + JsonNode node = arrayIterator.next(); + if (node.isContainerNode() && node.has("_id")) { + Dataset dataset = new Dataset(); + dataset.id = node.get("_id").asLong(); + if (node.has("_source")) { + JsonNode sourceNode = node.get("_source"); + if (sourceNode != null) { + if (sourceNode.has("name")) { + dataset.name = sourceNode.get("name").asText(); + } + if (sourceNode.has("source")) { + dataset.source = sourceNode.get("source").asText(); + } + if (sourceNode.has("urn")) { + dataset.urn = sourceNode.get("urn").asText(); + } + if (sourceNode.has("schema")) { + dataset.schema = sourceNode.get("schema").asText(); + } + } + } + pagedDatasets.add(dataset); + } + } + } + } + } + } + } + resultNode.put("count", count); + resultNode.put("totalPages", (int) Math.ceil(count / ((double) size))); + resultNode.set("data", _OM.valueToTree(pagedDatasets)); + return resultNode; + } +} diff --git a/wherehows-dao/src/main/java/wherehows/util/HttpClientFactory.java b/wherehows-dao/src/main/java/wherehows/util/HttpClientFactory.java new file mode 100644 index 0000000000..d521065498 --- /dev/null +++ b/wherehows-dao/src/main/java/wherehows/util/HttpClientFactory.java @@ -0,0 +1,41 @@ +/** + * Copyright 2015 LinkedIn Corp. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + */ +package wherehows.util; + +import org.apache.http.client.config.RequestConfig; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; + + +public class HttpClientFactory { + + private static final int DEFAULT_TIMEOUT = 1000; // default 1s timeout + + private HttpClientFactory() { + } + + public static CloseableHttpClient createHttpClient() { + return createHttpClient(DEFAULT_TIMEOUT); + } + + public static CloseableHttpClient createHttpClient(int timeoutMillis) { + RequestConfig requestConfig = RequestConfig.custom() + .setConnectTimeout(timeoutMillis) + .setConnectionRequestTimeout(timeoutMillis) + .setSocketTimeout(timeoutMillis) + .build(); + + return HttpClientBuilder.create().setDefaultRequestConfig(requestConfig).build(); + } +} diff --git a/wherehows-dao/src/main/java/wherehows/util/HttpUtil.java b/wherehows-dao/src/main/java/wherehows/util/HttpUtil.java new file mode 100644 index 0000000000..71b0829035 --- /dev/null +++ b/wherehows-dao/src/main/java/wherehows/util/HttpUtil.java @@ -0,0 +1,53 @@ +/** + * Copyright 2015 LinkedIn Corp. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + */ +package wherehows.util; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import lombok.extern.slf4j.Slf4j; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.util.EntityUtils; + + +@Slf4j +public class HttpUtil { + + private static final ObjectMapper _OM = new ObjectMapper(); + + private HttpUtil() { + } + + public static JsonNode httpPostRequest(String url, JsonNode content) throws IOException { + CloseableHttpClient client = HttpClientFactory.createHttpClient(); + HttpPost httpPost = new HttpPost(url); + + httpPost.setEntity(new StringEntity(content.toString())); + httpPost.setHeader("Accept", "application/json"); + httpPost.setHeader("Content-type", "application/json"); + + CloseableHttpResponse response = client.execute(httpPost); + if (response.getStatusLine().getStatusCode() != 200) { + log.error("ES request fail: {}", response.getStatusLine().getStatusCode()); + return null; + } + JsonNode respJson = _OM.readTree(EntityUtils.toString(response.getEntity())); + client.close(); + + return respJson; + } +} diff --git a/wherehows-dao/src/main/java/wherehows/util/Search.java b/wherehows-dao/src/main/java/wherehows/util/Search.java index e533d4527c..a79ca63764 100644 --- a/wherehows-dao/src/main/java/wherehows/util/Search.java +++ b/wherehows-dao/src/main/java/wherehows/util/Search.java @@ -16,20 +16,28 @@ package wherehows.util; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; -import java.nio.file.Paths; -import java.nio.file.Files; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; + @Slf4j public class Search { - public final static String DATASET_CATEGORY = "datasets"; - public final static String METRIC_CATEGORY = "metrics"; - public final static String COMMENT_CATEGORY = "comments"; - public final static String FLOW_CATEGORY = "flows"; - public final static String JOB_CATEGORY = "jobs"; + + private Search() { + } + + private static final ObjectMapper _OM = new ObjectMapper(); + + private final static String DATASET_CATEGORY = "datasets"; + private final static String METRIC_CATEGORY = "metrics"; + private final static String COMMENT_CATEGORY = "comments"; + private final static String FLOW_CATEGORY = "flows"; + private final static String JOB_CATEGORY = "jobs"; private static final String WHZ_ELASTICSEARCH_DATASET_QUERY_FILE = System.getenv("WHZ_ELASTICSEARCH_DATASET_QUERY_TEMPLATE"); private static final String WHZ_ELASTICSEARCH_METRIC_QUERY_FILE = System.getenv("WHZ_ELASTICSEARCH_METRIC_QUERY_TEMPLATE"); @@ -41,10 +49,8 @@ public class Search { public static String readJsonQueryFile(String jsonFile) { try { - ObjectMapper objectMapper = new ObjectMapper(); - String contents = new String(Files.readAllBytes(Paths.get(jsonFile))); - JsonNode json = objectMapper.readTree(contents); + JsonNode json = _OM.readTree(contents); return json.toString(); } catch (Exception e) { log.error("ReadJsonQueryFile failed. Error: " + e.getMessage()); @@ -68,10 +74,10 @@ public class Search { query = query.replace("$LIMIT", Integer.toString(limit)); - ObjectNode suggestNode = new ObjectMapper().createObjectNode(); + ObjectNode suggestNode = _OM.createObjectNode(); try { - ObjectNode textNode = (ObjectNode) new ObjectMapper().readTree(query); + ObjectNode textNode = (ObjectNode) _OM.readTree(query); suggestNode.putPOJO("suggest", textNode); } catch (Exception e) { log.error("suggest Exception = " + e.getMessage()); @@ -81,8 +87,7 @@ public class Search { return suggestNode; } - public static ObjectNode generateElasticSearchPhraseSuggesterQuery(String category, String field, - String searchKeyword) { + public static ObjectNode generateElasticSearchPhraseSuggesterQuery(String field, String searchKeyword) { if (StringUtils.isBlank(searchKeyword)) { return null; } @@ -94,9 +99,9 @@ public class Search { query = query.replace("$FIELD", field.toLowerCase()); } - ObjectNode suggestNode = new ObjectMapper().createObjectNode(); + ObjectNode suggestNode = _OM.createObjectNode(); try { - ObjectNode textNode = (ObjectNode) new ObjectMapper().readTree(query); + ObjectNode textNode = (ObjectNode) _OM.readTree(query); suggestNode.putPOJO("suggest", textNode); } catch (Exception e) { log.error("suggest Exception = " + e.getMessage()); @@ -105,8 +110,8 @@ public class Search { return suggestNode; } - public static ObjectNode generateElasticSearchQueryString(String category, String source, String keywords) throws Exception - { + public static ObjectNode generateElasticSearchQueryString(String category, String source, String keywords) + throws IOException { if (StringUtils.isBlank(keywords)) { return null; } @@ -127,25 +132,22 @@ public class Search { } } - ObjectMapper objectMapper = new ObjectMapper(); - for (String value : values) { if (StringUtils.isNotBlank(value)) { String query = queryTemplate.replace("$VALUE", value.replace("\"", "").toLowerCase().trim()); - shouldValueList.add(objectMapper.readTree(query)); + shouldValueList.add(_OM.readTree(query)); } } - ObjectNode shouldNode = new ObjectMapper().createObjectNode(); + ObjectNode shouldNode = _OM.createObjectNode(); shouldNode.putPOJO("should", shouldValueList); - ObjectNode queryNode = new ObjectMapper().createObjectNode(); + ObjectNode queryNode = _OM.createObjectNode(); queryNode.putPOJO("bool", shouldNode); return queryNode; } - - public static ObjectNode generateElasticSearchFilterString(String sources) throws Exception{ + public static ObjectNode generateElasticSearchFilterString(String sources) throws IOException { if (StringUtils.isBlank(sources)) { return null; } @@ -155,18 +157,16 @@ public class Search { String queryTemplate = readJsonQueryFile(WHZ_ELASTICSEARCH_FILTER_UNIT_FILE); String[] values = sources.trim().split(","); - ObjectMapper objectMapper = new ObjectMapper(); - for (String value : values) { if (StringUtils.isNotBlank(value)) { String query = queryTemplate.replace("$SOURCE", value.replace("\"", "").toLowerCase().trim()); - shouldValueList.add(objectMapper.readTree(query)); + shouldValueList.add(_OM.readTree(query)); } } - ObjectNode shouldNode = new ObjectMapper().createObjectNode(); + ObjectNode shouldNode = _OM.createObjectNode(); shouldNode.putPOJO("should", shouldValueList); - ObjectNode queryNode = new ObjectMapper().createObjectNode(); + ObjectNode queryNode = _OM.createObjectNode(); queryNode.putPOJO("bool", shouldNode); return queryNode; diff --git a/wherehows-frontend/app/controllers/api/v1/Search.java b/wherehows-frontend/app/controllers/api/v1/Search.java index 1f2ba6dafd..74ff8d4f0c 100644 --- a/wherehows-frontend/app/controllers/api/v1/Search.java +++ b/wherehows-frontend/app/controllers/api/v1/Search.java @@ -23,13 +23,19 @@ import play.cache.Cache; import play.libs.Json; import play.mvc.Controller; import play.mvc.Result; +import wherehows.dao.table.SearchDao; import static org.apache.commons.lang3.StringUtils.*; -import wherehows.dao.table.SearchDao; - public class Search extends Controller { + + private static final String ELASTICSEARCH_DATASET_URL_KEY = "elasticsearch.dataset.url"; + private static final String ELASTICSEARCH_DATASET_URL = + Play.application().configuration().getString(ELASTICSEARCH_DATASET_URL_KEY); + private static final String WHEREHOWS_SEARCH_ENGINE_KEY = "search.engine"; // TODO: deprecated this setting + private static final String SEARCH_ENGINE = Play.application().configuration().getString(WHEREHOWS_SEARCH_ENGINE_KEY); + private static final String AUTOCOMPLETE_ALL_KEY = "autocomplete.all"; private static final String AUTOCOMPLETE_DATASET_KEY = "autocomplete.dataset"; private static final int DEFAULT_AUTOCOMPLETE_SIZE = 20; @@ -48,7 +54,7 @@ public class Search extends Controller { String cacheKey = AUTOCOMPLETE_ALL_KEY + (isNotBlank(input) ? "." + input : "-all"); List names = (List) Cache.get(cacheKey); if (names == null || names.size() == 0) { - names = SEARCH_DAO.getAutoCompleteList(input, size); + names = SEARCH_DAO.getAutoCompleteList(ELASTICSEARCH_DATASET_URL, input, size); Cache.set(cacheKey, names, DEFAULT_AUTOCOMPLETE_CACHE_TIME); } @@ -70,7 +76,7 @@ public class Search extends Controller { String cacheKey = AUTOCOMPLETE_DATASET_KEY + (isNotBlank(input) ? "." + input : "-all"); List names = (List) Cache.get(cacheKey); if (names == null || names.size() == 0) { - names = SEARCH_DAO.getAutoCompleteListDataset(input, size); + names = SEARCH_DAO.getAutoCompleteListDataset(ELASTICSEARCH_DATASET_URL, input, size); Cache.set(cacheKey, names, DEFAULT_AUTOCOMPLETE_CACHE_TIME); } @@ -81,8 +87,6 @@ public class Search extends Controller { return ok(result); } - - public static Result searchByKeyword() { ObjectNode result = Json.newObject(); @@ -124,10 +128,8 @@ public class Search extends Controller { source = null; } - String searchEngine = Play.application().configuration().getString(SearchDao.WHEREHOWS_SEARCH_ENGINE_KEY); - Logger.info("searchEngine is: " + searchEngine); // TODO: deprecated this setting - - result.set("result", SEARCH_DAO.elasticSearchDatasetByKeyword(category, keyword, source, page, size)); + result.set("result", + SEARCH_DAO.elasticSearchDatasetByKeyword(ELASTICSEARCH_DATASET_URL, category, keyword, source, page, size)); return ok(result); }