mirror of
https://github.com/datahub-project/datahub.git
synced 2025-10-23 15:04:54 +00:00
Rewrite SearchDao to remove dependency on Play (#1358)
This commit is contained in:
parent
efa5f49f18
commit
17f1f68f95
@ -6,6 +6,7 @@ dependencies {
|
||||
compile externalDependency.commons_lang3
|
||||
compile externalDependency.slf4j_api
|
||||
compile externalDependency.spring_jdbc
|
||||
compile externalDependency.http_client
|
||||
compile externalDependency.jackson_databind
|
||||
compile externalDependency.jsr305
|
||||
compile externalDependency.guava
|
||||
@ -15,9 +16,6 @@ dependencies {
|
||||
runtime externalDependency.hikaricp
|
||||
|
||||
testCompile externalDependency.testng
|
||||
|
||||
compile externalDependency.play_java_ws
|
||||
|
||||
}
|
||||
|
||||
findbugs {
|
||||
|
@ -11,80 +11,65 @@
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
*/
|
||||
|
||||
package wherehows.dao.table;
|
||||
import java.util.*;
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.databind.node.ObjectNode;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import play.Logger;
|
||||
import play.Play;
|
||||
import play.libs.F.Promise;
|
||||
import play.libs.Json;
|
||||
import play.libs.ws.*;
|
||||
import java.util.Set;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import wherehows.models.table.Dataset;
|
||||
|
||||
import wherehows.util.HttpUtil;
|
||||
|
||||
import static wherehows.util.Search.*;
|
||||
|
||||
public class SearchDao
|
||||
{
|
||||
public static String ELASTICSEARCH_DATASET_URL_KEY = "elasticsearch.dataset.url";
|
||||
public static String WHEREHOWS_SEARCH_ENGINE_KEY = "search.engine";
|
||||
|
||||
public List<String> getAutoCompleteList(String input, int limit)
|
||||
{
|
||||
List<String> names = new ArrayList<>();
|
||||
names.addAll(getAutoCompleteListDataset(input, limit / 3));
|
||||
return names;
|
||||
@Slf4j
|
||||
public class SearchDao {
|
||||
|
||||
private static final ObjectMapper _OM = new ObjectMapper();
|
||||
|
||||
public List<String> getAutoCompleteList(String elasticSearchUrl, String input, int limit) {
|
||||
return getAutoCompleteListDataset(elasticSearchUrl, input, limit / 3);
|
||||
}
|
||||
|
||||
public List<String> getAutoCompleteListDataset(String input, int limit)
|
||||
{
|
||||
String elasticSearchTypeURLKey = "elasticsearch.dataset.url";
|
||||
public List<String> getAutoCompleteListDataset(String elasticSearchUrl, String input, int limit) {
|
||||
String fieldName = "name_suggest";
|
||||
return getAutoCompleteListbyES(elasticSearchTypeURLKey,fieldName,input,limit);
|
||||
return getAutoCompleteListbyES(elasticSearchUrl, fieldName, input, limit);
|
||||
}
|
||||
|
||||
public List<String> getAutoCompleteListbyES(String elasticSearchTypeURLKey, String fieldName, String input,
|
||||
int limit)
|
||||
{
|
||||
public List<String> getAutoCompleteListbyES(String elasticSearchUrl, String fieldName, String input, int limit) {
|
||||
// use elastic search completion suggester, ES will validate the input and limit
|
||||
List<String> completionSuggestionList = new ArrayList<String>();
|
||||
Set<String> completionSuggestionSet = new HashSet<String>();
|
||||
List<String> completionSuggestionList = new ArrayList<>();
|
||||
Set<String> completionSuggestionSet = new HashSet<>();
|
||||
|
||||
ObjectNode keywordNode = generateElasticSearchCompletionSuggesterQuery(fieldName, input, limit);
|
||||
log.info("The completion suggester query sent to Elastic Search was: " + keywordNode);
|
||||
|
||||
JsonNode responseNode = null;
|
||||
ObjectNode keywordNode = null;
|
||||
|
||||
try {
|
||||
keywordNode = generateElasticSearchCompletionSuggesterQuery(fieldName, input, limit);
|
||||
} catch (Exception e) {
|
||||
Logger.error("Elastic search completion suggester error. Error message :" + e.getMessage());
|
||||
responseNode = HttpUtil.httpPostRequest(elasticSearchUrl, keywordNode);
|
||||
} catch (IOException ex) {
|
||||
log.error("ES suggetion list query error: {}" + ex.toString());
|
||||
}
|
||||
|
||||
Logger.info("The completion suggester query sent to Elastic Search was: " + keywordNode.toString());
|
||||
|
||||
Promise<WSResponse> responsePromise =
|
||||
WS.url(Play.application().configuration().getString(elasticSearchTypeURLKey)).post(keywordNode);
|
||||
responseNode = responsePromise.get(1000).asJson();
|
||||
|
||||
if (responseNode == null || !responseNode.isContainerNode()) {
|
||||
return completionSuggestionList;
|
||||
}
|
||||
|
||||
JsonNode suggestNode = responseNode.get("suggest");
|
||||
if (suggestNode == null || !suggestNode.has("wh-suggest")) {
|
||||
Logger.error("Elastic search completion suggester response does not contain suggest node");
|
||||
log.error("Elastic search completion suggester response does not contain suggest node");
|
||||
return completionSuggestionList;
|
||||
}
|
||||
|
||||
JsonNode whSuggestNode = suggestNode.get("wh-suggest");
|
||||
if (whSuggestNode == null || !whSuggestNode.isArray()) {
|
||||
Logger.error("Elastic search completion suggester response does not contain wh-suggest node");
|
||||
log.error("Elastic search completion suggester response does not contain wh-suggest node");
|
||||
return completionSuggestionList;
|
||||
}
|
||||
|
||||
@ -120,66 +105,45 @@ public class SearchDao
|
||||
}
|
||||
|
||||
completionSuggestionList.addAll(completionSuggestionSet);
|
||||
Logger.info("Returned suggestion list is: " + completionSuggestionList);
|
||||
log.info("Returned suggestion list is: " + completionSuggestionList);
|
||||
return completionSuggestionList;
|
||||
}
|
||||
|
||||
// this is for did you mean feature
|
||||
public static List<String> getSuggestionList(String category, String searchKeyword)
|
||||
{
|
||||
List<String> SuggestionList = new ArrayList<String>();
|
||||
String elasticSearchType = "dataset";
|
||||
String elasticSearchTypeURLKey = "elasticsearch.dataset.url";
|
||||
public List<String> getSuggestionList(String elasticSearchUrl, String category, String searchKeyword) {
|
||||
List<String> suggestionList = new ArrayList<>();
|
||||
String fieldName = "name";
|
||||
|
||||
ObjectNode keywordNode = generateElasticSearchPhraseSuggesterQuery(fieldName, searchKeyword);
|
||||
|
||||
log.info("The suggest query sent to Elastic Search is: " + keywordNode);
|
||||
|
||||
JsonNode responseNode = null;
|
||||
ObjectNode keywordNode = null;
|
||||
|
||||
try {
|
||||
String lCategory = category.toLowerCase();
|
||||
Logger.info("lCategory is " + category);
|
||||
|
||||
// ToDO: deprecate category or reuse for entity
|
||||
switch (lCategory) {
|
||||
case "dataset":
|
||||
elasticSearchType = "dataset";
|
||||
elasticSearchTypeURLKey = "elasticsearch.dataset.url";
|
||||
fieldName = "name";
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
responseNode = HttpUtil.httpPostRequest(elasticSearchUrl, keywordNode);
|
||||
} catch (IOException ex) {
|
||||
log.error("ES suggetion list query error: {}" + ex.toString());
|
||||
}
|
||||
|
||||
keywordNode = generateElasticSearchPhraseSuggesterQuery(elasticSearchType, fieldName, searchKeyword);
|
||||
} catch (Exception e) {
|
||||
Logger.error("Elastic search phrase suggester error. Error message :" + e.getMessage());
|
||||
}
|
||||
|
||||
Logger.info("The suggest query sent to Elastic Search is: " + keywordNode.toString());
|
||||
|
||||
Promise<WSResponse> responsePromise =
|
||||
WS.url(Play.application().configuration().getString(elasticSearchTypeURLKey)).post(keywordNode);
|
||||
responseNode = responsePromise.get(1000).asJson();
|
||||
|
||||
if (responseNode == null || !responseNode.isContainerNode() || !responseNode.has("hits")) {
|
||||
return SuggestionList;
|
||||
return suggestionList;
|
||||
}
|
||||
|
||||
JsonNode suggestNode = responseNode.get("suggest");
|
||||
Logger.info("suggestNode is " + suggestNode.toString());
|
||||
log.info("suggestNode is " + suggestNode);
|
||||
|
||||
if (suggestNode == null || !suggestNode.has("simple_phrase")) {
|
||||
return SuggestionList;
|
||||
return suggestionList;
|
||||
}
|
||||
|
||||
JsonNode simplePhraseNode = suggestNode.get("simple_phrase");
|
||||
if (simplePhraseNode == null || !simplePhraseNode.isArray()) {
|
||||
return SuggestionList;
|
||||
return suggestionList;
|
||||
}
|
||||
|
||||
Iterator<JsonNode> arrayIterator = simplePhraseNode.elements();
|
||||
if (arrayIterator == null) {
|
||||
return SuggestionList;
|
||||
return suggestionList;
|
||||
}
|
||||
|
||||
while (arrayIterator.hasNext()) {
|
||||
@ -204,73 +168,59 @@ public class SearchDao
|
||||
continue;
|
||||
}
|
||||
String oneSuggestion = textNode.get("text").asText();
|
||||
SuggestionList.add(oneSuggestion);
|
||||
suggestionList.add(oneSuggestion);
|
||||
}
|
||||
}
|
||||
|
||||
return SuggestionList;
|
||||
return suggestionList;
|
||||
}
|
||||
|
||||
public JsonNode elasticSearchDatasetByKeyword(
|
||||
String category,
|
||||
String keywords,
|
||||
String source,
|
||||
int page,
|
||||
int size)
|
||||
{
|
||||
ObjectNode queryNode = Json.newObject();
|
||||
public JsonNode elasticSearchDatasetByKeyword(String elasticSearchUrl, String category, String keywords,
|
||||
String source, int page, int size) {
|
||||
ObjectNode queryNode = _OM.createObjectNode();
|
||||
queryNode.put("from", (page - 1) * size);
|
||||
queryNode.put("size", size);
|
||||
JsonNode responseNode = null;
|
||||
ObjectNode keywordNode = null;
|
||||
|
||||
try {
|
||||
keywordNode = generateElasticSearchQueryString(category, source, keywords);
|
||||
} catch (Exception e) {
|
||||
Logger.error("Elastic search dataset input query is not JSON format. Error message :" + e.getMessage());
|
||||
}
|
||||
ObjectNode keywordNode = generateElasticSearchQueryString(category, source, keywords);
|
||||
|
||||
if (keywordNode != null) {
|
||||
ObjectNode funcScoreNodes = Json.newObject();
|
||||
|
||||
ObjectNode fieldValueFactorNode = Json.newObject();
|
||||
ObjectNode fieldValueFactorNode = _OM.createObjectNode();
|
||||
fieldValueFactorNode.put("field", "static_boosting_score");
|
||||
fieldValueFactorNode.put("factor", 1);
|
||||
fieldValueFactorNode.put("modifier", "square");
|
||||
fieldValueFactorNode.put("missing", 1);
|
||||
|
||||
ObjectNode funcScoreNodes = _OM.createObjectNode();
|
||||
funcScoreNodes.put("query", keywordNode);
|
||||
funcScoreNodes.put("field_value_factor", fieldValueFactorNode);
|
||||
|
||||
ObjectNode funcScoreNodesWrapper = Json.newObject();
|
||||
ObjectNode funcScoreNodesWrapper = _OM.createObjectNode();
|
||||
funcScoreNodesWrapper.put("function_score", funcScoreNodes);
|
||||
|
||||
queryNode.put("query", funcScoreNodesWrapper);
|
||||
|
||||
ObjectNode filterNode = Json.newObject();
|
||||
ObjectNode filterNode = _OM.createObjectNode();
|
||||
try {
|
||||
filterNode = generateElasticSearchFilterString(source);
|
||||
} catch (Exception e) {
|
||||
Logger.error("Elastic search filter query node generation failed :" + e.getMessage());
|
||||
log.error("Elastic search filter query node generation failed :" + e.getMessage());
|
||||
}
|
||||
|
||||
if (filterNode != null) {
|
||||
queryNode.put("post_filter", filterNode);
|
||||
}
|
||||
|
||||
Logger.info(
|
||||
" === elasticSearchDatasetByKeyword === The query sent to Elastic Search is: " + queryNode.toString());
|
||||
|
||||
Promise<WSResponse> responsePromise =
|
||||
WS.url(Play.application().configuration().getString(SearchDao.ELASTICSEARCH_DATASET_URL_KEY)).post(queryNode);
|
||||
responseNode = responsePromise.get(1000).asJson();
|
||||
log.info(" === elasticSearchDatasetByKeyword === The query sent to Elastic Search is: " + queryNode.toString());
|
||||
|
||||
responseNode = HttpUtil.httpPostRequest(elasticSearchUrl, queryNode);
|
||||
// Logger.debug("The responseNode from Elastic Search is: " + responseNode.toString());
|
||||
|
||||
} catch (IOException e) {
|
||||
log.error("Elastic search dataset query error: {}", e.toString());
|
||||
}
|
||||
|
||||
ObjectNode resultNode = Json.newObject();
|
||||
Long count = 0L;
|
||||
ObjectNode resultNode = _OM.createObjectNode();
|
||||
long count = 0L;
|
||||
List<Dataset> pagedDatasets = new ArrayList<>();
|
||||
resultNode.put("page", page);
|
||||
resultNode.put("category", category);
|
||||
@ -278,49 +228,35 @@ public class SearchDao
|
||||
resultNode.put("itemsPerPage", size);
|
||||
resultNode.put("keywords", keywords);
|
||||
|
||||
if (responseNode != null && responseNode.isContainerNode() && responseNode.has("hits"))
|
||||
{
|
||||
if (responseNode != null && responseNode.isContainerNode() && responseNode.has("hits")) {
|
||||
JsonNode hitsNode = responseNode.get("hits");
|
||||
if (hitsNode != null)
|
||||
{
|
||||
if (hitsNode.has("total"))
|
||||
{
|
||||
if (hitsNode != null) {
|
||||
if (hitsNode.has("total")) {
|
||||
count = hitsNode.get("total").asLong();
|
||||
}
|
||||
if (hitsNode.has("hits"))
|
||||
{
|
||||
if (hitsNode.has("hits")) {
|
||||
JsonNode dataNode = hitsNode.get("hits");
|
||||
if (dataNode != null && dataNode.isArray())
|
||||
{
|
||||
if (dataNode != null && dataNode.isArray()) {
|
||||
Iterator<JsonNode> arrayIterator = dataNode.elements();
|
||||
if (arrayIterator != null)
|
||||
{
|
||||
while (arrayIterator.hasNext())
|
||||
{
|
||||
if (arrayIterator != null) {
|
||||
while (arrayIterator.hasNext()) {
|
||||
JsonNode node = arrayIterator.next();
|
||||
if (node.isContainerNode() && node.has("_id"))
|
||||
{
|
||||
if (node.isContainerNode() && node.has("_id")) {
|
||||
Dataset dataset = new Dataset();
|
||||
dataset.id = node.get("_id").asLong();
|
||||
if (node.has("_source"))
|
||||
{
|
||||
if (node.has("_source")) {
|
||||
JsonNode sourceNode = node.get("_source");
|
||||
if (sourceNode != null)
|
||||
{
|
||||
if (sourceNode.has("name"))
|
||||
{
|
||||
if (sourceNode != null) {
|
||||
if (sourceNode.has("name")) {
|
||||
dataset.name = sourceNode.get("name").asText();
|
||||
}
|
||||
if (sourceNode.has("source"))
|
||||
{
|
||||
if (sourceNode.has("source")) {
|
||||
dataset.source = sourceNode.get("source").asText();
|
||||
}
|
||||
if (sourceNode.has("urn"))
|
||||
{
|
||||
if (sourceNode.has("urn")) {
|
||||
dataset.urn = sourceNode.get("urn").asText();
|
||||
}
|
||||
if (sourceNode.has("schema"))
|
||||
{
|
||||
if (sourceNode.has("schema")) {
|
||||
dataset.schema = sourceNode.get("schema").asText();
|
||||
}
|
||||
}
|
||||
@ -329,16 +265,13 @@ public class SearchDao
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
resultNode.put("count", count);
|
||||
resultNode.put("totalPages", (int) Math.ceil(count / ((double) size)));
|
||||
resultNode.set("data", Json.toJson(pagedDatasets));
|
||||
resultNode.set("data", _OM.valueToTree(pagedDatasets));
|
||||
return resultNode;
|
||||
}
|
||||
|
||||
}
|
@ -0,0 +1,41 @@
|
||||
/**
|
||||
* Copyright 2015 LinkedIn Corp. All rights reserved.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
*/
|
||||
package wherehows.util;
|
||||
|
||||
import org.apache.http.client.config.RequestConfig;
|
||||
import org.apache.http.impl.client.CloseableHttpClient;
|
||||
import org.apache.http.impl.client.HttpClientBuilder;
|
||||
|
||||
|
||||
public class HttpClientFactory {
|
||||
|
||||
private static final int DEFAULT_TIMEOUT = 1000; // default 1s timeout
|
||||
|
||||
private HttpClientFactory() {
|
||||
}
|
||||
|
||||
public static CloseableHttpClient createHttpClient() {
|
||||
return createHttpClient(DEFAULT_TIMEOUT);
|
||||
}
|
||||
|
||||
public static CloseableHttpClient createHttpClient(int timeoutMillis) {
|
||||
RequestConfig requestConfig = RequestConfig.custom()
|
||||
.setConnectTimeout(timeoutMillis)
|
||||
.setConnectionRequestTimeout(timeoutMillis)
|
||||
.setSocketTimeout(timeoutMillis)
|
||||
.build();
|
||||
|
||||
return HttpClientBuilder.create().setDefaultRequestConfig(requestConfig).build();
|
||||
}
|
||||
}
|
53
wherehows-dao/src/main/java/wherehows/util/HttpUtil.java
Normal file
53
wherehows-dao/src/main/java/wherehows/util/HttpUtil.java
Normal file
@ -0,0 +1,53 @@
|
||||
/**
|
||||
* Copyright 2015 LinkedIn Corp. All rights reserved.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
*/
|
||||
package wherehows.util;
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import java.io.IOException;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.http.client.methods.CloseableHttpResponse;
|
||||
import org.apache.http.client.methods.HttpPost;
|
||||
import org.apache.http.entity.StringEntity;
|
||||
import org.apache.http.impl.client.CloseableHttpClient;
|
||||
import org.apache.http.util.EntityUtils;
|
||||
|
||||
|
||||
@Slf4j
|
||||
public class HttpUtil {
|
||||
|
||||
private static final ObjectMapper _OM = new ObjectMapper();
|
||||
|
||||
private HttpUtil() {
|
||||
}
|
||||
|
||||
public static JsonNode httpPostRequest(String url, JsonNode content) throws IOException {
|
||||
CloseableHttpClient client = HttpClientFactory.createHttpClient();
|
||||
HttpPost httpPost = new HttpPost(url);
|
||||
|
||||
httpPost.setEntity(new StringEntity(content.toString()));
|
||||
httpPost.setHeader("Accept", "application/json");
|
||||
httpPost.setHeader("Content-type", "application/json");
|
||||
|
||||
CloseableHttpResponse response = client.execute(httpPost);
|
||||
if (response.getStatusLine().getStatusCode() != 200) {
|
||||
log.error("ES request fail: {}", response.getStatusLine().getStatusCode());
|
||||
return null;
|
||||
}
|
||||
JsonNode respJson = _OM.readTree(EntityUtils.toString(response.getEntity()));
|
||||
client.close();
|
||||
|
||||
return respJson;
|
||||
}
|
||||
}
|
@ -16,20 +16,28 @@ package wherehows.util;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.databind.node.ObjectNode;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.nio.file.Paths;
|
||||
import java.nio.file.Files;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
||||
|
||||
@Slf4j
|
||||
public class Search {
|
||||
public final static String DATASET_CATEGORY = "datasets";
|
||||
public final static String METRIC_CATEGORY = "metrics";
|
||||
public final static String COMMENT_CATEGORY = "comments";
|
||||
public final static String FLOW_CATEGORY = "flows";
|
||||
public final static String JOB_CATEGORY = "jobs";
|
||||
|
||||
private Search() {
|
||||
}
|
||||
|
||||
private static final ObjectMapper _OM = new ObjectMapper();
|
||||
|
||||
private final static String DATASET_CATEGORY = "datasets";
|
||||
private final static String METRIC_CATEGORY = "metrics";
|
||||
private final static String COMMENT_CATEGORY = "comments";
|
||||
private final static String FLOW_CATEGORY = "flows";
|
||||
private final static String JOB_CATEGORY = "jobs";
|
||||
|
||||
private static final String WHZ_ELASTICSEARCH_DATASET_QUERY_FILE = System.getenv("WHZ_ELASTICSEARCH_DATASET_QUERY_TEMPLATE");
|
||||
private static final String WHZ_ELASTICSEARCH_METRIC_QUERY_FILE = System.getenv("WHZ_ELASTICSEARCH_METRIC_QUERY_TEMPLATE");
|
||||
@ -41,10 +49,8 @@ public class Search {
|
||||
|
||||
public static String readJsonQueryFile(String jsonFile) {
|
||||
try {
|
||||
ObjectMapper objectMapper = new ObjectMapper();
|
||||
|
||||
String contents = new String(Files.readAllBytes(Paths.get(jsonFile)));
|
||||
JsonNode json = objectMapper.readTree(contents);
|
||||
JsonNode json = _OM.readTree(contents);
|
||||
return json.toString();
|
||||
} catch (Exception e) {
|
||||
log.error("ReadJsonQueryFile failed. Error: " + e.getMessage());
|
||||
@ -68,10 +74,10 @@ public class Search {
|
||||
|
||||
query = query.replace("$LIMIT", Integer.toString(limit));
|
||||
|
||||
ObjectNode suggestNode = new ObjectMapper().createObjectNode();
|
||||
ObjectNode suggestNode = _OM.createObjectNode();
|
||||
|
||||
try {
|
||||
ObjectNode textNode = (ObjectNode) new ObjectMapper().readTree(query);
|
||||
ObjectNode textNode = (ObjectNode) _OM.readTree(query);
|
||||
suggestNode.putPOJO("suggest", textNode);
|
||||
} catch (Exception e) {
|
||||
log.error("suggest Exception = " + e.getMessage());
|
||||
@ -81,8 +87,7 @@ public class Search {
|
||||
return suggestNode;
|
||||
}
|
||||
|
||||
public static ObjectNode generateElasticSearchPhraseSuggesterQuery(String category, String field,
|
||||
String searchKeyword) {
|
||||
public static ObjectNode generateElasticSearchPhraseSuggesterQuery(String field, String searchKeyword) {
|
||||
if (StringUtils.isBlank(searchKeyword)) {
|
||||
return null;
|
||||
}
|
||||
@ -94,9 +99,9 @@ public class Search {
|
||||
query = query.replace("$FIELD", field.toLowerCase());
|
||||
}
|
||||
|
||||
ObjectNode suggestNode = new ObjectMapper().createObjectNode();
|
||||
ObjectNode suggestNode = _OM.createObjectNode();
|
||||
try {
|
||||
ObjectNode textNode = (ObjectNode) new ObjectMapper().readTree(query);
|
||||
ObjectNode textNode = (ObjectNode) _OM.readTree(query);
|
||||
suggestNode.putPOJO("suggest", textNode);
|
||||
} catch (Exception e) {
|
||||
log.error("suggest Exception = " + e.getMessage());
|
||||
@ -105,8 +110,8 @@ public class Search {
|
||||
return suggestNode;
|
||||
}
|
||||
|
||||
public static ObjectNode generateElasticSearchQueryString(String category, String source, String keywords) throws Exception
|
||||
{
|
||||
public static ObjectNode generateElasticSearchQueryString(String category, String source, String keywords)
|
||||
throws IOException {
|
||||
if (StringUtils.isBlank(keywords)) {
|
||||
return null;
|
||||
}
|
||||
@ -127,25 +132,22 @@ public class Search {
|
||||
}
|
||||
}
|
||||
|
||||
ObjectMapper objectMapper = new ObjectMapper();
|
||||
|
||||
for (String value : values) {
|
||||
if (StringUtils.isNotBlank(value)) {
|
||||
String query = queryTemplate.replace("$VALUE", value.replace("\"", "").toLowerCase().trim());
|
||||
shouldValueList.add(objectMapper.readTree(query));
|
||||
shouldValueList.add(_OM.readTree(query));
|
||||
}
|
||||
}
|
||||
|
||||
ObjectNode shouldNode = new ObjectMapper().createObjectNode();
|
||||
ObjectNode shouldNode = _OM.createObjectNode();
|
||||
shouldNode.putPOJO("should", shouldValueList);
|
||||
ObjectNode queryNode = new ObjectMapper().createObjectNode();
|
||||
ObjectNode queryNode = _OM.createObjectNode();
|
||||
queryNode.putPOJO("bool", shouldNode);
|
||||
|
||||
return queryNode;
|
||||
}
|
||||
|
||||
|
||||
public static ObjectNode generateElasticSearchFilterString(String sources) throws Exception{
|
||||
public static ObjectNode generateElasticSearchFilterString(String sources) throws IOException {
|
||||
if (StringUtils.isBlank(sources)) {
|
||||
return null;
|
||||
}
|
||||
@ -155,18 +157,16 @@ public class Search {
|
||||
String queryTemplate = readJsonQueryFile(WHZ_ELASTICSEARCH_FILTER_UNIT_FILE);
|
||||
String[] values = sources.trim().split(",");
|
||||
|
||||
ObjectMapper objectMapper = new ObjectMapper();
|
||||
|
||||
for (String value : values) {
|
||||
if (StringUtils.isNotBlank(value)) {
|
||||
String query = queryTemplate.replace("$SOURCE", value.replace("\"", "").toLowerCase().trim());
|
||||
shouldValueList.add(objectMapper.readTree(query));
|
||||
shouldValueList.add(_OM.readTree(query));
|
||||
}
|
||||
}
|
||||
|
||||
ObjectNode shouldNode = new ObjectMapper().createObjectNode();
|
||||
ObjectNode shouldNode = _OM.createObjectNode();
|
||||
shouldNode.putPOJO("should", shouldValueList);
|
||||
ObjectNode queryNode = new ObjectMapper().createObjectNode();
|
||||
ObjectNode queryNode = _OM.createObjectNode();
|
||||
queryNode.putPOJO("bool", shouldNode);
|
||||
|
||||
return queryNode;
|
||||
|
@ -23,13 +23,19 @@ import play.cache.Cache;
|
||||
import play.libs.Json;
|
||||
import play.mvc.Controller;
|
||||
import play.mvc.Result;
|
||||
import wherehows.dao.table.SearchDao;
|
||||
|
||||
import static org.apache.commons.lang3.StringUtils.*;
|
||||
|
||||
import wherehows.dao.table.SearchDao;
|
||||
|
||||
|
||||
public class Search extends Controller {
|
||||
|
||||
private static final String ELASTICSEARCH_DATASET_URL_KEY = "elasticsearch.dataset.url";
|
||||
private static final String ELASTICSEARCH_DATASET_URL =
|
||||
Play.application().configuration().getString(ELASTICSEARCH_DATASET_URL_KEY);
|
||||
private static final String WHEREHOWS_SEARCH_ENGINE_KEY = "search.engine"; // TODO: deprecated this setting
|
||||
private static final String SEARCH_ENGINE = Play.application().configuration().getString(WHEREHOWS_SEARCH_ENGINE_KEY);
|
||||
|
||||
private static final String AUTOCOMPLETE_ALL_KEY = "autocomplete.all";
|
||||
private static final String AUTOCOMPLETE_DATASET_KEY = "autocomplete.dataset";
|
||||
private static final int DEFAULT_AUTOCOMPLETE_SIZE = 20;
|
||||
@ -48,7 +54,7 @@ public class Search extends Controller {
|
||||
String cacheKey = AUTOCOMPLETE_ALL_KEY + (isNotBlank(input) ? "." + input : "-all");
|
||||
List<String> names = (List<String>) Cache.get(cacheKey);
|
||||
if (names == null || names.size() == 0) {
|
||||
names = SEARCH_DAO.getAutoCompleteList(input, size);
|
||||
names = SEARCH_DAO.getAutoCompleteList(ELASTICSEARCH_DATASET_URL, input, size);
|
||||
Cache.set(cacheKey, names, DEFAULT_AUTOCOMPLETE_CACHE_TIME);
|
||||
}
|
||||
|
||||
@ -70,7 +76,7 @@ public class Search extends Controller {
|
||||
String cacheKey = AUTOCOMPLETE_DATASET_KEY + (isNotBlank(input) ? "." + input : "-all");
|
||||
List<String> names = (List<String>) Cache.get(cacheKey);
|
||||
if (names == null || names.size() == 0) {
|
||||
names = SEARCH_DAO.getAutoCompleteListDataset(input, size);
|
||||
names = SEARCH_DAO.getAutoCompleteListDataset(ELASTICSEARCH_DATASET_URL, input, size);
|
||||
Cache.set(cacheKey, names, DEFAULT_AUTOCOMPLETE_CACHE_TIME);
|
||||
}
|
||||
|
||||
@ -81,8 +87,6 @@ public class Search extends Controller {
|
||||
return ok(result);
|
||||
}
|
||||
|
||||
|
||||
|
||||
public static Result searchByKeyword() {
|
||||
ObjectNode result = Json.newObject();
|
||||
|
||||
@ -124,10 +128,8 @@ public class Search extends Controller {
|
||||
source = null;
|
||||
}
|
||||
|
||||
String searchEngine = Play.application().configuration().getString(SearchDao.WHEREHOWS_SEARCH_ENGINE_KEY);
|
||||
Logger.info("searchEngine is: " + searchEngine); // TODO: deprecated this setting
|
||||
|
||||
result.set("result", SEARCH_DAO.elasticSearchDatasetByKeyword(category, keyword, source, page, size));
|
||||
result.set("result",
|
||||
SEARCH_DAO.elasticSearchDatasetByKeyword(ELASTICSEARCH_DATASET_URL, category, keyword, source, page, size));
|
||||
|
||||
return ok(result);
|
||||
}
|
||||
|
Loading…
x
Reference in New Issue
Block a user