yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-10-23 15:04:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

Rewrite SearchDao to remove dependency on Play (#1358)

Browse Source
This commit is contained in:
Yi (Alan) Wang 2018-09-04 16:40:02 -07:00 committed by GitHub
parent efa5f49f18
commit 17f1f68f95
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 348 additions and 321 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
wherehows-dao/build.gradle
View File

@ -6,6 +6,7 @@ dependencies {
compile externalDependency.commons_lang3 compile externalDependency.commons_lang3
compile externalDependency.slf4j_api compile externalDependency.slf4j_api
compile externalDependency.spring_jdbc compile externalDependency.spring_jdbc
compile externalDependency.http_client
compile externalDependency.jackson_databind compile externalDependency.jackson_databind
compile externalDependency.jsr305 compile externalDependency.jsr305
compile externalDependency.guava compile externalDependency.guava
@ -15,9 +16,6 @@ dependencies {
runtime externalDependency.hikaricp runtime externalDependency.hikaricp
testCompile externalDependency.testng testCompile externalDependency.testng
compile externalDependency.play_java_ws
} }
findbugs { findbugs {

211
wherehows-dao/src/main/java/wherehows/dao/table/SearchDao.java
View File

@ -11,80 +11,65 @@
* distributed under the License is distributed on an "AS IS" BASIS, * distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*/ */
package wherehows.dao.table; package wherehows.dao.table;
import java.util.*;
import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.databind.node.ObjectNode;
import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashSet; import java.util.HashSet;
import java.util.Iterator; import java.util.Iterator;
import java.util.List; import java.util.List;
import play.Logger; import java.util.Set;
import play.Play; import lombok.extern.slf4j.Slf4j;
import play.libs.F.Promise;
import play.libs.Json;
import play.libs.ws.*;
import wherehows.models.table.Dataset; import wherehows.models.table.Dataset;
import wherehows.util.HttpUtil;
import static wherehows.util.Search.*; import static wherehows.util.Search.*;
public class SearchDao
{
public static String ELASTICSEARCH_DATASET_URL_KEY = "elasticsearch.dataset.url";
public static String WHEREHOWS_SEARCH_ENGINE_KEY = "search.engine";
public List<String> getAutoCompleteList(String input, int limit) @Slf4j
{ public class SearchDao {
List<String> names = new ArrayList<>();
names.addAll(getAutoCompleteListDataset(input, limit / 3)); private static final ObjectMapper _OM = new ObjectMapper();
return names;
public List<String> getAutoCompleteList(String elasticSearchUrl, String input, int limit) {
return getAutoCompleteListDataset(elasticSearchUrl, input, limit / 3);
} }
public List<String> getAutoCompleteListDataset(String input, int limit) public List<String> getAutoCompleteListDataset(String elasticSearchUrl, String input, int limit) {
{
String elasticSearchTypeURLKey = "elasticsearch.dataset.url";
String fieldName = "name_suggest"; String fieldName = "name_suggest";
return getAutoCompleteListbyES(elasticSearchTypeURLKey,fieldName,input,limit); return getAutoCompleteListbyES(elasticSearchUrl, fieldName, input, limit);
} }
public List<String> getAutoCompleteListbyES(String elasticSearchTypeURLKey, String fieldName, String input, public List<String> getAutoCompleteListbyES(String elasticSearchUrl, String fieldName, String input, int limit) {
int limit)
{
// use elastic search completion suggester, ES will validate the input and limit // use elastic search completion suggester, ES will validate the input and limit
List<String> completionSuggestionList = new ArrayList<String>(); List<String> completionSuggestionList = new ArrayList<>();
Set<String> completionSuggestionSet = new HashSet<String>(); Set<String> completionSuggestionSet = new HashSet<>();
ObjectNode keywordNode = generateElasticSearchCompletionSuggesterQuery(fieldName, input, limit);
log.info("The completion suggester query sent to Elastic Search was: " + keywordNode);
JsonNode responseNode = null; JsonNode responseNode = null;
ObjectNode keywordNode = null;
try { try {
keywordNode = generateElasticSearchCompletionSuggesterQuery(fieldName, input, limit); responseNode = HttpUtil.httpPostRequest(elasticSearchUrl, keywordNode);
} catch (Exception e) { } catch (IOException ex) {
Logger.error("Elastic search completion suggester error. Error message :" + e.getMessage()); log.error("ES suggetion list query error: {}" + ex.toString());
} }
Logger.info("The completion suggester query sent to Elastic Search was: " + keywordNode.toString());
Promise<WSResponse> responsePromise =
WS.url(Play.application().configuration().getString(elasticSearchTypeURLKey)).post(keywordNode);
responseNode = responsePromise.get(1000).asJson();
if (responseNode == null || !responseNode.isContainerNode()) { if (responseNode == null || !responseNode.isContainerNode()) {
return completionSuggestionList; return completionSuggestionList;
} }
JsonNode suggestNode = responseNode.get("suggest"); JsonNode suggestNode = responseNode.get("suggest");
if (suggestNode == null || !suggestNode.has("wh-suggest")) { if (suggestNode == null || !suggestNode.has("wh-suggest")) {
Logger.error("Elastic search completion suggester response does not contain suggest node"); log.error("Elastic search completion suggester response does not contain suggest node");
return completionSuggestionList; return completionSuggestionList;
} }
JsonNode whSuggestNode = suggestNode.get("wh-suggest"); JsonNode whSuggestNode = suggestNode.get("wh-suggest");
if (whSuggestNode == null || !whSuggestNode.isArray()) { if (whSuggestNode == null || !whSuggestNode.isArray()) {
Logger.error("Elastic search completion suggester response does not contain wh-suggest node"); log.error("Elastic search completion suggester response does not contain wh-suggest node");
return completionSuggestionList; return completionSuggestionList;
} }
@ -120,66 +105,45 @@ public class SearchDao
} }
completionSuggestionList.addAll(completionSuggestionSet); completionSuggestionList.addAll(completionSuggestionSet);
Logger.info("Returned suggestion list is: " + completionSuggestionList); log.info("Returned suggestion list is: " + completionSuggestionList);
return completionSuggestionList; return completionSuggestionList;
} }
// this is for did you mean feature // this is for did you mean feature
public static List<String> getSuggestionList(String category, String searchKeyword) public List<String> getSuggestionList(String elasticSearchUrl, String category, String searchKeyword) {
{ List<String> suggestionList = new ArrayList<>();
List<String> SuggestionList = new ArrayList<String>();
String elasticSearchType = "dataset";
String elasticSearchTypeURLKey = "elasticsearch.dataset.url";
String fieldName = "name"; String fieldName = "name";
ObjectNode keywordNode = generateElasticSearchPhraseSuggesterQuery(fieldName, searchKeyword);
log.info("The suggest query sent to Elastic Search is: " + keywordNode);
JsonNode responseNode = null; JsonNode responseNode = null;
ObjectNode keywordNode = null;
try { try {
String lCategory = category.toLowerCase(); responseNode = HttpUtil.httpPostRequest(elasticSearchUrl, keywordNode);
Logger.info("lCategory is " + category); } catch (IOException ex) {
log.error("ES suggetion list query error: {}" + ex.toString());
// ToDO: deprecate category or reuse for entity
switch (lCategory) {
case "dataset":
elasticSearchType = "dataset";
elasticSearchTypeURLKey = "elasticsearch.dataset.url";
fieldName = "name";
break;
default:
break;
} }
keywordNode = generateElasticSearchPhraseSuggesterQuery(elasticSearchType, fieldName, searchKeyword);
} catch (Exception e) {
Logger.error("Elastic search phrase suggester error. Error message :" + e.getMessage());
}
Logger.info("The suggest query sent to Elastic Search is: " + keywordNode.toString());
Promise<WSResponse> responsePromise =
WS.url(Play.application().configuration().getString(elasticSearchTypeURLKey)).post(keywordNode);
responseNode = responsePromise.get(1000).asJson();
if (responseNode == null || !responseNode.isContainerNode() || !responseNode.has("hits")) { if (responseNode == null || !responseNode.isContainerNode() || !responseNode.has("hits")) {
return SuggestionList; return suggestionList;
} }
JsonNode suggestNode = responseNode.get("suggest"); JsonNode suggestNode = responseNode.get("suggest");
Logger.info("suggestNode is " + suggestNode.toString()); log.info("suggestNode is " + suggestNode);
if (suggestNode == null || !suggestNode.has("simple_phrase")) { if (suggestNode == null || !suggestNode.has("simple_phrase")) {
return SuggestionList; return suggestionList;
} }
JsonNode simplePhraseNode = suggestNode.get("simple_phrase"); JsonNode simplePhraseNode = suggestNode.get("simple_phrase");
if (simplePhraseNode == null || !simplePhraseNode.isArray()) { if (simplePhraseNode == null || !simplePhraseNode.isArray()) {
return SuggestionList; return suggestionList;
} }
Iterator<JsonNode> arrayIterator = simplePhraseNode.elements(); Iterator<JsonNode> arrayIterator = simplePhraseNode.elements();
if (arrayIterator == null) { if (arrayIterator == null) {
return SuggestionList; return suggestionList;
} }
while (arrayIterator.hasNext()) { while (arrayIterator.hasNext()) {
@ -204,73 +168,59 @@ public class SearchDao
continue; continue;
} }
String oneSuggestion = textNode.get("text").asText(); String oneSuggestion = textNode.get("text").asText();
SuggestionList.add(oneSuggestion); suggestionList.add(oneSuggestion);
} }
} }
return SuggestionList; return suggestionList;
} }
public JsonNode elasticSearchDatasetByKeyword( public JsonNode elasticSearchDatasetByKeyword(String elasticSearchUrl, String category, String keywords,
String category, String source, int page, int size) {
String keywords, ObjectNode queryNode = _OM.createObjectNode();
String source,
int page,
int size)
{
ObjectNode queryNode = Json.newObject();
queryNode.put("from", (page - 1) * size); queryNode.put("from", (page - 1) * size);
queryNode.put("size", size); queryNode.put("size", size);
JsonNode responseNode = null; JsonNode responseNode = null;
ObjectNode keywordNode = null;
try { try {
keywordNode = generateElasticSearchQueryString(category, source, keywords); ObjectNode keywordNode = generateElasticSearchQueryString(category, source, keywords);
} catch (Exception e) {
Logger.error("Elastic search dataset input query is not JSON format. Error message :" + e.getMessage());
}
if (keywordNode != null) { ObjectNode fieldValueFactorNode = _OM.createObjectNode();
ObjectNode funcScoreNodes = Json.newObject();
ObjectNode fieldValueFactorNode = Json.newObject();
fieldValueFactorNode.put("field", "static_boosting_score"); fieldValueFactorNode.put("field", "static_boosting_score");
fieldValueFactorNode.put("factor", 1); fieldValueFactorNode.put("factor", 1);
fieldValueFactorNode.put("modifier", "square"); fieldValueFactorNode.put("modifier", "square");
fieldValueFactorNode.put("missing", 1); fieldValueFactorNode.put("missing", 1);
ObjectNode funcScoreNodes = _OM.createObjectNode();
funcScoreNodes.put("query", keywordNode); funcScoreNodes.put("query", keywordNode);
funcScoreNodes.put("field_value_factor", fieldValueFactorNode); funcScoreNodes.put("field_value_factor", fieldValueFactorNode);
ObjectNode funcScoreNodesWrapper = Json.newObject(); ObjectNode funcScoreNodesWrapper = _OM.createObjectNode();
funcScoreNodesWrapper.put("function_score", funcScoreNodes); funcScoreNodesWrapper.put("function_score", funcScoreNodes);
queryNode.put("query", funcScoreNodesWrapper); queryNode.put("query", funcScoreNodesWrapper);
ObjectNode filterNode = Json.newObject(); ObjectNode filterNode = _OM.createObjectNode();
try { try {
filterNode = generateElasticSearchFilterString(source); filterNode = generateElasticSearchFilterString(source);
} catch (Exception e) { } catch (Exception e) {
Logger.error("Elastic search filter query node generation failed :" + e.getMessage()); log.error("Elastic search filter query node generation failed :" + e.getMessage());
} }
if (filterNode != null) { if (filterNode != null) {
queryNode.put("post_filter", filterNode); queryNode.put("post_filter", filterNode);
} }
Logger.info( log.info(" === elasticSearchDatasetByKeyword === The query sent to Elastic Search is: " + queryNode.toString());
" === elasticSearchDatasetByKeyword === The query sent to Elastic Search is: " + queryNode.toString());
Promise<WSResponse> responsePromise =
WS.url(Play.application().configuration().getString(SearchDao.ELASTICSEARCH_DATASET_URL_KEY)).post(queryNode);
responseNode = responsePromise.get(1000).asJson();
responseNode = HttpUtil.httpPostRequest(elasticSearchUrl, queryNode);
// Logger.debug("The responseNode from Elastic Search is: " + responseNode.toString()); // Logger.debug("The responseNode from Elastic Search is: " + responseNode.toString());
} catch (IOException e) {
log.error("Elastic search dataset query error: {}", e.toString());
} }
ObjectNode resultNode = Json.newObject(); ObjectNode resultNode = _OM.createObjectNode();
Long count = 0L; long count = 0L;
List<Dataset> pagedDatasets = new ArrayList<>(); List<Dataset> pagedDatasets = new ArrayList<>();
resultNode.put("page", page); resultNode.put("page", page);
resultNode.put("category", category); resultNode.put("category", category);
@ -278,49 +228,35 @@ public class SearchDao
resultNode.put("itemsPerPage", size); resultNode.put("itemsPerPage", size);
resultNode.put("keywords", keywords); resultNode.put("keywords", keywords);
if (responseNode != null && responseNode.isContainerNode() && responseNode.has("hits")) if (responseNode != null && responseNode.isContainerNode() && responseNode.has("hits")) {
{
JsonNode hitsNode = responseNode.get("hits"); JsonNode hitsNode = responseNode.get("hits");
if (hitsNode != null) if (hitsNode != null) {
{ if (hitsNode.has("total")) {
if (hitsNode.has("total"))
{
count = hitsNode.get("total").asLong(); count = hitsNode.get("total").asLong();
} }
if (hitsNode.has("hits")) if (hitsNode.has("hits")) {
{
JsonNode dataNode = hitsNode.get("hits"); JsonNode dataNode = hitsNode.get("hits");
if (dataNode != null && dataNode.isArray()) if (dataNode != null && dataNode.isArray()) {
{
Iterator<JsonNode> arrayIterator = dataNode.elements(); Iterator<JsonNode> arrayIterator = dataNode.elements();
if (arrayIterator != null) if (arrayIterator != null) {
{ while (arrayIterator.hasNext()) {
while (arrayIterator.hasNext())
{
JsonNode node = arrayIterator.next(); JsonNode node = arrayIterator.next();
if (node.isContainerNode() && node.has("_id")) if (node.isContainerNode() && node.has("_id")) {
{
Dataset dataset = new Dataset(); Dataset dataset = new Dataset();
dataset.id = node.get("_id").asLong(); dataset.id = node.get("_id").asLong();
if (node.has("_source")) if (node.has("_source")) {
{
JsonNode sourceNode = node.get("_source"); JsonNode sourceNode = node.get("_source");
if (sourceNode != null) if (sourceNode != null) {
{ if (sourceNode.has("name")) {
if (sourceNode.has("name"))
{
dataset.name = sourceNode.get("name").asText(); dataset.name = sourceNode.get("name").asText();
} }
if (sourceNode.has("source")) if (sourceNode.has("source")) {
{
dataset.source = sourceNode.get("source").asText(); dataset.source = sourceNode.get("source").asText();
} }
if (sourceNode.has("urn")) if (sourceNode.has("urn")) {
{
dataset.urn = sourceNode.get("urn").asText(); dataset.urn = sourceNode.get("urn").asText();
} }
if (sourceNode.has("schema")) if (sourceNode.has("schema")) {
{
dataset.schema = sourceNode.get("schema").asText(); dataset.schema = sourceNode.get("schema").asText();
} }
} }
@ -329,16 +265,13 @@ public class SearchDao
} }
} }
} }
} }
} }
} }
} }
resultNode.put("count", count); resultNode.put("count", count);
resultNode.put("totalPages", (int) Math.ceil(count / ((double) size))); resultNode.put("totalPages", (int) Math.ceil(count / ((double) size)));
resultNode.set("data", Json.toJson(pagedDatasets)); resultNode.set("data", _OM.valueToTree(pagedDatasets));
return resultNode; return resultNode;
} }
} }

41
wherehows-dao/src/main/java/wherehows/util/HttpClientFactory.java Normal file
View File

@ -0,0 +1,41 @@
/**
* Copyright 2015 LinkedIn Corp. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*/
package wherehows.util;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
public class HttpClientFactory {
private static final int DEFAULT_TIMEOUT = 1000; // default 1s timeout
private HttpClientFactory() {
}
public static CloseableHttpClient createHttpClient() {
return createHttpClient(DEFAULT_TIMEOUT);
}
public static CloseableHttpClient createHttpClient(int timeoutMillis) {
RequestConfig requestConfig = RequestConfig.custom()
.setConnectTimeout(timeoutMillis)
.setConnectionRequestTimeout(timeoutMillis)
.setSocketTimeout(timeoutMillis)
.build();
return HttpClientBuilder.create().setDefaultRequestConfig(requestConfig).build();
}
}

53
wherehows-dao/src/main/java/wherehows/util/HttpUtil.java Normal file
View File

@ -0,0 +1,53 @@
/**
* Copyright 2015 LinkedIn Corp. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*/
package wherehows.util;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import java.io.IOException;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.util.EntityUtils;
@Slf4j
public class HttpUtil {
private static final ObjectMapper _OM = new ObjectMapper();
private HttpUtil() {
}
public static JsonNode httpPostRequest(String url, JsonNode content) throws IOException {
CloseableHttpClient client = HttpClientFactory.createHttpClient();
HttpPost httpPost = new HttpPost(url);
httpPost.setEntity(new StringEntity(content.toString()));
httpPost.setHeader("Accept", "application/json");
httpPost.setHeader("Content-type", "application/json");
CloseableHttpResponse response = client.execute(httpPost);
if (response.getStatusLine().getStatusCode() != 200) {
log.error("ES request fail: {}", response.getStatusLine().getStatusCode());
return null;
}
JsonNode respJson = _OM.readTree(EntityUtils.toString(response.getEntity()));
client.close();
return respJson;
}
}

60
wherehows-dao/src/main/java/wherehows/util/Search.java
View File

@ -16,20 +16,28 @@ package wherehows.util;
import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.databind.node.ObjectNode;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.nio.file.Paths;
import java.nio.file.Files;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
@Slf4j @Slf4j
public class Search { public class Search {
public final static String DATASET_CATEGORY = "datasets";
public final static String METRIC_CATEGORY = "metrics"; private Search() {
public final static String COMMENT_CATEGORY = "comments"; }
public final static String FLOW_CATEGORY = "flows";
public final static String JOB_CATEGORY = "jobs"; private static final ObjectMapper _OM = new ObjectMapper();
private final static String DATASET_CATEGORY = "datasets";
private final static String METRIC_CATEGORY = "metrics";
private final static String COMMENT_CATEGORY = "comments";
private final static String FLOW_CATEGORY = "flows";
private final static String JOB_CATEGORY = "jobs";
private static final String WHZ_ELASTICSEARCH_DATASET_QUERY_FILE = System.getenv("WHZ_ELASTICSEARCH_DATASET_QUERY_TEMPLATE"); private static final String WHZ_ELASTICSEARCH_DATASET_QUERY_FILE = System.getenv("WHZ_ELASTICSEARCH_DATASET_QUERY_TEMPLATE");
private static final String WHZ_ELASTICSEARCH_METRIC_QUERY_FILE = System.getenv("WHZ_ELASTICSEARCH_METRIC_QUERY_TEMPLATE"); private static final String WHZ_ELASTICSEARCH_METRIC_QUERY_FILE = System.getenv("WHZ_ELASTICSEARCH_METRIC_QUERY_TEMPLATE");
@ -41,10 +49,8 @@ public class Search {
public static String readJsonQueryFile(String jsonFile) { public static String readJsonQueryFile(String jsonFile) {
try { try {
ObjectMapper objectMapper = new ObjectMapper();
String contents = new String(Files.readAllBytes(Paths.get(jsonFile))); String contents = new String(Files.readAllBytes(Paths.get(jsonFile)));
JsonNode json = objectMapper.readTree(contents); JsonNode json = _OM.readTree(contents);
return json.toString(); return json.toString();
} catch (Exception e) { } catch (Exception e) {
log.error("ReadJsonQueryFile failed. Error: " + e.getMessage()); log.error("ReadJsonQueryFile failed. Error: " + e.getMessage());
@ -68,10 +74,10 @@ public class Search {
query = query.replace("$LIMIT", Integer.toString(limit)); query = query.replace("$LIMIT", Integer.toString(limit));
ObjectNode suggestNode = new ObjectMapper().createObjectNode(); ObjectNode suggestNode = _OM.createObjectNode();
try { try {
ObjectNode textNode = (ObjectNode) new ObjectMapper().readTree(query); ObjectNode textNode = (ObjectNode) _OM.readTree(query);
suggestNode.putPOJO("suggest", textNode); suggestNode.putPOJO("suggest", textNode);
} catch (Exception e) { } catch (Exception e) {
log.error("suggest Exception = " + e.getMessage()); log.error("suggest Exception = " + e.getMessage());
@ -81,8 +87,7 @@ public class Search {
return suggestNode; return suggestNode;
} }
public static ObjectNode generateElasticSearchPhraseSuggesterQuery(String category, String field, public static ObjectNode generateElasticSearchPhraseSuggesterQuery(String field, String searchKeyword) {
String searchKeyword) {
if (StringUtils.isBlank(searchKeyword)) { if (StringUtils.isBlank(searchKeyword)) {
return null; return null;
} }
@ -94,9 +99,9 @@ public class Search {
query = query.replace("$FIELD", field.toLowerCase()); query = query.replace("$FIELD", field.toLowerCase());
} }
ObjectNode suggestNode = new ObjectMapper().createObjectNode(); ObjectNode suggestNode = _OM.createObjectNode();
try { try {
ObjectNode textNode = (ObjectNode) new ObjectMapper().readTree(query); ObjectNode textNode = (ObjectNode) _OM.readTree(query);
suggestNode.putPOJO("suggest", textNode); suggestNode.putPOJO("suggest", textNode);
} catch (Exception e) { } catch (Exception e) {
log.error("suggest Exception = " + e.getMessage()); log.error("suggest Exception = " + e.getMessage());
@ -105,8 +110,8 @@ public class Search {
return suggestNode; return suggestNode;
} }
public static ObjectNode generateElasticSearchQueryString(String category, String source, String keywords) throws Exception public static ObjectNode generateElasticSearchQueryString(String category, String source, String keywords)
{ throws IOException {
if (StringUtils.isBlank(keywords)) { if (StringUtils.isBlank(keywords)) {
return null; return null;
} }
@ -127,25 +132,22 @@ public class Search {
} }
} }
ObjectMapper objectMapper = new ObjectMapper();
for (String value : values) { for (String value : values) {
if (StringUtils.isNotBlank(value)) { if (StringUtils.isNotBlank(value)) {
String query = queryTemplate.replace("$VALUE", value.replace("\"", "").toLowerCase().trim()); String query = queryTemplate.replace("$VALUE", value.replace("\"", "").toLowerCase().trim());
shouldValueList.add(objectMapper.readTree(query)); shouldValueList.add(_OM.readTree(query));
} }
} }
ObjectNode shouldNode = new ObjectMapper().createObjectNode(); ObjectNode shouldNode = _OM.createObjectNode();
shouldNode.putPOJO("should", shouldValueList); shouldNode.putPOJO("should", shouldValueList);
ObjectNode queryNode = new ObjectMapper().createObjectNode(); ObjectNode queryNode = _OM.createObjectNode();
queryNode.putPOJO("bool", shouldNode); queryNode.putPOJO("bool", shouldNode);
return queryNode; return queryNode;
} }
public static ObjectNode generateElasticSearchFilterString(String sources) throws IOException {
public static ObjectNode generateElasticSearchFilterString(String sources) throws Exception{
if (StringUtils.isBlank(sources)) { if (StringUtils.isBlank(sources)) {
return null; return null;
} }
@ -155,18 +157,16 @@ public class Search {
String queryTemplate = readJsonQueryFile(WHZ_ELASTICSEARCH_FILTER_UNIT_FILE); String queryTemplate = readJsonQueryFile(WHZ_ELASTICSEARCH_FILTER_UNIT_FILE);
String[] values = sources.trim().split(","); String[] values = sources.trim().split(",");
ObjectMapper objectMapper = new ObjectMapper();
for (String value : values) { for (String value : values) {
if (StringUtils.isNotBlank(value)) { if (StringUtils.isNotBlank(value)) {
String query = queryTemplate.replace("$SOURCE", value.replace("\"", "").toLowerCase().trim()); String query = queryTemplate.replace("$SOURCE", value.replace("\"", "").toLowerCase().trim());
shouldValueList.add(objectMapper.readTree(query)); shouldValueList.add(_OM.readTree(query));
} }
} }
ObjectNode shouldNode = new ObjectMapper().createObjectNode(); ObjectNode shouldNode = _OM.createObjectNode();
shouldNode.putPOJO("should", shouldValueList); shouldNode.putPOJO("should", shouldValueList);
ObjectNode queryNode = new ObjectMapper().createObjectNode(); ObjectNode queryNode = _OM.createObjectNode();
queryNode.putPOJO("bool", shouldNode); queryNode.putPOJO("bool", shouldNode);
return queryNode; return queryNode;

22
wherehows-frontend/app/controllers/api/v1/Search.java
View File

@ -23,13 +23,19 @@ import play.cache.Cache;
import play.libs.Json; import play.libs.Json;
import play.mvc.Controller; import play.mvc.Controller;
import play.mvc.Result; import play.mvc.Result;
import wherehows.dao.table.SearchDao;
import static org.apache.commons.lang3.StringUtils.*; import static org.apache.commons.lang3.StringUtils.*;
import wherehows.dao.table.SearchDao;
public class Search extends Controller { public class Search extends Controller {
private static final String ELASTICSEARCH_DATASET_URL_KEY = "elasticsearch.dataset.url";
private static final String ELASTICSEARCH_DATASET_URL =
Play.application().configuration().getString(ELASTICSEARCH_DATASET_URL_KEY);
private static final String WHEREHOWS_SEARCH_ENGINE_KEY = "search.engine"; // TODO: deprecated this setting
private static final String SEARCH_ENGINE = Play.application().configuration().getString(WHEREHOWS_SEARCH_ENGINE_KEY);
private static final String AUTOCOMPLETE_ALL_KEY = "autocomplete.all"; private static final String AUTOCOMPLETE_ALL_KEY = "autocomplete.all";
private static final String AUTOCOMPLETE_DATASET_KEY = "autocomplete.dataset"; private static final String AUTOCOMPLETE_DATASET_KEY = "autocomplete.dataset";
private static final int DEFAULT_AUTOCOMPLETE_SIZE = 20; private static final int DEFAULT_AUTOCOMPLETE_SIZE = 20;
@ -48,7 +54,7 @@ public class Search extends Controller {
String cacheKey = AUTOCOMPLETE_ALL_KEY + (isNotBlank(input) ? "." + input : "-all"); String cacheKey = AUTOCOMPLETE_ALL_KEY + (isNotBlank(input) ? "." + input : "-all");
List<String> names = (List<String>) Cache.get(cacheKey); List<String> names = (List<String>) Cache.get(cacheKey);
if (names == null || names.size() == 0) { if (names == null || names.size() == 0) {
names = SEARCH_DAO.getAutoCompleteList(input, size); names = SEARCH_DAO.getAutoCompleteList(ELASTICSEARCH_DATASET_URL, input, size);
Cache.set(cacheKey, names, DEFAULT_AUTOCOMPLETE_CACHE_TIME); Cache.set(cacheKey, names, DEFAULT_AUTOCOMPLETE_CACHE_TIME);
} }
@ -70,7 +76,7 @@ public class Search extends Controller {
String cacheKey = AUTOCOMPLETE_DATASET_KEY + (isNotBlank(input) ? "." + input : "-all"); String cacheKey = AUTOCOMPLETE_DATASET_KEY + (isNotBlank(input) ? "." + input : "-all");
List<String> names = (List<String>) Cache.get(cacheKey); List<String> names = (List<String>) Cache.get(cacheKey);
if (names == null || names.size() == 0) { if (names == null || names.size() == 0) {
names = SEARCH_DAO.getAutoCompleteListDataset(input, size); names = SEARCH_DAO.getAutoCompleteListDataset(ELASTICSEARCH_DATASET_URL, input, size);
Cache.set(cacheKey, names, DEFAULT_AUTOCOMPLETE_CACHE_TIME); Cache.set(cacheKey, names, DEFAULT_AUTOCOMPLETE_CACHE_TIME);
} }
@ -81,8 +87,6 @@ public class Search extends Controller {
return ok(result); return ok(result);
} }
public static Result searchByKeyword() { public static Result searchByKeyword() {
ObjectNode result = Json.newObject(); ObjectNode result = Json.newObject();
@ -124,10 +128,8 @@ public class Search extends Controller {
source = null; source = null;
} }
String searchEngine = Play.application().configuration().getString(SearchDao.WHEREHOWS_SEARCH_ENGINE_KEY); result.set("result",
Logger.info("searchEngine is: " + searchEngine); // TODO: deprecated this setting SEARCH_DAO.elasticSearchDatasetByKeyword(ELASTICSEARCH_DATASET_URL, category, keyword, source, page, size));
result.set("result", SEARCH_DAO.elasticSearchDatasetByKeyword(category, keyword, source, page, size));
return ok(result); return ok(result);
} }