From 4025153efb2314d1ceec36a38f5cc25b38cc7054 Mon Sep 17 00:00:00 2001 From: Na Zhang Date: Thu, 23 Mar 2017 17:26:29 -0700 Subject: [PATCH] new type ahead api and phrase suggester support --- web/app/controllers/api/v1/Search.java | 28 +++++ web/app/dao/SearchDAO.java | 149 ++++++++++++++++++++++++- web/app/utils/Search.java | 47 ++++++++ web/conf/routes | 6 + 4 files changed, 224 insertions(+), 6 deletions(-) diff --git a/web/app/controllers/api/v1/Search.java b/web/app/controllers/api/v1/Search.java index 07fdb6c664..e0bdc2cfa7 100644 --- a/web/app/controllers/api/v1/Search.java +++ b/web/app/controllers/api/v1/Search.java @@ -39,6 +39,34 @@ public class Search extends Controller return ok(result); } + public static Result getSearchAutoCompleteForDataset() + { + ObjectNode result = Json.newObject(); + result.put("status", "ok"); + result.set("source", Json.toJson(SearchDAO.getAutoCompleteListForDataset())); + + return ok(result); + } + + public static Result getSearchAutoCompleteForMetric() + { + ObjectNode result = Json.newObject(); + result.put("status", "ok"); + result.set("source", Json.toJson(SearchDAO.getAutoCompleteListForMetric())); + + return ok(result); + } + + public static Result getSearchAutoCompleteForFlow() + { + ObjectNode result = Json.newObject(); + result.put("status", "ok"); + result.set("source", Json.toJson(SearchDAO.getAutoCompleteListForFlow())); + + return ok(result); + } + + public static Result searchByKeyword() { ObjectNode result = Json.newObject(); diff --git a/web/app/dao/SearchDAO.java b/web/app/dao/SearchDAO.java index 7f3d33cadd..41585ad9f9 100644 --- a/web/app/dao/SearchDAO.java +++ b/web/app/dao/SearchDAO.java @@ -147,15 +147,14 @@ public class SearchDAO extends AbstractMySQLOpenSourceDAO "ORDER BY 2 LIMIT ?, ?;"; public final static String SEARCH_AUTOCOMPLETE_LIST = "searchSource"; - - public final static String GET_METRIC_AUTO_COMPLETE_LIST = "SELECT DISTINCT CASE " + - "WHEN parent_path is null or parent_path = '' THEN field_name " + - "ELSE CONCAT_WS('.', parent_path, field_name) END as full_name FROM dict_field_detail"; + public final static String SEARCH_AUTOCOMPLETE_LIST_DATASET = "searchSourceDataset"; + public final static String SEARCH_AUTOCOMPLETE_LIST_METRIC = "searchSourceMetric"; + public final static String SEARCH_AUTOCOMPLETE_LIST_FLOW = "searchSourceFlow"; public final static String GET_DATASET_AUTO_COMPLETE_LIST = "SELECT DISTINCT name FROM dict_dataset"; + public final static String GET_METRIC_AUTO_COMPLETE_LIST = "SELECT DISTINCT metric_name FROM dict_business_metric"; public final static String GET_FLOW_AUTO_COMPLETE_LIST = "SELECT DISTINCT flow_name FROM flow"; - public final static String GET_JOB_AUTO_COMPLETE_LIST = "SELECT DISTINCT job_name FROM flow_job"; public static List getAutoCompleteList() @@ -174,10 +173,148 @@ public class SearchDAO extends AbstractMySQLOpenSourceDAO Cache.set(SEARCH_AUTOCOMPLETE_LIST, cachedAutoCompleteList, 60*60); } - return cachedAutoCompleteList; } + public static List getAutoCompleteListForDataset() + { + List cachedAutoCompleteListForDataset = (List)Cache.get(SEARCH_AUTOCOMPLETE_LIST_DATASET); + if (cachedAutoCompleteListForDataset == null || cachedAutoCompleteListForDataset.size() == 0) + { + List datasetList = getJdbcTemplate().queryForList(GET_DATASET_AUTO_COMPLETE_LIST, String.class); + cachedAutoCompleteListForDataset = datasetList.stream().collect(Collectors.toList()); + Collections.sort(cachedAutoCompleteListForDataset); + Cache.set(SEARCH_AUTOCOMPLETE_LIST_DATASET, cachedAutoCompleteListForDataset, 60*60); + } + + return cachedAutoCompleteListForDataset; + } + + public static List getAutoCompleteListForMetric() + { + List cachedAutoCompleteListForMetric = (List)Cache.get(SEARCH_AUTOCOMPLETE_LIST_METRIC); + if (cachedAutoCompleteListForMetric == null || cachedAutoCompleteListForMetric.size() == 0) + { + List metricList = getJdbcTemplate().queryForList(GET_METRIC_AUTO_COMPLETE_LIST, String.class); + cachedAutoCompleteListForMetric = metricList.stream().collect(Collectors.toList()); + Collections.sort(cachedAutoCompleteListForMetric); + Cache.set(SEARCH_AUTOCOMPLETE_LIST_METRIC, cachedAutoCompleteListForMetric, 60*60); + } + + return cachedAutoCompleteListForMetric; + } + + public static List getAutoCompleteListForFlow() + { + List cachedAutoCompleteListForFlow = (List)Cache.get(SEARCH_AUTOCOMPLETE_LIST_FLOW); + if (cachedAutoCompleteListForFlow == null || cachedAutoCompleteListForFlow.size() == 0) + { + List flowList = getJdbcTemplate().queryForList(GET_FLOW_AUTO_COMPLETE_LIST, String.class); + List jobList = getJdbcTemplate().queryForList(GET_JOB_AUTO_COMPLETE_LIST, String.class); + cachedAutoCompleteListForFlow = + Stream.concat(flowList.stream(), jobList.stream()).collect(Collectors.toList()); + Collections.sort(cachedAutoCompleteListForFlow); + Cache.set(SEARCH_AUTOCOMPLETE_LIST_FLOW, cachedAutoCompleteListForFlow, 60*60); + } + + return cachedAutoCompleteListForFlow; + } + + public static List getSuggestionList(String category, String searchKeyword) + { + List SuggestionList = new ArrayList(); + String elasticSearchType = "dataset"; + String elasticSearchTypeURLKey = "elasticsearch.dataset.url"; + String fieldName = "name"; + + JsonNode responseNode = null; + ObjectNode keywordNode = null; + + try + { + String lCategory = category.toLowerCase(); + Logger.info("lCategory is " + category); + + switch (lCategory) { + case "dataset": + elasticSearchType = "dataset"; + elasticSearchTypeURLKey = "elasticsearch.dataset.url"; + fieldName = "name"; + break; + case "metric": + elasticSearchType = "metric"; + elasticSearchTypeURLKey = "elasticsearch.metric.url"; + fieldName = "metric_name"; + break; + case "flow": + elasticSearchType = "flow"; + elasticSearchTypeURLKey = "elasticsearch.flow.url"; + fieldName = "flow_name"; + break; + default: + break; + } + + keywordNode = utils.Search.generateElasticSearchPhraseSuggesterQuery(elasticSearchType, fieldName, searchKeyword); + } + catch(Exception e) + { + Logger.error("Elastic search phrase suggester error. Error message :" + e.getMessage()); + } + + Logger.info("The suggest query sent to Elastic Search is: " + keywordNode.toString()); + + Promise responsePromise = WS.url(Play.application().configuration().getString( + elasticSearchTypeURLKey)).post(keywordNode); + responseNode = responsePromise.get(1000).asJson(); + + // Logger.info("responseNode for getSuggestionList is " + responseNode.toString()); + + if (responseNode != null && responseNode.isContainerNode() && responseNode.has("hits")) + { + JsonNode suggestNode = responseNode.get("suggest"); + Logger.info("suggestNode is " + suggestNode.toString()); + if (suggestNode != null && suggestNode.has("simple_phrase")) + { + JsonNode simplePhraseNode = suggestNode.get("simple_phrase"); + + if (simplePhraseNode != null && simplePhraseNode.isArray()) + { + Iterator arrayIterator = simplePhraseNode.elements(); + if (arrayIterator != null) + { + while (arrayIterator.hasNext()) + { + JsonNode node = arrayIterator.next(); + if (node.isContainerNode() && node.has("options")) + { + JsonNode optionsNode = node.get("options"); + if (optionsNode != null && optionsNode.isArray()) + { + Iterator arrayIteratorOptions = optionsNode.elements(); + if (arrayIteratorOptions != null) + { + while (arrayIteratorOptions.hasNext()) + { + JsonNode textNode = arrayIteratorOptions.next(); + if (textNode != null && textNode.has("text")) + { + String oneSuggestion = textNode.get("text").asText(); + Logger.info("oneSuggestion is " + oneSuggestion); + SuggestionList.add(oneSuggestion); + } + } + } + } + } + } + } + } + } + } + return SuggestionList; + } + public static JsonNode elasticSearchDatasetByKeyword( String category, String keywords, diff --git a/web/app/utils/Search.java b/web/app/utils/Search.java index 9a310b29f4..e70228a50a 100644 --- a/web/app/utils/Search.java +++ b/web/app/utils/Search.java @@ -15,6 +15,7 @@ package utils; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.commons.lang3.StringUtils; import play.Logger; @@ -109,6 +110,25 @@ public class Search "}" + "}"; + private final static String suggesterQueryTemplateSub = + "{" + + "\"text\": \"$SEARCHKEYWORD\", " + + "\"simple_phrase\": { " + + "\"phrase\": { " + + "\"field\": \"$FIELD\", " + + "\"size\": 1, " + + "\"direct_generator\": [ " + + "{ " + + "\"field\": \"$FIELD\", " + + "\"suggest_mode\": \"always\", " + + "\"min_word_length\": 1 " + + "}" + + "]" + + "}" + + "}" + + "}"; + + public final static String DATASET_CATEGORY = "datasets"; public final static String METRIC_CATEGORY = "metrics"; @@ -119,6 +139,33 @@ public class Search public final static String JOB_CATEGORY = "jobs"; + public static ObjectNode generateElasticSearchPhraseSuggesterQuery(String category, String field, String searchKeyword) + { + if (StringUtils.isBlank(searchKeyword)) + return null; + + String queryTemplate = suggesterQueryTemplateSub; + String query= queryTemplate.replace("$SEARCHKEYWORD", searchKeyword.toLowerCase()); + + if (StringUtils.isNotBlank(field)) + { + query = query.replace("$FIELD", field.toLowerCase()); + } + + ObjectNode suggestNode = Json.newObject(); + ObjectNode textNode = Json.newObject(); + try { + textNode = (ObjectNode) new ObjectMapper().readTree(query); + suggestNode.put("suggest", textNode); + } + catch (Exception e) + { + Logger.error("suggest Exception = " + e.getMessage()); + } + Logger.info("suggestNode is " + suggestNode.toString()); + return suggestNode; + } + public static ObjectNode generateElasticSearchQueryString(String category, String source, String keywords) { if (StringUtils.isBlank(keywords)) diff --git a/web/conf/routes b/web/conf/routes index 99b67138c2..e5a4b1ed73 100644 --- a/web/conf/routes +++ b/web/conf/routes @@ -43,6 +43,12 @@ GET /api/v1/party/groups controllers.api.v1.User.getAllGroups GET /api/v1/autocomplete/search controllers.api.v1.Search.getSearchAutoComplete() +GET /api/v2/autocomplete/searchDataset controllers.api.v1.Search.getSearchAutoCompleteForDataset() + +GET /api/v2/autocomplete/searchMetric controllers.api.v1.Search.getSearchAutoCompleteForMetric() + +GET /api/v2/autocomplete/searchFlow controllers.api.v1.Search.getSearchAutoCompleteForFlow() + GET /api/v1/list/datasets controllers.api.v1.Dataset.getDatasetListNodes() GET /api/v1/list/flows controllers.api.v1.Flow.getFlowListViewClusters()