yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-08-19 22:58:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

new type ahead api and phrase suggester support

Browse Source
This commit is contained in:
Na Zhang 2017-03-23 17:26:29 -07:00
parent 9ab223af4c
commit 4025153efb
4 changed files with 224 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
web/app/controllers/api/v1/Search.java
View File

@ -39,6 +39,34 @@ public class Search extends Controller
return ok(result);
}
public static Result getSearchAutoCompleteForDataset()
{
ObjectNode result = Json.newObject();
result.put("status", "ok");
result.set("source", Json.toJson(SearchDAO.getAutoCompleteListForDataset()));
return ok(result);
}
public static Result getSearchAutoCompleteForMetric()
{
ObjectNode result = Json.newObject();
result.put("status", "ok");
result.set("source", Json.toJson(SearchDAO.getAutoCompleteListForMetric()));
return ok(result);
}
public static Result getSearchAutoCompleteForFlow()
{
ObjectNode result = Json.newObject();
result.put("status", "ok");
result.set("source", Json.toJson(SearchDAO.getAutoCompleteListForFlow()));
return ok(result);
}
public static Result searchByKeyword()
{
ObjectNode result = Json.newObject();

149
web/app/dao/SearchDAO.java
View File

@ -147,15 +147,14 @@ public class SearchDAO extends AbstractMySQLOpenSourceDAO
"ORDER BY 2 LIMIT ?, ?;";
public final static String SEARCH_AUTOCOMPLETE_LIST = "searchSource";
public final static String GET_METRIC_AUTO_COMPLETE_LIST = "SELECT DISTINCT CASE " +
"WHEN parent_path is null or parent_path = '' THEN field_name " +
"ELSE CONCAT_WS('.', parent_path, field_name) END as full_name FROM dict_field_detail";
public final static String SEARCH_AUTOCOMPLETE_LIST_DATASET = "searchSourceDataset";
public final static String SEARCH_AUTOCOMPLETE_LIST_METRIC = "searchSourceMetric";
public final static String SEARCH_AUTOCOMPLETE_LIST_FLOW = "searchSourceFlow";
public final static String GET_DATASET_AUTO_COMPLETE_LIST = "SELECT DISTINCT name FROM dict_dataset";
public final static String GET_METRIC_AUTO_COMPLETE_LIST = "SELECT DISTINCT metric_name FROM dict_business_metric";
public final static String GET_FLOW_AUTO_COMPLETE_LIST = "SELECT DISTINCT flow_name FROM flow";
public final static String GET_JOB_AUTO_COMPLETE_LIST = "SELECT DISTINCT job_name FROM flow_job";
public static List<String> getAutoCompleteList()
@ -174,10 +173,148 @@ public class SearchDAO extends AbstractMySQLOpenSourceDAO
Cache.set(SEARCH_AUTOCOMPLETE_LIST, cachedAutoCompleteList, 60*60);
}
return cachedAutoCompleteList;
}
public static List<String> getAutoCompleteListForDataset()
{
List<String> cachedAutoCompleteListForDataset = (List<String>)Cache.get(SEARCH_AUTOCOMPLETE_LIST_DATASET);
if (cachedAutoCompleteListForDataset == null || cachedAutoCompleteListForDataset.size() == 0)
{
List<String> datasetList = getJdbcTemplate().queryForList(GET_DATASET_AUTO_COMPLETE_LIST, String.class);
cachedAutoCompleteListForDataset = datasetList.stream().collect(Collectors.toList());
Collections.sort(cachedAutoCompleteListForDataset);
Cache.set(SEARCH_AUTOCOMPLETE_LIST_DATASET, cachedAutoCompleteListForDataset, 60*60);
}
return cachedAutoCompleteListForDataset;
}
public static List<String> getAutoCompleteListForMetric()
{
List<String> cachedAutoCompleteListForMetric = (List<String>)Cache.get(SEARCH_AUTOCOMPLETE_LIST_METRIC);
if (cachedAutoCompleteListForMetric == null || cachedAutoCompleteListForMetric.size() == 0)
{
List<String> metricList = getJdbcTemplate().queryForList(GET_METRIC_AUTO_COMPLETE_LIST, String.class);
cachedAutoCompleteListForMetric = metricList.stream().collect(Collectors.toList());
Collections.sort(cachedAutoCompleteListForMetric);
Cache.set(SEARCH_AUTOCOMPLETE_LIST_METRIC, cachedAutoCompleteListForMetric, 60*60);
}
return cachedAutoCompleteListForMetric;
}
public static List<String> getAutoCompleteListForFlow()
{
List<String> cachedAutoCompleteListForFlow = (List<String>)Cache.get(SEARCH_AUTOCOMPLETE_LIST_FLOW);
if (cachedAutoCompleteListForFlow == null || cachedAutoCompleteListForFlow.size() == 0)
{
List<String> flowList = getJdbcTemplate().queryForList(GET_FLOW_AUTO_COMPLETE_LIST, String.class);
List<String> jobList = getJdbcTemplate().queryForList(GET_JOB_AUTO_COMPLETE_LIST, String.class);
cachedAutoCompleteListForFlow =
Stream.concat(flowList.stream(), jobList.stream()).collect(Collectors.toList());
Collections.sort(cachedAutoCompleteListForFlow);
Cache.set(SEARCH_AUTOCOMPLETE_LIST_FLOW, cachedAutoCompleteListForFlow, 60*60);
}
return cachedAutoCompleteListForFlow;
}
public static List<String> getSuggestionList(String category, String searchKeyword)
{
List<String> SuggestionList = new ArrayList<String>();
String elasticSearchType = "dataset";
String elasticSearchTypeURLKey = "elasticsearch.dataset.url";
String fieldName = "name";
JsonNode responseNode = null;
ObjectNode keywordNode = null;
try
{
String lCategory = category.toLowerCase();
Logger.info("lCategory is " + category);
switch (lCategory) {
case "dataset":
elasticSearchType = "dataset";
elasticSearchTypeURLKey = "elasticsearch.dataset.url";
fieldName = "name";
break;
case "metric":
elasticSearchType = "metric";
elasticSearchTypeURLKey = "elasticsearch.metric.url";
fieldName = "metric_name";
break;
case "flow":
elasticSearchType = "flow";
elasticSearchTypeURLKey = "elasticsearch.flow.url";
fieldName = "flow_name";
break;
default:
break;
}
keywordNode = utils.Search.generateElasticSearchPhraseSuggesterQuery(elasticSearchType, fieldName, searchKeyword);
}
catch(Exception e)
{
Logger.error("Elastic search phrase suggester error. Error message :" + e.getMessage());
}
Logger.info("The suggest query sent to Elastic Search is: " + keywordNode.toString());
Promise<WSResponse> responsePromise = WS.url(Play.application().configuration().getString(
elasticSearchTypeURLKey)).post(keywordNode);
responseNode = responsePromise.get(1000).asJson();
// Logger.info("responseNode for getSuggestionList is " + responseNode.toString());
if (responseNode != null && responseNode.isContainerNode() && responseNode.has("hits"))
{
JsonNode suggestNode = responseNode.get("suggest");
Logger.info("suggestNode is " + suggestNode.toString());
if (suggestNode != null && suggestNode.has("simple_phrase"))
{
JsonNode simplePhraseNode = suggestNode.get("simple_phrase");
if (simplePhraseNode != null && simplePhraseNode.isArray())
{
Iterator<JsonNode> arrayIterator = simplePhraseNode.elements();
if (arrayIterator != null)
{
while (arrayIterator.hasNext())
{
JsonNode node = arrayIterator.next();
if (node.isContainerNode() && node.has("options"))
{
JsonNode optionsNode = node.get("options");
if (optionsNode != null && optionsNode.isArray())
{
Iterator<JsonNode> arrayIteratorOptions = optionsNode.elements();
if (arrayIteratorOptions != null)
{
while (arrayIteratorOptions.hasNext())
{
JsonNode textNode = arrayIteratorOptions.next();
if (textNode != null && textNode.has("text"))
{
String oneSuggestion = textNode.get("text").asText();
Logger.info("oneSuggestion is " + oneSuggestion);
SuggestionList.add(oneSuggestion);
}
}
}
}
}
}
}
}
}
}
return SuggestionList;
}
public static JsonNode elasticSearchDatasetByKeyword(
String category,
String keywords,

47
web/app/utils/Search.java
View File

@ -15,6 +15,7 @@ package utils;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.commons.lang3.StringUtils;
import play.Logger;
@ -109,6 +110,25 @@ public class Search
"}" +
"}";
private final static String suggesterQueryTemplateSub =
"{" +
"\"text\": \"$SEARCHKEYWORD\", " +
"\"simple_phrase\": { " +
"\"phrase\": { " +
"\"field\": \"$FIELD\", " +
"\"size\": 1, " +
"\"direct_generator\": [ " +
"{ " +
"\"field\": \"$FIELD\", " +
"\"suggest_mode\": \"always\", " +
"\"min_word_length\": 1 " +
"}" +
"]" +
"}" +
"}" +
"}";
public final static String DATASET_CATEGORY = "datasets";
public final static String METRIC_CATEGORY = "metrics";
@ -119,6 +139,33 @@ public class Search
public final static String JOB_CATEGORY = "jobs";
public static ObjectNode generateElasticSearchPhraseSuggesterQuery(String category, String field, String searchKeyword)
{
if (StringUtils.isBlank(searchKeyword))
return null;
String queryTemplate = suggesterQueryTemplateSub;
String query= queryTemplate.replace("$SEARCHKEYWORD", searchKeyword.toLowerCase());
if (StringUtils.isNotBlank(field))
{
query = query.replace("$FIELD", field.toLowerCase());
}
ObjectNode suggestNode = Json.newObject();
ObjectNode textNode = Json.newObject();
try {
textNode = (ObjectNode) new ObjectMapper().readTree(query);
suggestNode.put("suggest", textNode);
}
catch (Exception e)
{
Logger.error("suggest Exception = " + e.getMessage());
}
Logger.info("suggestNode is " + suggestNode.toString());
return suggestNode;
}
public static ObjectNode generateElasticSearchQueryString(String category, String source, String keywords)
{
if (StringUtils.isBlank(keywords))

6
web/conf/routes
View File

@ -43,6 +43,12 @@ GET /api/v1/party/groups controllers.api.v1.User.getAllGroups
GET /api/v1/autocomplete/search controllers.api.v1.Search.getSearchAutoComplete()
GET /api/v2/autocomplete/searchDataset controllers.api.v1.Search.getSearchAutoCompleteForDataset()
GET /api/v2/autocomplete/searchMetric controllers.api.v1.Search.getSearchAutoCompleteForMetric()
GET /api/v2/autocomplete/searchFlow controllers.api.v1.Search.getSearchAutoCompleteForFlow()
GET /api/v1/list/datasets controllers.api.v1.Dataset.getDatasetListNodes()
GET /api/v1/list/flows controllers.api.v1.Flow.getFlowListViewClusters()