yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-11-03 20:27:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor search dao and remove unused routes

Browse Source
This commit is contained in:
Na Zhang 2018-08-23 16:17:27 -07:00
parent 1558e8b127
commit 8ff0899f9d
9 changed files with 526 additions and 4449 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
wherehows-dao/build.gradle
View File

@ -15,6 +15,9 @@ dependencies {
runtime externalDependency.hikaricp
testCompile externalDependency.testng
compile externalDependency.play_java_ws
}
findbugs {
@ -31,4 +34,4 @@ tasks.withType(FindBugs) {
test {
// enable TestNG support (default is JUnit)
useTestNG()
}
}

344
wherehows-dao/src/main/java/wherehows/dao/table/SearchDAO.java Normal file
View File

@ -0,0 +1,344 @@
/**
* Copyright 2015 LinkedIn Corp. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*/
package wherehows.dao.table;
import java.util.*;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import play.Logger;
import play.Play;
import play.libs.F.Promise;
import play.libs.Json;
import play.libs.ws.*;
import wherehows.models.table.Dataset;
import static wherehows.util.Search.*;
public class SearchDAO
{
public static String ELASTICSEARCH_DATASET_URL_KEY = "elasticsearch.dataset.url";
public static String WHEREHOWS_SEARCH_ENGINE_KEY = "search.engine";
public static List<String> getAutoCompleteList(String input, int limit)
{
List<String> names = new ArrayList<>();
names.addAll(getAutoCompleteListDataset(input, limit / 3));
return names;
}
public static List<String> getAutoCompleteListDataset(String input, int limit)
{
String elasticSearchTypeURLKey = "elasticsearch.dataset.url";
String fieldName = "name_suggest";
return getAutoCompleteListbyES(elasticSearchTypeURLKey,fieldName,input,limit);
}
public static List<String> getAutoCompleteListbyES(String elasticSearchTypeURLKey, String fieldName, String input,
int limit)
{
// use elastic search completion suggester, ES will validate the input and limit
List<String> completionSuggestionList = new ArrayList<String>();
Set<String> completionSuggestionSet = new HashSet<String>();
JsonNode responseNode = null;
ObjectNode keywordNode = null;
try {
keywordNode = generateElasticSearchCompletionSuggesterQuery(fieldName, input, limit);
} catch (Exception e) {
Logger.error("Elastic search completion suggester error. Error message :" + e.getMessage());
}
Logger.info("The completion suggester query sent to Elastic Search was: " + keywordNode.toString());
Promise<WSResponse> responsePromise =
WS.url(Play.application().configuration().getString(elasticSearchTypeURLKey)).post(keywordNode);
responseNode = responsePromise.get(1000).asJson();
if (responseNode == null || !responseNode.isContainerNode()) {
return completionSuggestionList;
}
JsonNode suggestNode = responseNode.get("suggest");
if (suggestNode == null || !suggestNode.has("wh-suggest")) {
Logger.error("Elastic search completion suggester response does not contain suggest node");
return completionSuggestionList;
}
JsonNode whSuggestNode = suggestNode.get("wh-suggest");
if (whSuggestNode == null || !whSuggestNode.isArray()) {
Logger.error("Elastic search completion suggester response does not contain wh-suggest node");
return completionSuggestionList;
}
Iterator<JsonNode> arrayIterator = whSuggestNode.elements();
if (arrayIterator == null) {
return completionSuggestionList;
}
while (arrayIterator.hasNext()) {
JsonNode node = arrayIterator.next();
if (!node.isContainerNode() || !node.has("options")) {
continue;
}
JsonNode optionsNode = node.get("options");
if (optionsNode == null || !optionsNode.isArray()) {
continue;
}
Iterator<JsonNode> arrayIteratorOptions = optionsNode.elements();
if (arrayIteratorOptions == null) {
continue;
}
while (arrayIteratorOptions.hasNext()) {
JsonNode textNode = arrayIteratorOptions.next();
if (textNode == null || !textNode.has("text")) {
continue;
}
String oneSuggestion = textNode.get("text").asText();
completionSuggestionSet.add(oneSuggestion);
}
}
completionSuggestionList.addAll(completionSuggestionSet);
Logger.info("Returned suggestion list is: " + completionSuggestionList);
return completionSuggestionList;
}
// this is for did you mean feature
public static List<String> getSuggestionList(String category, String searchKeyword)
{
List<String> SuggestionList = new ArrayList<String>();
String elasticSearchType = "dataset";
String elasticSearchTypeURLKey = "elasticsearch.dataset.url";
String fieldName = "name";
JsonNode responseNode = null;
ObjectNode keywordNode = null;
try {
String lCategory = category.toLowerCase();
Logger.info("lCategory is " + category);
// ToDO: deprecate category or reuse for entity
switch (lCategory) {
case "dataset":
elasticSearchType = "dataset";
elasticSearchTypeURLKey = "elasticsearch.dataset.url";
fieldName = "name";
break;
default:
break;
}
keywordNode = generateElasticSearchPhraseSuggesterQuery(elasticSearchType, fieldName, searchKeyword);
} catch (Exception e) {
Logger.error("Elastic search phrase suggester error. Error message :" + e.getMessage());
}
Logger.info("The suggest query sent to Elastic Search is: " + keywordNode.toString());
Promise<WSResponse> responsePromise =
WS.url(Play.application().configuration().getString(elasticSearchTypeURLKey)).post(keywordNode);
responseNode = responsePromise.get(1000).asJson();
if (responseNode == null || !responseNode.isContainerNode() || !responseNode.has("hits")) {
return SuggestionList;
}
JsonNode suggestNode = responseNode.get("suggest");
Logger.info("suggestNode is " + suggestNode.toString());
if (suggestNode == null || !suggestNode.has("simple_phrase")) {
return SuggestionList;
}
JsonNode simplePhraseNode = suggestNode.get("simple_phrase");
if (simplePhraseNode == null || !simplePhraseNode.isArray()) {
return SuggestionList;
}
Iterator<JsonNode> arrayIterator = simplePhraseNode.elements();
if (arrayIterator == null) {
return SuggestionList;
}
while (arrayIterator.hasNext()) {
JsonNode node = arrayIterator.next();
if (!node.isContainerNode() || !node.has("options")) {
continue;
}
JsonNode optionsNode = node.get("options");
if (optionsNode == null || !optionsNode.isArray()) {
continue;
}
Iterator<JsonNode> arrayIteratorOptions = optionsNode.elements();
if (arrayIteratorOptions == null) {
continue;
}
while (arrayIteratorOptions.hasNext()) {
JsonNode textNode = arrayIteratorOptions.next();
if (textNode == null || !textNode.has("text")) {
continue;
}
String oneSuggestion = textNode.get("text").asText();
SuggestionList.add(oneSuggestion);
}
}
return SuggestionList;
}
public static JsonNode elasticSearchDatasetByKeyword(
String category,
String keywords,
String source,
int page,
int size)
{
ObjectNode queryNode = Json.newObject();
queryNode.put("from", (page-1)*size);
queryNode.put("size", size);
JsonNode responseNode = null;
ObjectNode keywordNode = null;
try {
keywordNode = generateElasticSearchQueryString(category, source, keywords);
} catch (Exception e) {
Logger.error("Elastic search dataset input query is not JSON format. Error message :" + e.getMessage());
}
if (keywordNode != null) {
ObjectNode funcScoreNodes = Json.newObject();
ObjectNode fieldValueFactorNode = Json.newObject();
fieldValueFactorNode.put("field", "static_boosting_score");
fieldValueFactorNode.put("factor", 1);
fieldValueFactorNode.put("modifier", "square");
fieldValueFactorNode.put("missing", 1);
funcScoreNodes.put("query", keywordNode);
funcScoreNodes.put("field_value_factor", fieldValueFactorNode);
ObjectNode funcScoreNodesWrapper = Json.newObject();
funcScoreNodesWrapper.put("function_score", funcScoreNodes);
queryNode.put("query", funcScoreNodesWrapper);
ObjectNode filterNode = Json.newObject();
try {
filterNode = generateElasticSearchFilterString(source);
} catch (Exception e) {
Logger.error("Elastic search filter query node generation failed :" + e.getMessage());
}
if (filterNode != null) {
queryNode.put("post_filter", filterNode);
}
Logger.info(
" === elasticSearchDatasetByKeyword === The query sent to Elastic Search is: " + queryNode.toString());
Promise<WSResponse> responsePromise =
WS.url(Play.application().configuration().getString(SearchDAO.ELASTICSEARCH_DATASET_URL_KEY)).post(queryNode);
responseNode = responsePromise.get(1000).asJson();
// Logger.debug("The responseNode from Elastic Search is: " + responseNode.toString());
}
ObjectNode resultNode = Json.newObject();
Long count = 0L;
List<Dataset> pagedDatasets = new ArrayList<>();
resultNode.put("page", page);
resultNode.put("category", category);
resultNode.put("source", source);
resultNode.put("itemsPerPage", size);
resultNode.put("keywords", keywords);
if (responseNode != null && responseNode.isContainerNode() && responseNode.has("hits"))
{
JsonNode hitsNode = responseNode.get("hits");
if (hitsNode != null)
{
if (hitsNode.has("total"))
{
count = hitsNode.get("total").asLong();
}
if (hitsNode.has("hits"))
{
JsonNode dataNode = hitsNode.get("hits");
if (dataNode != null && dataNode.isArray())
{
Iterator<JsonNode> arrayIterator = dataNode.elements();
if (arrayIterator != null)
{
while (arrayIterator.hasNext())
{
JsonNode node = arrayIterator.next();
if (node.isContainerNode() && node.has("_id"))
{
Dataset dataset = new Dataset();
dataset.id = node.get("_id").asLong();
if (node.has("_source"))
{
JsonNode sourceNode = node.get("_source");
if (sourceNode != null)
{
if (sourceNode.has("name"))
{
dataset.name = sourceNode.get("name").asText();
}
if (sourceNode.has("source"))
{
dataset.source = sourceNode.get("source").asText();
}
if (sourceNode.has("urn"))
{
dataset.urn = sourceNode.get("urn").asText();
}
if (sourceNode.has("schema"))
{
dataset.schema = sourceNode.get("schema").asText();
}
}
}
pagedDatasets.add(dataset);
}
}
}
}
}
}
}
resultNode.put("count", count);
resultNode.put("totalPages", (int)Math.ceil(count/((double)size)));
resultNode.set("data", Json.toJson(pagedDatasets));
return resultNode;
}
}

174
wherehows-dao/src/main/java/wherehows/util/Search.java Normal file
View File

@ -0,0 +1,174 @@
/**
* Copyright 2015 LinkedIn Corp. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*/
package wherehows.util;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import java.util.ArrayList;
import java.util.List;
import java.nio.file.Paths;
import java.nio.file.Files;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
@Slf4j
public class Search {
public final static String DATASET_CATEGORY = "datasets";
public final static String METRIC_CATEGORY = "metrics";
public final static String COMMENT_CATEGORY = "comments";
public final static String FLOW_CATEGORY = "flows";
public final static String JOB_CATEGORY = "jobs";
private static final String WHZ_ELASTICSEARCH_DATASET_QUERY_FILE = System.getenv("WHZ_ELASTICSEARCH_DATASET_QUERY_TEMPLATE");
private static final String WHZ_ELASTICSEARCH_METRIC_QUERY_FILE = System.getenv("WHZ_ELASTICSEARCH_METRIC_QUERY_TEMPLATE");
private static final String WHZ_ELASTICSEARCH_FLOW_QUERY_FILE = System.getenv("WHZ_ELASTICSEARCH_FLOW_QUERY_TEMPLATE");
private static final String WHZ_ELASTICSEARCH_COMMENT_QUERY_FILE = System.getenv("WHZ_ELASTICSEARCH_COMMENT_QUERY_TEMPLATE");
private static final String WHZ_ELASTICSEARCH_SUGGESTER_QUERY_FILE = System.getenv("WHZ_ELASTICSEARCH_SUGGESTER_QUERY_TEMPLATE");
private static final String WHZ_ELASTICSEARCH_AUTO_COMPLETION_QUERY_FILE = System.getenv("WHZ_ELASTICSEARCH_AUTO_COMPLETION_QUERY_TEMPLATE");
private static final String WHZ_ELASTICSEARCH_FILTER_UNIT_FILE = System.getenv("WHZ_ELASTICSEARCH_FILTER_UNIT");
public static String readJsonQueryFile(String jsonFile) {
try {
ObjectMapper objectMapper = new ObjectMapper();
String contents = new String(Files.readAllBytes(Paths.get(jsonFile)));
JsonNode json = objectMapper.readTree(contents);
return json.toString();
} catch (Exception e) {
log.error("ReadJsonQueryFile failed. Error: " + e.getMessage());
e.printStackTrace();
return null;
}
}
public static ObjectNode generateElasticSearchCompletionSuggesterQuery(String field, String searchKeyword,
int limit) {
if (StringUtils.isBlank(searchKeyword)) {
return null;
}
String queryTemplate = readJsonQueryFile(WHZ_ELASTICSEARCH_AUTO_COMPLETION_QUERY_FILE);
String query = queryTemplate.replace("$SEARCHKEYWORD", searchKeyword.toLowerCase());
if (StringUtils.isNotBlank(field)) {
query = query.replace("$FIELD", field.toLowerCase());
}
query = query.replace("$LIMIT", Integer.toString(limit));
ObjectNode suggestNode = new ObjectMapper().createObjectNode();
try {
ObjectNode textNode = (ObjectNode) new ObjectMapper().readTree(query);
suggestNode.putPOJO("suggest", textNode);
} catch (Exception e) {
log.error("suggest Exception = " + e.getMessage());
}
log.info("completionSuggesterQuery suggestNode is " + suggestNode.toString());
return suggestNode;
}
public static ObjectNode generateElasticSearchPhraseSuggesterQuery(String category, String field,
String searchKeyword) {
if (StringUtils.isBlank(searchKeyword)) {
return null;
}
String queryTemplate = readJsonQueryFile(WHZ_ELASTICSEARCH_SUGGESTER_QUERY_FILE);
String query = queryTemplate.replace("$SEARCHKEYWORD", searchKeyword.toLowerCase());
if (StringUtils.isNotBlank(field)) {
query = query.replace("$FIELD", field.toLowerCase());
}
ObjectNode suggestNode = new ObjectMapper().createObjectNode();
try {
ObjectNode textNode = (ObjectNode) new ObjectMapper().readTree(query);
suggestNode.putPOJO("suggest", textNode);
} catch (Exception e) {
log.error("suggest Exception = " + e.getMessage());
}
log.info("suggestNode is " + suggestNode.toString());
return suggestNode;
}
public static ObjectNode generateElasticSearchQueryString(String category, String source, String keywords) throws Exception
{
if (StringUtils.isBlank(keywords)) {
return null;
}
List<JsonNode> shouldValueList = new ArrayList<JsonNode>();
String queryTemplate = readJsonQueryFile(WHZ_ELASTICSEARCH_DATASET_QUERY_FILE);
String[] values = keywords.trim().split(",");
if (StringUtils.isNotBlank(category)) {
if (category.equalsIgnoreCase(METRIC_CATEGORY)) {
queryTemplate = readJsonQueryFile(WHZ_ELASTICSEARCH_METRIC_QUERY_FILE);
;
} else if (category.equalsIgnoreCase(COMMENT_CATEGORY)) {
queryTemplate = readJsonQueryFile(WHZ_ELASTICSEARCH_COMMENT_QUERY_FILE);
} else if (category.equalsIgnoreCase(FLOW_CATEGORY) || category.equalsIgnoreCase(JOB_CATEGORY)) {
queryTemplate = readJsonQueryFile(WHZ_ELASTICSEARCH_FLOW_QUERY_FILE);
}
}
ObjectMapper objectMapper = new ObjectMapper();
for (String value : values) {
if (StringUtils.isNotBlank(value)) {
String query = queryTemplate.replace("$VALUE", value.replace("\"", "").toLowerCase().trim());
shouldValueList.add(objectMapper.readTree(query));
}
}
ObjectNode shouldNode = new ObjectMapper().createObjectNode();
shouldNode.putPOJO("should", shouldValueList);
ObjectNode queryNode = new ObjectMapper().createObjectNode();
queryNode.putPOJO("bool", shouldNode);
return queryNode;
}
public static ObjectNode generateElasticSearchFilterString(String sources) throws Exception{
if (StringUtils.isBlank(sources)) {
return null;
}
List<JsonNode> shouldValueList = new ArrayList<JsonNode>();
String queryTemplate = readJsonQueryFile(WHZ_ELASTICSEARCH_FILTER_UNIT_FILE);
String[] values = sources.trim().split(",");
ObjectMapper objectMapper = new ObjectMapper();
for (String value : values) {
if (StringUtils.isNotBlank(value)) {
String query = queryTemplate.replace("$SOURCE", value.replace("\"", "").toLowerCase().trim());
shouldValueList.add(objectMapper.readTree(query));
}
}
ObjectNode shouldNode = new ObjectMapper().createObjectNode();
shouldNode.putPOJO("should", shouldValueList);
ObjectNode queryNode = new ObjectMapper().createObjectNode();
queryNode.putPOJO("bool", shouldNode);
return queryNode;
}
}

181
wherehows-frontend/app/controllers/api/v1/AdvSearch.java
View File

@ -1,181 +0,0 @@
/**
* Copyright 2015 LinkedIn Corp. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*/
package controllers.api.v1;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import dao.AdvSearchDAO;
import dao.SearchDAO;
import org.apache.commons.lang3.StringUtils;
import play.Logger;
import play.Play;
import play.libs.Json;
import play.mvc.Controller;
import play.mvc.Result;
public class AdvSearch extends Controller {
public static Result getDatasetSources() {
ObjectNode result = Json.newObject();
result.put("status", "ok");
result.set("sources", Json.toJson(AdvSearchDAO.getDatasetSources()));
return ok(result);
}
public static Result getDatasetScopes() {
ObjectNode result = Json.newObject();
result.put("status", "ok");
result.set("scopes", Json.toJson(AdvSearchDAO.getDatasetScopes()));
return ok(result);
}
public static Result getDatasetTableNames() {
ObjectNode result = Json.newObject();
String scopes = request().getQueryString("scopes");
result.put("status", "ok");
result.set("tables", Json.toJson(AdvSearchDAO.getTableNames(scopes)));
return ok(result);
}
public static Result getDatasetFields() {
ObjectNode result = Json.newObject();
String tables = request().getQueryString("tables");
result.put("status", "ok");
result.set("fields", Json.toJson(AdvSearchDAO.getFields(tables)));
return ok(result);
}
public static Result getFlowApplicationCodes() {
ObjectNode result = Json.newObject();
result.put("status", "ok");
result.set("appcodes", Json.toJson(AdvSearchDAO.getFlowApplicationCodes()));
return ok(result);
}
public static Result getFlowNames() {
ObjectNode result = Json.newObject();
String apps = request().getQueryString("apps");
result.put("status", "ok");
result.set("flowNames", Json.toJson(AdvSearchDAO.getFlowNames(apps)));
return ok(result);
}
public static Result getJobNames() {
ObjectNode result = Json.newObject();
result.put("status", "ok");
result.set("jobNames", Json.toJson(AdvSearchDAO.getFlowJobNames()));
return ok(result);
}
public static Result getDashboardNames() {
ObjectNode result = Json.newObject();
result.put("status", "ok");
result.set("dashboardNames", Json.toJson(AdvSearchDAO.getMetricDashboardNames()));
return ok(result);
}
public static Result getMetricGroups() {
ObjectNode result = Json.newObject();
result.put("status", "ok");
result.set("metricGroups", Json.toJson(AdvSearchDAO.getMetricGroups()));
return ok(result);
}
public static Result getMetricCategories() {
ObjectNode result = Json.newObject();
result.put("status", "ok");
result.set("metricCategories", Json.toJson(AdvSearchDAO.getMetricCategories()));
return ok(result);
}
public static Result getMetricNames() {
ObjectNode result = Json.newObject();
result.put("status", "ok");
result.set("metricNames", Json.toJson(AdvSearchDAO.getMetricNames()));
return ok(result);
}
public static Result search() {
ObjectNode result = Json.newObject();
String searchOptStr = request().getQueryString("searchOpts");
JsonNode searchOpt = Json.parse(searchOptStr);
int page = 1;
int size = 10;
String pageStr = request().getQueryString("page");
if (StringUtils.isBlank(pageStr)) {
page = 1;
} else {
try {
page = Integer.parseInt(pageStr);
} catch (NumberFormatException e) {
Logger.error("AdvSearch Controller search wrong page parameter. Error message: " + e.getMessage());
page = 1;
}
}
String sizeStr = request().getQueryString("size");
if (StringUtils.isBlank(sizeStr)) {
size = 10;
} else {
try {
size = Integer.parseInt(sizeStr);
} catch (NumberFormatException e) {
Logger.error("AdvSearch Controller search wrong page parameter. Error message: " + e.getMessage());
size = 10;
}
}
result.put("status", "ok");
String searchEngine = Play.application().configuration().getString(SearchDAO.WHEREHOWS_SEARCH_ENGINE_KEY);
if (searchOpt != null && searchOpt.has("category")) {
String category = searchOpt.get("category").asText();
if (category.equalsIgnoreCase("flow")) {
if (StringUtils.isNotBlank(searchEngine) && searchEngine.equalsIgnoreCase("elasticsearch")) {
result.set("result", Json.toJson(AdvSearchDAO.elasticSearchFlowJobs(searchOpt, page, size)));
} else {
result.set("result", Json.toJson(AdvSearchDAO.searchFlows(searchOpt, page, size)));
}
return ok(result);
} else if (category.equalsIgnoreCase("metric")) {
if (StringUtils.isNotBlank(searchEngine) && searchEngine.equalsIgnoreCase("elasticsearch")) {
result.set("result", Json.toJson(AdvSearchDAO.elasticSearchMetric(searchOpt, page, size)));
} else {
result.set("result", Json.toJson(AdvSearchDAO.searchMetrics(searchOpt, page, size)));
}
return ok(result);
}
}
if (StringUtils.isNotBlank(searchEngine) && searchEngine.equalsIgnoreCase("elasticsearch")) {
result.set("result", Json.toJson(AdvSearchDAO.elasticSearch(searchOpt, page, size)));
} else {
result.set("result", Json.toJson(AdvSearchDAO.search(searchOpt, page, size)));
}
return ok(result);
}
}

80
wherehows-frontend/app/controllers/api/v1/Search.java
View File

@ -14,7 +14,6 @@
package controllers.api.v1;
import com.fasterxml.jackson.databind.node.ObjectNode;
import dao.SearchDAO;
import java.util.List;
import org.apache.commons.lang3.math.NumberUtils;
import play.Logger;
@ -26,12 +25,12 @@ import play.mvc.Result;
import static org.apache.commons.lang3.StringUtils.*;
import wherehows.dao.table.SearchDAO;
public class Search extends Controller {
private static final String AUTOCOMPLETE_ALL_KEY = "autocomplete.all";
private static final String AUTOCOMPLETE_DATASET_KEY = "autocomplete.dataset";
private static final String AUTOCOMPLETE_FLOW_KEY = "autocomplete.flow";
private static final String AUTOCOMPLETE_METRIC_KEY = "autocomplete.metric";
private static final int DEFAULT_AUTOCOMPLETE_SIZE = 20;
private static final int DEFAULT_AUTOCOMPLETE_CACHE_TIME = 3600; // cache for an hour
@ -79,49 +78,7 @@ public class Search extends Controller {
return ok(result);
}
public static Result getSearchAutoCompleteForMetric() {
// if not input, then get all search names (without limit).
String input = request().getQueryString("input");
int size = 0; // size 0 means no limit
if (isNotBlank(input)) {
size = NumberUtils.toInt(request().getQueryString("size"), DEFAULT_AUTOCOMPLETE_SIZE);
}
String cacheKey = AUTOCOMPLETE_METRIC_KEY + (isNotBlank(input) ? "." + input : "-all");
List<String> names = (List<String>) Cache.get(cacheKey);
if (names == null || names.size() == 0) {
names = SearchDAO.getAutoCompleteListMetric(input, size);
Cache.set(cacheKey, names, DEFAULT_AUTOCOMPLETE_CACHE_TIME);
}
ObjectNode result = Json.newObject();
result.put("status", "ok");
result.put("input", input);
result.set("source", Json.toJson(names));
return ok(result);
}
public static Result getSearchAutoCompleteForFlow() {
// if not input, then get all search names (without limit).
String input = request().getQueryString("input");
int size = 0; // size 0 means no limit
if (isNotBlank(input)) {
size = NumberUtils.toInt(request().getQueryString("size"), DEFAULT_AUTOCOMPLETE_SIZE);
}
String cacheKey = AUTOCOMPLETE_FLOW_KEY + (isNotBlank(input) ? "." + input : "-all");
List<String> names = (List<String>) Cache.get(cacheKey);
if (names == null || names.size() == 0) {
names = SearchDAO.getAutoCompleteListFlow(input, size);
Cache.set(cacheKey, names, DEFAULT_AUTOCOMPLETE_CACHE_TIME);
}
ObjectNode result = Json.newObject();
result.put("status", "ok");
result.put("input", input);
result.set("source", Json.toJson(names));
return ok(result);
}
public static Result searchByKeyword() {
ObjectNode result = Json.newObject();
@ -165,38 +122,9 @@ public class Search extends Controller {
}
String searchEngine = Play.application().configuration().getString(SearchDAO.WHEREHOWS_SEARCH_ENGINE_KEY);
Logger.info("searchEngine is: " + searchEngine); // TODO: deprecated this setting
if (category.toLowerCase().equalsIgnoreCase("metrics")) {
if (isNotBlank(searchEngine) && searchEngine.equalsIgnoreCase("elasticsearch")) {
result.set("result", SearchDAO.elasticSearchMetricByKeyword(category, keyword, page, size));
} else {
result.set("result", SearchDAO.getPagedMetricByKeyword(category, keyword, page, size));
}
} else if (category.toLowerCase().equalsIgnoreCase("flows")) {
if (isNotBlank(searchEngine) && searchEngine.equalsIgnoreCase("elasticsearch")) {
result.set("result", SearchDAO.elasticSearchFlowByKeyword(category, keyword, page, size));
} else {
result.set("result", SearchDAO.getPagedFlowByKeyword(category, keyword, page, size));
}
} else if (category.toLowerCase().equalsIgnoreCase("jobs")) {
if (isNotBlank(searchEngine) && searchEngine.equalsIgnoreCase("elasticsearch")) {
result.set("result", SearchDAO.elasticSearchFlowByKeyword(category, keyword, page, size));
} else {
result.set("result", SearchDAO.getPagedJobByKeyword(category, keyword, page, size));
}
} else if (category.toLowerCase().equalsIgnoreCase("comments")) {
if (isNotBlank(searchEngine) && searchEngine.equalsIgnoreCase("elasticsearch")) {
result.set("result", SearchDAO.elasticSearchDatasetByKeyword(category, keyword, null, page, size));
} else {
result.set("result", SearchDAO.getPagedCommentsByKeyword(category, keyword, page, size));
}
} else {
if (isNotBlank(searchEngine) && searchEngine.equalsIgnoreCase("elasticsearch")) {
result.set("result", SearchDAO.elasticSearchDatasetByKeyword(category, keyword, source, page, size));
} else {
result.set("result", SearchDAO.getPagedDatasetByKeyword(category, keyword, source, page, size));
}
}
result.set("result", SearchDAO.elasticSearchDatasetByKeyword(category, keyword, source, page, size));
return ok(result);
}

2071
wherehows-frontend/app/dao/AdvSearchDAO.java
View File

File diff suppressed because it is too large Load Diff

1052
wherehows-frontend/app/dao/SearchDAO.java
View File

File diff suppressed because it is too large Load Diff

1040
wherehows-frontend/app/utils/Search.java
View File

File diff suppressed because it is too large Load Diff

28
wherehows-frontend/conf/routes
View File

@ -54,10 +54,6 @@ GET /api/v1/autocomplete/search
GET /api/v1/autocomplete/datasets controllers.api.v1.Search.getSearchAutoCompleteForDataset()
GET /api/v1/autocomplete/metrics controllers.api.v1.Search.getSearchAutoCompleteForMetric()
GET /api/v1/autocomplete/flows controllers.api.v1.Search.getSearchAutoCompleteForFlow()
GET /api/v1/list/datasets controllers.api.v1.Dataset.getDatasetListNodes()
GET /api/v1/list/metrics controllers.api.v1.Metric.getMetricListViewDashboards()
@ -190,30 +186,6 @@ GET /api/v1/flows/:application/:project
GET /api/v1/flow/:application/:flowId controllers.api.v1.Flow.getPagedJobs(application: String, flowId: Long)
GET /api/v1/advsearch/sources controllers.api.v1.AdvSearch.getDatasetSources()
GET /api/v1/advsearch/scopes controllers.api.v1.AdvSearch.getDatasetScopes()
GET /api/v1/advsearch/tables controllers.api.v1.AdvSearch.getDatasetTableNames()
GET /api/v1/advsearch/fields controllers.api.v1.AdvSearch.getDatasetFields()
GET /api/v1/advsearch/appcodes controllers.api.v1.AdvSearch.getFlowApplicationCodes()
GET /api/v1/advsearch/flowNames controllers.api.v1.AdvSearch.getFlowNames()
GET /api/v1/advsearch/jobNames controllers.api.v1.AdvSearch.getJobNames()
GET /api/v1/advsearch/dashboards controllers.api.v1.AdvSearch.getDashboardNames()
GET /api/v1/advsearch/metricGroups controllers.api.v1.AdvSearch.getMetricGroups()
GET /api/v1/advsearch/metricCategories controllers.api.v1.AdvSearch.getMetricCategories()
GET /api/v1/advsearch/metricNames controllers.api.v1.AdvSearch.getMetricNames()
GET /api/v1/advsearch/search controllers.api.v1.AdvSearch.search()
GET /api/v1/lineage/dataset/:id controllers.api.v1.Lineage.getDatasetLineageGraphData(id:Int)
GET /api/v1/lineage/flow/:application/:project/:flowId controllers.api.v1.Lineage.getFlowLineageGraphData(application: String, project: String, flowId: Long)