yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-10-31 10:49:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

add fabric facet for search

Browse Source
This commit is contained in:
Na Zhang 2018-09-10 14:16:27 -07:00
parent 67ccb31c65
commit d14911c231
4 changed files with 85 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
wherehows-dao/src/main/java/wherehows/dao/table/SearchDao.java
View File

@ -176,7 +176,7 @@ public class SearchDao {
}
public JsonNode elasticSearchDatasetByKeyword(String elasticSearchUrl, String category, String keywords,
String source, int page, int size) {
String source, int page, int size, String fabric) {
ObjectNode queryNode = _OM.createObjectNode();
queryNode.put("from", (page - 1) * size);
queryNode.put("size", size);
@ -202,7 +202,7 @@ public class SearchDao {
ObjectNode filterNode = _OM.createObjectNode();
try {
filterNode = generateElasticSearchFilterString(source);
filterNode = generateElasticSearchFilterString(source, fabric);
} catch (Exception e) {
log.error("Elastic search filter query node generation failed :" + e.getMessage());
}
@ -214,7 +214,7 @@ public class SearchDao {
log.info(" === elasticSearchDatasetByKeyword === The query sent to Elastic Search is: " + queryNode.toString());
responseNode = HttpUtil.httpPostRequest(elasticSearchUrl, queryNode);
// Logger.debug("The responseNode from Elastic Search is: " + responseNode.toString());
} catch (IOException e) {
log.error("Elastic search dataset query error: {}", e.toString());
}

75
wherehows-dao/src/main/java/wherehows/util/Search.java
View File

@ -39,13 +39,21 @@ public class Search {
private final static String FLOW_CATEGORY = "flows";
private final static String JOB_CATEGORY = "jobs";
private static final String WHZ_ELASTICSEARCH_DATASET_QUERY_FILE = System.getenv("WHZ_ELASTICSEARCH_DATASET_QUERY_TEMPLATE");
private static final String WHZ_ELASTICSEARCH_METRIC_QUERY_FILE = System.getenv("WHZ_ELASTICSEARCH_METRIC_QUERY_TEMPLATE");
private static final String WHZ_ELASTICSEARCH_FLOW_QUERY_FILE = System.getenv("WHZ_ELASTICSEARCH_FLOW_QUERY_TEMPLATE");
private static final String WHZ_ELASTICSEARCH_COMMENT_QUERY_FILE = System.getenv("WHZ_ELASTICSEARCH_COMMENT_QUERY_TEMPLATE");
private static final String WHZ_ELASTICSEARCH_SUGGESTER_QUERY_FILE = System.getenv("WHZ_ELASTICSEARCH_SUGGESTER_QUERY_TEMPLATE");
private static final String WHZ_ELASTICSEARCH_AUTO_COMPLETION_QUERY_FILE = System.getenv("WHZ_ELASTICSEARCH_AUTO_COMPLETION_QUERY_TEMPLATE");
private static final String WHZ_ELASTICSEARCH_DATASET_QUERY_FILE =
System.getenv("WHZ_ELASTICSEARCH_DATASET_QUERY_TEMPLATE");
private static final String WHZ_ELASTICSEARCH_METRIC_QUERY_FILE =
System.getenv("WHZ_ELASTICSEARCH_METRIC_QUERY_TEMPLATE");
private static final String WHZ_ELASTICSEARCH_FLOW_QUERY_FILE =
System.getenv("WHZ_ELASTICSEARCH_FLOW_QUERY_TEMPLATE");
private static final String WHZ_ELASTICSEARCH_COMMENT_QUERY_FILE =
System.getenv("WHZ_ELASTICSEARCH_COMMENT_QUERY_TEMPLATE");
private static final String WHZ_ELASTICSEARCH_SUGGESTER_QUERY_FILE =
System.getenv("WHZ_ELASTICSEARCH_SUGGESTER_QUERY_TEMPLATE");
private static final String WHZ_ELASTICSEARCH_AUTO_COMPLETION_QUERY_FILE =
System.getenv("WHZ_ELASTICSEARCH_AUTO_COMPLETION_QUERY_TEMPLATE");
private static final String WHZ_ELASTICSEARCH_FILTER_UNIT_FILE = System.getenv("WHZ_ELASTICSEARCH_FILTER_UNIT");
private static final String WHZ_ELASTICSEARCH_FILTER_UNIT_FILE_FABRIC =
System.getenv("WHZ_ELASTICSEARCH_FILTER_UNIT_FABRIC");
public static String readJsonQueryFile(String jsonFile) {
try {
@ -147,7 +155,7 @@ public class Search {
return queryNode;
}
public static ObjectNode generateElasticSearchFilterString(String sources) throws IOException {
public static ObjectNode generateESFilterStringForSources(String sources) throws IOException {
if (StringUtils.isBlank(sources)) {
return null;
}
@ -171,4 +179,57 @@ public class Search {
return queryNode;
}
public static ObjectNode generateESFilterStringForFabrics(String fabrics) throws IOException {
if (StringUtils.isBlank(fabrics)) {
return null;
}
List<JsonNode> shouldValueList = new ArrayList<JsonNode>();
String queryTemplate = readJsonQueryFile(WHZ_ELASTICSEARCH_FILTER_UNIT_FILE_FABRIC);
String[] values = fabrics.trim().split(",");
for (String value : values) {
if (StringUtils.isNotBlank(value)) {
String query = queryTemplate.replace("$FABRIC", value.replace("\"", "").toLowerCase().trim());
shouldValueList.add(_OM.readTree(query));
}
}
ObjectNode shouldNode = _OM.createObjectNode();
shouldNode.putPOJO("should", shouldValueList);
ObjectNode queryNode = _OM.createObjectNode();
queryNode.putPOJO("bool", shouldNode);
return queryNode;
}
public static ObjectNode generateElasticSearchFilterString(String sources, String fabrics) throws Exception {
ObjectNode sourcesNode = generateESFilterStringForSources(sources);
ObjectNode fabricsNode = generateESFilterStringForFabrics(fabrics);
if (sourcesNode == null && fabricsNode == null) {
return null;
}
List<JsonNode> mustValueList = new ArrayList<JsonNode>();
if (sourcesNode != null) {
mustValueList.add(generateESFilterStringForSources(sources));
}
if (fabricsNode != null) {
mustValueList.add(generateESFilterStringForFabrics(fabrics));
}
ObjectNode mustNode = _OM.createObjectNode();
mustNode.putPOJO("must", mustValueList);
ObjectNode queryNode = _OM.createObjectNode();
queryNode.putPOJO("bool", mustNode);
return queryNode;
}
}

5
wherehows-data-model/ELASTICSEARCH/filter_unit_fabric.json Normal file
View File

@ -0,0 +1,5 @@
{
"match": {
"dataorigin": "$FABRIC"
}
}

10
wherehows-frontend/app/controllers/api/v1/Search.java
View File

@ -96,6 +96,8 @@ public class Search extends Controller {
String category = request().getQueryString("category");
String source = request().getQueryString("source");
String pageStr = request().getQueryString("page");
String fabric = request().getQueryString("fabric");
if (isBlank(pageStr)) {
page = 1;
} else {
@ -124,12 +126,18 @@ public class Search extends Controller {
if (isBlank(category)) {
category = "datasets";
}
// Filter on platform
if (isBlank(source) || source.equalsIgnoreCase("all") || source.equalsIgnoreCase("default")) {
source = null;
}
// Filter on fabric
if (isBlank(fabric) || fabric.equalsIgnoreCase("all") || fabric.equalsIgnoreCase("default")) {
fabric = null;
}
result.set("result",
SEARCH_DAO.elasticSearchDatasetByKeyword(ELASTICSEARCH_DATASET_URL, category, keyword, source, page, size));
SEARCH_DAO.elasticSearchDatasetByKeyword(ELASTICSEARCH_DATASET_URL, category, keyword, source, page, size, fabric));
return ok(result);
}