diff --git a/wherehows-dao/src/main/java/wherehows/dao/table/SearchDao.java b/wherehows-dao/src/main/java/wherehows/dao/table/SearchDao.java index 0a4dd4ac72..8356b7ee85 100644 --- a/wherehows-dao/src/main/java/wherehows/dao/table/SearchDao.java +++ b/wherehows-dao/src/main/java/wherehows/dao/table/SearchDao.java @@ -176,7 +176,7 @@ public class SearchDao { } public JsonNode elasticSearchDatasetByKeyword(String elasticSearchUrl, String category, String keywords, - String source, int page, int size) { + String source, int page, int size, String fabric) { ObjectNode queryNode = _OM.createObjectNode(); queryNode.put("from", (page - 1) * size); queryNode.put("size", size); @@ -202,7 +202,7 @@ public class SearchDao { ObjectNode filterNode = _OM.createObjectNode(); try { - filterNode = generateElasticSearchFilterString(source); + filterNode = generateElasticSearchFilterString(source, fabric); } catch (Exception e) { log.error("Elastic search filter query node generation failed :" + e.getMessage()); } @@ -214,7 +214,7 @@ public class SearchDao { log.info(" === elasticSearchDatasetByKeyword === The query sent to Elastic Search is: " + queryNode.toString()); responseNode = HttpUtil.httpPostRequest(elasticSearchUrl, queryNode); - // Logger.debug("The responseNode from Elastic Search is: " + responseNode.toString()); + } catch (IOException e) { log.error("Elastic search dataset query error: {}", e.toString()); } diff --git a/wherehows-dao/src/main/java/wherehows/util/Search.java b/wherehows-dao/src/main/java/wherehows/util/Search.java index a79ca63764..bc413fd0ac 100644 --- a/wherehows-dao/src/main/java/wherehows/util/Search.java +++ b/wherehows-dao/src/main/java/wherehows/util/Search.java @@ -39,13 +39,21 @@ public class Search { private final static String FLOW_CATEGORY = "flows"; private final static String JOB_CATEGORY = "jobs"; - private static final String WHZ_ELASTICSEARCH_DATASET_QUERY_FILE = System.getenv("WHZ_ELASTICSEARCH_DATASET_QUERY_TEMPLATE"); - private static final String WHZ_ELASTICSEARCH_METRIC_QUERY_FILE = System.getenv("WHZ_ELASTICSEARCH_METRIC_QUERY_TEMPLATE"); - private static final String WHZ_ELASTICSEARCH_FLOW_QUERY_FILE = System.getenv("WHZ_ELASTICSEARCH_FLOW_QUERY_TEMPLATE"); - private static final String WHZ_ELASTICSEARCH_COMMENT_QUERY_FILE = System.getenv("WHZ_ELASTICSEARCH_COMMENT_QUERY_TEMPLATE"); - private static final String WHZ_ELASTICSEARCH_SUGGESTER_QUERY_FILE = System.getenv("WHZ_ELASTICSEARCH_SUGGESTER_QUERY_TEMPLATE"); - private static final String WHZ_ELASTICSEARCH_AUTO_COMPLETION_QUERY_FILE = System.getenv("WHZ_ELASTICSEARCH_AUTO_COMPLETION_QUERY_TEMPLATE"); + private static final String WHZ_ELASTICSEARCH_DATASET_QUERY_FILE = + System.getenv("WHZ_ELASTICSEARCH_DATASET_QUERY_TEMPLATE"); + private static final String WHZ_ELASTICSEARCH_METRIC_QUERY_FILE = + System.getenv("WHZ_ELASTICSEARCH_METRIC_QUERY_TEMPLATE"); + private static final String WHZ_ELASTICSEARCH_FLOW_QUERY_FILE = + System.getenv("WHZ_ELASTICSEARCH_FLOW_QUERY_TEMPLATE"); + private static final String WHZ_ELASTICSEARCH_COMMENT_QUERY_FILE = + System.getenv("WHZ_ELASTICSEARCH_COMMENT_QUERY_TEMPLATE"); + private static final String WHZ_ELASTICSEARCH_SUGGESTER_QUERY_FILE = + System.getenv("WHZ_ELASTICSEARCH_SUGGESTER_QUERY_TEMPLATE"); + private static final String WHZ_ELASTICSEARCH_AUTO_COMPLETION_QUERY_FILE = + System.getenv("WHZ_ELASTICSEARCH_AUTO_COMPLETION_QUERY_TEMPLATE"); private static final String WHZ_ELASTICSEARCH_FILTER_UNIT_FILE = System.getenv("WHZ_ELASTICSEARCH_FILTER_UNIT"); + private static final String WHZ_ELASTICSEARCH_FILTER_UNIT_FILE_FABRIC = + System.getenv("WHZ_ELASTICSEARCH_FILTER_UNIT_FABRIC"); public static String readJsonQueryFile(String jsonFile) { try { @@ -147,7 +155,7 @@ public class Search { return queryNode; } - public static ObjectNode generateElasticSearchFilterString(String sources) throws IOException { + public static ObjectNode generateESFilterStringForSources(String sources) throws IOException { if (StringUtils.isBlank(sources)) { return null; } @@ -171,4 +179,57 @@ public class Search { return queryNode; } + + public static ObjectNode generateESFilterStringForFabrics(String fabrics) throws IOException { + + if (StringUtils.isBlank(fabrics)) { + return null; + } + + List shouldValueList = new ArrayList(); + + String queryTemplate = readJsonQueryFile(WHZ_ELASTICSEARCH_FILTER_UNIT_FILE_FABRIC); + String[] values = fabrics.trim().split(","); + + for (String value : values) { + if (StringUtils.isNotBlank(value)) { + String query = queryTemplate.replace("$FABRIC", value.replace("\"", "").toLowerCase().trim()); + shouldValueList.add(_OM.readTree(query)); + } + } + + ObjectNode shouldNode = _OM.createObjectNode(); + shouldNode.putPOJO("should", shouldValueList); + ObjectNode queryNode = _OM.createObjectNode(); + queryNode.putPOJO("bool", shouldNode); + + return queryNode; + } + + public static ObjectNode generateElasticSearchFilterString(String sources, String fabrics) throws Exception { + + ObjectNode sourcesNode = generateESFilterStringForSources(sources); + ObjectNode fabricsNode = generateESFilterStringForFabrics(fabrics); + + if (sourcesNode == null && fabricsNode == null) { + return null; + } + + List mustValueList = new ArrayList(); + + if (sourcesNode != null) { + mustValueList.add(generateESFilterStringForSources(sources)); + } + + if (fabricsNode != null) { + mustValueList.add(generateESFilterStringForFabrics(fabrics)); + } + + ObjectNode mustNode = _OM.createObjectNode(); + mustNode.putPOJO("must", mustValueList); + ObjectNode queryNode = _OM.createObjectNode(); + queryNode.putPOJO("bool", mustNode); + + return queryNode; + } } \ No newline at end of file diff --git a/wherehows-data-model/ELASTICSEARCH/filter_unit_fabric.json b/wherehows-data-model/ELASTICSEARCH/filter_unit_fabric.json new file mode 100644 index 0000000000..32d1b8e698 --- /dev/null +++ b/wherehows-data-model/ELASTICSEARCH/filter_unit_fabric.json @@ -0,0 +1,5 @@ +{ + "match": { + "dataorigin": "$FABRIC" + } +} diff --git a/wherehows-frontend/app/controllers/api/v1/Search.java b/wherehows-frontend/app/controllers/api/v1/Search.java index 74ff8d4f0c..ca23339958 100644 --- a/wherehows-frontend/app/controllers/api/v1/Search.java +++ b/wherehows-frontend/app/controllers/api/v1/Search.java @@ -96,6 +96,8 @@ public class Search extends Controller { String category = request().getQueryString("category"); String source = request().getQueryString("source"); String pageStr = request().getQueryString("page"); + String fabric = request().getQueryString("fabric"); + if (isBlank(pageStr)) { page = 1; } else { @@ -124,12 +126,18 @@ public class Search extends Controller { if (isBlank(category)) { category = "datasets"; } + // Filter on platform if (isBlank(source) || source.equalsIgnoreCase("all") || source.equalsIgnoreCase("default")) { source = null; } + // Filter on fabric + if (isBlank(fabric) || fabric.equalsIgnoreCase("all") || fabric.equalsIgnoreCase("default")) { + fabric = null; + } + result.set("result", - SEARCH_DAO.elasticSearchDatasetByKeyword(ELASTICSEARCH_DATASET_URL, category, keyword, source, page, size)); + SEARCH_DAO.elasticSearchDatasetByKeyword(ELASTICSEARCH_DATASET_URL, category, keyword, source, page, size, fabric)); return ok(result); }