yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-12-26 09:26:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(search): missing model updates and tests (#7617)

Browse Source 
Co-authored-by: John Joyce <john@acryl.io>
This commit is contained in:
david-leifker 2023-03-21 12:18:51 -05:00 committed by GitHub
parent 9ec86a5cc0
commit faf0ac1572
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 44 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/SettingsBuilder.java
View File

@ -84,6 +84,8 @@ public class SettingsBuilder {
public static final String WORD_DELIMITER = "word_delimiter";
public static final String WORD_DELIMITER_GRAPH = "word_delimiter_graph";
public static final String TRIM = "trim";
// MultiFilters
public static final String MULTIFILTER_GRAPH_1 = String.join(",", LOWERCASE, STICKY_DELIMITER_GRAPH);
public static final String MULTIFILTER_GRAPH_2 = String.join(",", LOWERCASE, ALPHANUM_SPACE_ONLY,
@ -117,6 +119,7 @@ public class SettingsBuilder {
public static final List<String> INDEX_TOKEN_FILTERS = ImmutableList.of(
ASCII_FOLDING,
MULTIFILTER,
TRIM,
LOWERCASE,
DATAHUB_STOP_WORDS,
STOP,
@ -129,6 +132,7 @@ public class SettingsBuilder {
public static final List<String> SEARCH_TOKEN_FILTERS = ImmutableList.of(
ASCII_FOLDING,
MULTIFILTER_GRAPH,
TRIM,
LOWERCASE,
DATAHUB_STOP_WORDS,
STOP,

31
metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/fixtures/SampleDataFixtureTests.java
View File

@ -175,6 +175,17 @@ public class SampleDataFixtureTests extends AbstractTestNGSpringContextTests {
}
}
@Test
public void testDatasetHasTags() throws IOException {
GetMappingsRequest req = new GetMappingsRequest()
.indices("smpldat_datasetindex_v2");
GetMappingsResponse resp = _searchClient.indices().getMapping(req, RequestOptions.DEFAULT);
Map<String, Map<String, String>> mappings = (Map<String, Map<String, String>>) resp.mappings()
.get("smpldat_datasetindex_v2").sourceAsMap().get("properties");
assertTrue(mappings.containsKey("hasTags"));
assertEquals(mappings.get("hasTags"), Map.of("type", "boolean"));
}
@Test
public void testFixtureInitialization() {
assertNotNull(searchService);
@ -184,7 +195,7 @@ public class SampleDataFixtureTests extends AbstractTestNGSpringContextTests {
final SearchResult result = search(searchService, "test");
Map<String, Integer> expectedTypes = Map.of(
"dataset", 7,
"dataset", 10,
"chart", 0,
"container", 1,
"dashboard", 0,
@ -1121,6 +1132,24 @@ public class SampleDataFixtureTests extends AbstractTestNGSpringContextTests {
"Expected exact match and 1st position");
}
@Test
public void testPrefixVsExactCaseSensitivity() {
List<String> insensitiveExactMatches = List.of("testExactMatchCase", "testexactmatchcase", "TESTEXACTMATCHCASE");
for (String query : insensitiveExactMatches) {
SearchResult result = search(searchService, query);
assertTrue(result.hasEntities() && !result.getEntities().isEmpty(),
String.format("%s - Expected search results", query));
assertTrue(result.getEntities().stream().noneMatch(e -> e.getMatchedFields().isEmpty()),
String.format("%s - Expected search results to include matched fields", query));
assertEquals(result.getEntities().size(), insensitiveExactMatches.size());
assertEquals(result.getEntities().get(0).getEntity().toString(),
"urn:li:dataset:(urn:li:dataPlatform:testOnly," + query + ",PROD)",
"Expected exact match as first match with matching case");
}
}
private Stream<AnalyzeResponse.AnalyzeToken> getTokens(AnalyzeRequest request) throws IOException {
return _searchClient.indices().analyze(request, RequestOptions.DEFAULT).getTokens().stream();
}

BIN
metadata-io/src/test/resources/elasticsearch/sample_data/datasetindex_v2.json.gz
View File

Binary file not shown.

12
metadata-models/src/main/pegasus/com/linkedin/common/GlobalTags.pdl
View File

@ -11,13 +11,21 @@ record GlobalTags {
/**
* Tags associated with a given entity
*/
@Searchable = {
@Relationship = {
"/*/tag": {
"name": "TaggedWith",
"entityTypes": [ "tag" ]
}
}
@Searchable = {
"/*/tag": {
"fieldName": "tags",
"fieldType": "URN",
"boostScore": 0.5,
"queryByDefault": true,
"addToFilters": true
"addToFilters": true,
"hasValuesFieldName": "hasTags",
"filterNameOverride": "Tag"
}
}
tags: array[TagAssociation]

11
metadata-models/src/main/pegasus/com/linkedin/common/TagAssociation.pdl
View File

@ -8,17 +8,6 @@ record TagAssociation {
/**
* Urn of the applied tag
*/
@Relationship = {
"name": "TaggedWith",
"entityTypes": [ "tag" ]
}
@Searchable = {
"fieldName": "tags",
"fieldType": "URN",
"hasValuesFieldName": "hasTags",
"addToFilters": true,
"filterNameOverride": "Tag"
}
tag: TagUrn
/**