yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-12-27 09:58:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(search): Fix urn component settings (#4672)

Browse Source 
* Fix urn component settings

* Add keyword subfield to all fields

* Fix tests
This commit is contained in:
Dexter Lee 2022-04-14 15:40:03 -07:00 committed by GitHub
parent d468208fc8
commit 46c15ef0a7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 31 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/MappingsBuilder.java
View File

@ -40,10 +40,8 @@ public class MappingsBuilder {
if (fieldType == FieldType.KEYWORD) {
mappingForField.put("type", "keyword");
mappingForField.put("normalizer", "keyword_normalizer");
if (addToFilters) {
// Add keyword subfield without lowercase filter
mappingForField.put("fields", ImmutableMap.of("keyword", ImmutableMap.of("type", "keyword")));
}
// Add keyword subfield without lowercase filter
mappingForField.put("fields", ImmutableMap.of("keyword", ImmutableMap.of("type", "keyword")));
} else if (fieldType == FieldType.TEXT || fieldType == FieldType.TEXT_PARTIAL) {
mappingForField.put("type", "keyword");
mappingForField.put("normalizer", "keyword_normalizer");
@ -52,10 +50,8 @@ public class MappingsBuilder {
subFields.put("ngram", ImmutableMap.of("type", "text", "analyzer", "partial"));
}
subFields.put("delimited", ImmutableMap.of("type", "text", "analyzer", "word_delimited"));
if (addToFilters) {
// Add keyword subfield without lowercase filter
subFields.put("keyword", ImmutableMap.of("type", "keyword"));
}
// Add keyword subfield without lowercase filter
subFields.put("keyword", ImmutableMap.of("type", "keyword"));
mappingForField.put("fields", subFields);
} else if (fieldType == FieldType.BROWSE_PATH) {
mappingForField.put("type", "text");
@ -70,12 +66,8 @@ public class MappingsBuilder {
if (fieldType == FieldType.URN_PARTIAL) {
subFields.put("ngram", ImmutableMap.of("type", "text", "analyzer", "partial_urn_component"));
}
if (addToFilters) {
subFields.put("keyword", ImmutableMap.of("type", "keyword"));
}
if (!subFields.isEmpty()) {
mappingForField.put("fields", subFields);
}
subFields.put("keyword", ImmutableMap.of("type", "keyword"));
mappingForField.put("fields", subFields);
} else if (fieldType == FieldType.BOOLEAN) {
mappingForField.put("type", "boolean");
} else if (fieldType == FieldType.COUNT) {

2
metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/SettingsBuilder.java
View File

@ -65,7 +65,7 @@ public class SettingsBuilder {
// Tokenize for urns
tokenizers.put("urn_char_group",
ImmutableMap.<String, Object>builder().put("type", "pattern").put("pattern", "[:\\s(), ./]").build());
ImmutableMap.<String, Object>builder().put("type", "pattern").put("pattern", "[:\\s(),]").build());
return tokenizers.build();
}

6
metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/AutocompleteRequestHandler.java
View File

@ -121,11 +121,7 @@ public class AutocompleteRequestHandler {
} catch (URISyntaxException e) {
throw new RuntimeException(String.format("Failed to create urn %s", matchedUrn.get()), e);
}
if (matchedFieldValue.isPresent()) {
results.add(matchedFieldValue.get());
} else {
log.info("No highlighted field for query {}, hit {}", input, hit);
}
matchedFieldValue.ifPresent(results::add);
}
return new AutoCompleteResult()
.setQuery(input)

33
metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/MappingsBuilderTest.java
View File

@ -6,7 +6,6 @@ import java.util.Map;
import org.testng.annotations.Test;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertFalse;
import static org.testng.Assert.assertTrue;
@ -21,22 +20,33 @@ public class MappingsBuilderTest {
assertEquals(properties.get("urn"), ImmutableMap.of("type", "keyword"));
assertTrue(properties.containsKey("browsePaths"));
// KEYWORD
assertEquals(properties.get("keyPart3"), ImmutableMap.of("type", "keyword", "normalizer", "keyword_normalizer"));
assertEquals(properties.get("customProperties"),
ImmutableMap.of("type", "keyword", "normalizer", "keyword_normalizer"));
Map<String, Object> keyPart3Field = (Map<String, Object>) properties.get("keyPart3");
assertEquals(keyPart3Field.get("type"), "keyword");
assertEquals(keyPart3Field.get("normalizer"), "keyword_normalizer");
Map<String, Object> keyPart3FieldSubfields = (Map<String, Object>) keyPart3Field.get("fields");
assertEquals(keyPart3FieldSubfields.size(), 1);
assertTrue(keyPart3FieldSubfields.containsKey("keyword"));
Map<String, Object> customPropertiesField = (Map<String, Object>) properties.get("customProperties");
assertEquals(customPropertiesField.get("type"), "keyword");
assertEquals(customPropertiesField.get("normalizer"), "keyword_normalizer");
Map<String, Object> customPropertiesFieldSubfields = (Map<String, Object>) customPropertiesField.get("fields");
assertEquals(customPropertiesFieldSubfields.size(), 1);
assertTrue(customPropertiesFieldSubfields.containsKey("keyword"));
// TEXT
Map<String, Object> nestedArrayStringField = (Map<String, Object>) properties.get("nestedArrayStringField");
assertEquals(nestedArrayStringField.get("type"), "keyword");
assertEquals(nestedArrayStringField.get("normalizer"), "keyword_normalizer");
Map<String, Object> nestedArrayStringFieldSubfields = (Map<String, Object>) nestedArrayStringField.get("fields");
assertEquals(nestedArrayStringFieldSubfields.size(), 1);
assertEquals(nestedArrayStringFieldSubfields.size(), 2);
assertTrue(nestedArrayStringFieldSubfields.containsKey("delimited"));
assertTrue(nestedArrayStringFieldSubfields.containsKey("keyword"));
Map<String, Object> nestedArrayArrayField = (Map<String, Object>) properties.get("nestedArrayArrayField");
assertEquals(nestedArrayArrayField.get("type"), "keyword");
assertEquals(nestedArrayArrayField.get("normalizer"), "keyword_normalizer");
Map<String, Object> nestedArrayArrayFieldSubfields = (Map<String, Object>) nestedArrayArrayField.get("fields");
assertEquals(nestedArrayArrayFieldSubfields.size(), 1);
assertEquals(nestedArrayArrayFieldSubfields.size(), 2);
assertTrue(nestedArrayArrayFieldSubfields.containsKey("delimited"));
assertTrue(nestedArrayArrayFieldSubfields.containsKey("keyword"));
// TEXT with addToFilters
Map<String, Object> textField = (Map<String, Object>) properties.get("textFieldOverride");
@ -52,22 +62,26 @@ public class MappingsBuilderTest {
assertEquals(textArrayField.get("type"), "keyword");
assertEquals(textArrayField.get("normalizer"), "keyword_normalizer");
Map<String, Object> textArrayFieldSubfields = (Map<String, Object>) textArrayField.get("fields");
assertEquals(textArrayFieldSubfields.size(), 2);
assertEquals(textArrayFieldSubfields.size(), 3);
assertTrue(textArrayFieldSubfields.containsKey("delimited"));
assertTrue(textArrayFieldSubfields.containsKey("ngram"));
assertTrue(textArrayFieldSubfields.containsKey("keyword"));
// URN
Map<String, Object> foreignKey = (Map<String, Object>) properties.get("foreignKey");
assertEquals(foreignKey.get("type"), "text");
assertEquals(foreignKey.get("analyzer"), "urn_component");
assertFalse(foreignKey.containsKey("fields"));
Map<String, Object> foreignKeySubfields = (Map<String, Object>) foreignKey.get("fields");
assertEquals(foreignKeySubfields.size(), 1);
assertTrue(foreignKeySubfields.containsKey("keyword"));
// URN_PARTIAL
Map<String, Object> nestedForeignKey = (Map<String, Object>) properties.get("nestedForeignKey");
assertEquals(nestedForeignKey.get("type"), "text");
assertEquals(nestedForeignKey.get("analyzer"), "urn_component");
Map<String, Object> nestedForeignKeySubfields = (Map<String, Object>) nestedForeignKey.get("fields");
assertEquals(nestedForeignKeySubfields.size(), 1);
assertEquals(nestedForeignKeySubfields.size(), 2);
assertTrue(nestedForeignKeySubfields.containsKey("keyword"));
assertTrue(nestedForeignKeySubfields.containsKey("ngram"));
// Scores
@ -75,6 +89,5 @@ public class MappingsBuilderTest {
assertEquals(feature1.get("type"), "double");
Map<String, Object> feature2 = (Map<String, Object>) properties.get("feature2");
assertEquals(feature2.get("type"), "double");
}
}