yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-11-13 09:52:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(browsepathv2): enable incremental update browsepath (#8354)

Browse Source
This commit is contained in:
david-leifker 2023-07-06 10:42:07 -05:00 committed by GitHub
parent 393b5a3039
commit a5f420ffd4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 128 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESBrowseDAO.java
View File

@ -437,7 +437,7 @@ public class ESBrowseDAO {
queryBuilder.filter(QueryBuilders.rangeQuery(BROWSE_PATH_V2_DEPTH).gt(browseDepthVal)); queryBuilder.filter(QueryBuilders.rangeQuery(BROWSE_PATH_V2_DEPTH).gt(browseDepthVal));
queryBuilder.must(SearchRequestHandler.getFilterQuery(filter)); queryBuilder.filter(SearchRequestHandler.getFilterQuery(filter));
return queryBuilder; return queryBuilder;
} }

4
metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandler.java
View File

@ -191,7 +191,7 @@ public class SearchRequestHandler {
BoolQueryBuilder filterQuery = getFilterQuery(filter); BoolQueryBuilder filterQuery = getFilterQuery(filter);
searchSourceBuilder.query(QueryBuilders.boolQuery() searchSourceBuilder.query(QueryBuilders.boolQuery()
.must(getQuery(input, finalSearchFlags.isFulltext())) .must(getQuery(input, finalSearchFlags.isFulltext()))
.must(filterQuery)); .filter(filterQuery));
if (!finalSearchFlags.isSkipAggregates()) { if (!finalSearchFlags.isSkipAggregates()) {
_aggregationQueryBuilder.getAggregations(facets).forEach(searchSourceBuilder::aggregation); _aggregationQueryBuilder.getAggregations(facets).forEach(searchSourceBuilder::aggregation);
} }
@ -231,7 +231,7 @@ public class SearchRequestHandler {
searchSourceBuilder.fetchSource("urn", null); searchSourceBuilder.fetchSource("urn", null);
BoolQueryBuilder filterQuery = getFilterQuery(filter); BoolQueryBuilder filterQuery = getFilterQuery(filter);
searchSourceBuilder.query(QueryBuilders.boolQuery().must(getQuery(input, finalSearchFlags.isFulltext())).must(filterQuery)); searchSourceBuilder.query(QueryBuilders.boolQuery().must(getQuery(input, finalSearchFlags.isFulltext())).filter(filterQuery));
_aggregationQueryBuilder.getAggregations().forEach(searchSourceBuilder::aggregation); _aggregationQueryBuilder.getAggregations().forEach(searchSourceBuilder::aggregation);
searchSourceBuilder.highlighter(getHighlights()); searchSourceBuilder.highlighter(getHighlights());
ESUtils.buildSortOrder(searchSourceBuilder, sortCriterion); ESUtils.buildSortOrder(searchSourceBuilder, sortCriterion);

21
metadata-io/src/main/java/com/linkedin/metadata/search/utils/ESUtils.java
View File

@ -109,10 +109,11 @@ public class ESUtils {
boolean isTimeseries) { boolean isTimeseries) {
final BoolQueryBuilder andQueryBuilder = new BoolQueryBuilder(); final BoolQueryBuilder andQueryBuilder = new BoolQueryBuilder();
conjunctiveCriterion.getAnd().forEach(criterion -> { conjunctiveCriterion.getAnd().forEach(criterion -> {
if (!criterion.getValue().trim().isEmpty() || criterion.hasValues() if (Set.of(Condition.EXISTS, Condition.IS_NULL).contains(criterion.getCondition())
|| criterion.getCondition() == Condition.IS_NULL) { || !criterion.getValue().trim().isEmpty() || criterion.hasValues()) {
if (!criterion.isNegated()) { if (!criterion.isNegated()) {
andQueryBuilder.must(getQueryBuilderFromCriterion(criterion, isTimeseries)); // `filter` instead of `must` (enables caching and bypasses scoring)
andQueryBuilder.filter(getQueryBuilderFromCriterion(criterion, isTimeseries));
} else { } else {
andQueryBuilder.mustNot(getQueryBuilderFromCriterion(criterion, isTimeseries)); andQueryBuilder.mustNot(getQueryBuilderFromCriterion(criterion, isTimeseries));
} }
@ -261,8 +262,12 @@ public class ESUtils {
Criterion criterionToQuery = new Criterion(); Criterion criterionToQuery = new Criterion();
criterionToQuery.setCondition(criterion.getCondition()); criterionToQuery.setCondition(criterion.getCondition());
criterionToQuery.setNegated(criterion.isNegated()); criterionToQuery.setNegated(criterion.isNegated());
criterionToQuery.setValue(criterion.getValue()); if (criterion.hasValues()) {
criterionToQuery.setValues(criterion.getValues()); criterionToQuery.setValues(criterion.getValues());
}
if (criterion.hasValue()) {
criterionToQuery.setValue(criterion.getValue());
}
criterionToQuery.setField(toKeywordField(field, isTimeseries)); criterionToQuery.setField(toKeywordField(field, isTimeseries));
orQueryBuilder.should(getQueryBuilderFromCriterionForSingleField(criterionToQuery, isTimeseries)); orQueryBuilder.should(getQueryBuilderFromCriterionForSingleField(criterionToQuery, isTimeseries));
} }
@ -274,12 +279,13 @@ public class ESUtils {
final Condition condition = criterion.getCondition(); final Condition condition = criterion.getCondition();
final String fieldName = toFacetField(criterion.getField()); final String fieldName = toFacetField(criterion.getField());
if (condition == Condition.EQUAL) { if (condition == Condition.IS_NULL) {
return buildEqualsConditionFromCriterion(fieldName, criterion, isTimeseries);
} else if (condition == Condition.IS_NULL) {
return QueryBuilders.boolQuery().mustNot(QueryBuilders.existsQuery(criterion.getField())).queryName(fieldName); return QueryBuilders.boolQuery().mustNot(QueryBuilders.existsQuery(criterion.getField())).queryName(fieldName);
} else if (condition == Condition.EXISTS) { } else if (condition == Condition.EXISTS) {
return QueryBuilders.boolQuery().must(QueryBuilders.existsQuery(criterion.getField())).queryName(fieldName); return QueryBuilders.boolQuery().must(QueryBuilders.existsQuery(criterion.getField())).queryName(fieldName);
} else if (criterion.hasValues() || criterion.hasValue()) {
if (condition == Condition.EQUAL) {
return buildEqualsConditionFromCriterion(fieldName, criterion, isTimeseries);
// TODO: Support multi-match on the following operators (using new 'values' field) // TODO: Support multi-match on the following operators (using new 'values' field)
} else if (condition == Condition.GREATER_THAN) { } else if (condition == Condition.GREATER_THAN) {
return QueryBuilders.rangeQuery(criterion.getField()).gt(criterion.getValue().trim()).queryName(fieldName); return QueryBuilders.rangeQuery(criterion.getField()).gt(criterion.getValue().trim()).queryName(fieldName);
@ -299,6 +305,7 @@ public class ESUtils {
return QueryBuilders.wildcardQuery(toKeywordField(criterion.getField(), isTimeseries), return QueryBuilders.wildcardQuery(toKeywordField(criterion.getField(), isTimeseries),
"*" + ESUtils.escapeReservedCharacters(criterion.getValue().trim())).queryName(fieldName); "*" + ESUtils.escapeReservedCharacters(criterion.getValue().trim())).queryName(fieldName);
} }
}
throw new UnsupportedOperationException("Unsupported condition: " + condition); throw new UnsupportedOperationException("Unsupported condition: " + condition);
} }

63
metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandlerTest.java
View File

@ -29,6 +29,7 @@ import com.linkedin.metadata.query.filter.CriterionArray;
import com.linkedin.metadata.query.filter.Filter; import com.linkedin.metadata.query.filter.Filter;
import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.ExistsQueryBuilder;
import org.elasticsearch.index.query.MatchQueryBuilder; import org.elasticsearch.index.query.MatchQueryBuilder;
import org.elasticsearch.index.query.MultiMatchQueryBuilder; import org.elasticsearch.index.query.MultiMatchQueryBuilder;
import org.elasticsearch.index.query.TermsQueryBuilder; import org.elasticsearch.index.query.TermsQueryBuilder;
@ -215,7 +216,7 @@ public class SearchRequestHandlerTest extends AbstractTestNGSpringContextTests {
} }
private void testFilterQuery(BoolQueryBuilder testQuery) { private void testFilterQuery(BoolQueryBuilder testQuery) {
Optional<MatchQueryBuilder> mustNotHaveRemovedCondition = testQuery.must() Optional<MatchQueryBuilder> mustNotHaveRemovedCondition = testQuery.filter()
.stream() .stream()
.filter(or -> or instanceof BoolQueryBuilder) .filter(or -> or instanceof BoolQueryBuilder)
.map(or -> (BoolQueryBuilder) or) .map(or -> (BoolQueryBuilder) or)
@ -292,13 +293,13 @@ public class SearchRequestHandlerTest extends AbstractTestNGSpringContextTests {
final BoolQueryBuilder testQuery = getQuery(filterCriterion); final BoolQueryBuilder testQuery = getQuery(filterCriterion);
// bool -> must -> [bool] -> should -> [bool] -> must -> [bool] -> should -> [terms] // bool -> filter -> [bool] -> should -> [bool] -> filter -> [bool] -> should -> [terms]
List<TermsQueryBuilder> termsQueryBuilders = testQuery.must() List<TermsQueryBuilder> termsQueryBuilders = testQuery.filter()
.stream() .stream()
.filter(or -> or instanceof BoolQueryBuilder) .filter(or -> or instanceof BoolQueryBuilder)
.flatMap(or -> ((BoolQueryBuilder) or).should().stream()) .flatMap(or -> ((BoolQueryBuilder) or).should().stream())
.filter(should -> should instanceof BoolQueryBuilder) .filter(should -> should instanceof BoolQueryBuilder)
.flatMap(should -> ((BoolQueryBuilder) should).must().stream()) .flatMap(should -> ((BoolQueryBuilder) should).filter().stream())
.filter(must -> must instanceof BoolQueryBuilder) .filter(must -> must instanceof BoolQueryBuilder)
.flatMap(must -> ((BoolQueryBuilder) must).should().stream()) .flatMap(must -> ((BoolQueryBuilder) must).should().stream())
.filter(should -> should instanceof TermsQueryBuilder) .filter(should -> should instanceof TermsQueryBuilder)
@ -342,12 +343,12 @@ public class SearchRequestHandlerTest extends AbstractTestNGSpringContextTests {
final BoolQueryBuilder testQuery = getQuery(filterCriterion); final BoolQueryBuilder testQuery = getQuery(filterCriterion);
// bool -> must -> [bool] -> should -> [bool] -> must -> [bool] -> should -> [bool] -> should -> [match] // bool -> must -> [bool] -> should -> [bool] -> must -> [bool] -> should -> [bool] -> should -> [match]
List<MultiMatchQueryBuilder> matchQueryBuilders = testQuery.must() List<MultiMatchQueryBuilder> matchQueryBuilders = testQuery.filter()
.stream() .stream()
.filter(or -> or instanceof BoolQueryBuilder) .filter(or -> or instanceof BoolQueryBuilder)
.flatMap(or -> ((BoolQueryBuilder) or).should().stream()) .flatMap(or -> ((BoolQueryBuilder) or).should().stream())
.filter(should -> should instanceof BoolQueryBuilder) .filter(should -> should instanceof BoolQueryBuilder)
.flatMap(should -> ((BoolQueryBuilder) should).must().stream()) .flatMap(should -> ((BoolQueryBuilder) should).filter().stream())
.filter(must -> must instanceof BoolQueryBuilder) .filter(must -> must instanceof BoolQueryBuilder)
.flatMap(must -> ((BoolQueryBuilder) must).should().stream()) .flatMap(must -> ((BoolQueryBuilder) must).should().stream())
.filter(should -> should instanceof BoolQueryBuilder) .filter(should -> should instanceof BoolQueryBuilder)
@ -381,13 +382,13 @@ public class SearchRequestHandlerTest extends AbstractTestNGSpringContextTests {
final BoolQueryBuilder testQuery = getQuery(filterCriterion); final BoolQueryBuilder testQuery = getQuery(filterCriterion);
// bool -> must -> [bool] -> should -> [bool] -> must -> [bool] -> should -> [match] // bool -> filter -> [bool] -> should -> [bool] -> filter -> [bool] -> should -> [match]
List<MultiMatchQueryBuilder> matchQueryBuilders = testQuery.must() List<MultiMatchQueryBuilder> matchQueryBuilders = testQuery.filter()
.stream() .stream()
.filter(or -> or instanceof BoolQueryBuilder) .filter(or -> or instanceof BoolQueryBuilder)
.flatMap(or -> ((BoolQueryBuilder) or).should().stream()) .flatMap(or -> ((BoolQueryBuilder) or).should().stream())
.filter(should -> should instanceof BoolQueryBuilder) .filter(should -> should instanceof BoolQueryBuilder)
.flatMap(should -> ((BoolQueryBuilder) should).must().stream()) .flatMap(should -> ((BoolQueryBuilder) should).filter().stream())
.filter(must -> must instanceof BoolQueryBuilder) .filter(must -> must instanceof BoolQueryBuilder)
.flatMap(must -> ((BoolQueryBuilder) must).should().stream()) .flatMap(must -> ((BoolQueryBuilder) must).should().stream())
.filter(should -> should instanceof MultiMatchQueryBuilder) .filter(should -> should instanceof MultiMatchQueryBuilder)
@ -414,13 +415,13 @@ public class SearchRequestHandlerTest extends AbstractTestNGSpringContextTests {
final BoolQueryBuilder testQuery = getQuery(filterCriterion); final BoolQueryBuilder testQuery = getQuery(filterCriterion);
// bool -> must -> [bool] -> should -> [bool] -> must -> [terms] // bool -> filter -> [bool] -> should -> [bool] -> filter -> [terms]
List<TermsQueryBuilder> termsQueryBuilders = testQuery.must() List<TermsQueryBuilder> termsQueryBuilders = testQuery.filter()
.stream() .stream()
.filter(must -> must instanceof BoolQueryBuilder) .filter(must -> must instanceof BoolQueryBuilder)
.flatMap(must -> ((BoolQueryBuilder) must).should().stream()) .flatMap(must -> ((BoolQueryBuilder) must).should().stream())
.filter(should -> should instanceof BoolQueryBuilder) .filter(should -> should instanceof BoolQueryBuilder)
.flatMap(should -> ((BoolQueryBuilder) should).must().stream()) .flatMap(should -> ((BoolQueryBuilder) should).filter().stream())
.filter(must -> must instanceof TermsQueryBuilder) .filter(must -> must instanceof TermsQueryBuilder)
.map(must -> (TermsQueryBuilder) must) .map(must -> (TermsQueryBuilder) must)
.collect(Collectors.toList()); .collect(Collectors.toList());
@ -439,6 +440,44 @@ public class SearchRequestHandlerTest extends AbstractTestNGSpringContextTests {
assertTrue(values.contains("bigquery")); assertTrue(values.contains("bigquery"));
} }
@Test
public void testBrowsePathQueryFilter() {
// Condition: has `browsePaths` AND does NOT have `browsePathV2`
Criterion missingBrowsePathV2 = new Criterion();
missingBrowsePathV2.setCondition(Condition.IS_NULL);
missingBrowsePathV2.setField("browsePathV2");
// Excludes entities without browsePaths
Criterion hasBrowsePathV1 = new Criterion();
hasBrowsePathV1.setCondition(Condition.EXISTS);
hasBrowsePathV1.setField("browsePaths");
CriterionArray criterionArray = new CriterionArray();
criterionArray.add(missingBrowsePathV2);
criterionArray.add(hasBrowsePathV1);
ConjunctiveCriterion conjunctiveCriterion = new ConjunctiveCriterion();
conjunctiveCriterion.setAnd(criterionArray);
ConjunctiveCriterionArray conjunctiveCriterionArray = new ConjunctiveCriterionArray();
conjunctiveCriterionArray.add(conjunctiveCriterion);
Filter filter = new Filter();
filter.setOr(conjunctiveCriterionArray);
BoolQueryBuilder test = SearchRequestHandler.getFilterQuery(filter);
assertEquals(test.should().size(), 1);
BoolQueryBuilder shouldQuery = (BoolQueryBuilder) test.should().get(0);
assertEquals(shouldQuery.filter().size(), 2);
BoolQueryBuilder mustNotHaveV2 = (BoolQueryBuilder) shouldQuery.filter().get(0);
assertEquals(((ExistsQueryBuilder) mustNotHaveV2.mustNot().get(0)).fieldName(), "browsePathV2");
BoolQueryBuilder mustHaveV1 = (BoolQueryBuilder) shouldQuery.filter().get(1);
assertEquals(((ExistsQueryBuilder) mustHaveV1.must().get(0)).fieldName(), "browsePaths");
}
private BoolQueryBuilder getQuery(final Criterion filterCriterion) { private BoolQueryBuilder getQuery(final Criterion filterCriterion) {
final Filter filter = new Filter().setOr( final Filter filter = new Filter().setOr(
new ConjunctiveCriterionArray( new ConjunctiveCriterionArray(

30
metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/BackfillBrowsePathsV2Step.java
View File

@ -9,6 +9,12 @@ import com.linkedin.events.metadata.ChangeType;
import com.linkedin.metadata.Constants; import com.linkedin.metadata.Constants;
import com.linkedin.metadata.boot.UpgradeStep; import com.linkedin.metadata.boot.UpgradeStep;
import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.entity.EntityService;
import com.linkedin.metadata.query.filter.Condition;
import com.linkedin.metadata.query.filter.ConjunctiveCriterion;
import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray;
import com.linkedin.metadata.query.filter.Criterion;
import com.linkedin.metadata.query.filter.CriterionArray;
import com.linkedin.metadata.query.filter.Filter;
import com.linkedin.metadata.search.ScrollResult; import com.linkedin.metadata.search.ScrollResult;
import com.linkedin.metadata.search.SearchEntity; import com.linkedin.metadata.search.SearchEntity;
import com.linkedin.metadata.search.SearchService; import com.linkedin.metadata.search.SearchService;
@ -71,10 +77,32 @@ public class BackfillBrowsePathsV2Step extends UpgradeStep {
private String backfillBrowsePathsV2(String entityType, AuditStamp auditStamp, String scrollId) private String backfillBrowsePathsV2(String entityType, AuditStamp auditStamp, String scrollId)
throws Exception { throws Exception {
// Condition: has `browsePaths` AND does NOT have `browsePathV2`
Criterion missingBrowsePathV2 = new Criterion();
missingBrowsePathV2.setCondition(Condition.IS_NULL);
missingBrowsePathV2.setField("browsePathV2");
// Excludes entities without browsePaths
Criterion hasBrowsePathV1 = new Criterion();
hasBrowsePathV1.setCondition(Condition.EXISTS);
hasBrowsePathV1.setField("browsePaths");
CriterionArray criterionArray = new CriterionArray();
criterionArray.add(missingBrowsePathV2);
criterionArray.add(hasBrowsePathV1);
ConjunctiveCriterion conjunctiveCriterion = new ConjunctiveCriterion();
conjunctiveCriterion.setAnd(criterionArray);
ConjunctiveCriterionArray conjunctiveCriterionArray = new ConjunctiveCriterionArray();
conjunctiveCriterionArray.add(conjunctiveCriterion);
Filter filter = new Filter();
filter.setOr(conjunctiveCriterionArray);
final ScrollResult scrollResult = _searchService.scrollAcrossEntities( final ScrollResult scrollResult = _searchService.scrollAcrossEntities(
ImmutableList.of(entityType), ImmutableList.of(entityType),
"*", "*",
null, filter,
null, null,
scrollId, scrollId,
"5m", "5m",

5
metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/BackfillBrowsePathsV2StepTest.java
View File

@ -14,6 +14,7 @@ import com.linkedin.entity.EnvelopedAspectMap;
import com.linkedin.metadata.Constants; import com.linkedin.metadata.Constants;
import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.entity.EntityService;
import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.models.registry.EntityRegistry;
import com.linkedin.metadata.query.filter.Filter;
import com.linkedin.metadata.search.ScrollResult; import com.linkedin.metadata.search.ScrollResult;
import com.linkedin.metadata.search.SearchEntity; import com.linkedin.metadata.search.SearchEntity;
import com.linkedin.metadata.search.SearchEntityArray; import com.linkedin.metadata.search.SearchEntityArray;
@ -88,7 +89,7 @@ public class BackfillBrowsePathsV2StepTest {
Mockito.verify(mockSearchService, Mockito.times(9)).scrollAcrossEntities( Mockito.verify(mockSearchService, Mockito.times(9)).scrollAcrossEntities(
Mockito.any(), Mockito.any(),
Mockito.eq("*"), Mockito.eq("*"),
Mockito.eq(null), Mockito.any(Filter.class),
Mockito.eq(null), Mockito.eq(null),
Mockito.eq(null), Mockito.eq(null),
Mockito.eq("5m"), Mockito.eq("5m"),
@ -157,7 +158,7 @@ public class BackfillBrowsePathsV2StepTest {
Mockito.when(mockSearchService.scrollAcrossEntities( Mockito.when(mockSearchService.scrollAcrossEntities(
Mockito.eq(ImmutableList.of(ENTITY_TYPES.get(i))), Mockito.eq(ImmutableList.of(ENTITY_TYPES.get(i))),
Mockito.eq("*"), Mockito.eq("*"),
Mockito.eq(null), Mockito.any(Filter.class),
Mockito.eq(null), Mockito.eq(null),
Mockito.eq(null), Mockito.eq(null),
Mockito.eq("5m"), Mockito.eq("5m"),

2
smoke-test/tests/cypress/cypress/e2e/mutations/managed_ingestion.js
View File

@ -7,7 +7,7 @@ describe("run managed ingestion", () => {
it("create run managed ingestion source", () => { it("create run managed ingestion source", () => {
let number = Math.floor(Math.random() * 100000); let number = Math.floor(Math.random() * 100000);
let testName = `cypress test source ${number}` let testName = `cypress test source ${number}`
let cli_version = "0.9.3.3rc5"; let cli_version = "0.10.4.3";
cy.login(); cy.login();
cy.goToIngestionPage(); cy.goToIngestionPage();
cy.clickOptionWithText("Create new source"); cy.clickOptionWithText("Create new source");