yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-07-23 09:22:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix #14038: Inconsistent search results between auto-suggest box and … (#14747)

Browse Source 
* Fix #14038: Inconsistent search results between auto-suggest box and actual search results

* use dataAsset instead of all alias

---------

Co-authored-by: karanh37 <karanh37@gmail.com>
This commit is contained in:
Sriharsha Chintalapani 2024-01-17 21:37:36 -08:00 committed by GitHub
parent 19db3c060c
commit bfa8827d38
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
13 changed files with 259 additions and 112 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java
View File

@ -330,6 +330,8 @@ public class ElasticSearchClient implements SearchClient {
"search_service_index",
"metadata_service_index" -> buildServiceSearchBuilder(
request.getQuery(), request.getFrom(), request.getSize());
case "all", "dataAsset" -> buildSearchAcrossIndexesBuilder(
request.getQuery(), request.getFrom(), request.getSize());
default -> buildAggregateSearchBuilder(
request.getQuery(), request.getFrom(), request.getSize());
};
@ -717,11 +719,34 @@ public class ElasticSearchClient implements SearchClient {
return addAggregation(searchSourceBuilder);
}
private static SearchSourceBuilder buildSearchAcrossIndexesBuilder(
String query, int from, int size) {
QueryStringQueryBuilder queryStringBuilder =
QueryBuilders.queryStringQuery(query)
.fields(SearchIndex.getAllFields())
.type(MultiMatchQueryBuilder.Type.MOST_FIELDS)
.fuzziness(Fuzziness.AUTO);
FieldValueFactorFunctionBuilder boostScoreBuilder =
ScoreFunctionBuilders.fieldValueFactorFunction("usageSummary.weeklyStats.count")
.missing(0)
.factor(0.2f);
FunctionScoreQueryBuilder.FilterFunctionBuilder[] functions =
new FunctionScoreQueryBuilder.FilterFunctionBuilder[] {
new FunctionScoreQueryBuilder.FilterFunctionBuilder(boostScoreBuilder)
};
FunctionScoreQueryBuilder queryBuilder =
QueryBuilders.functionScoreQuery(queryStringBuilder, functions);
queryBuilder.boostMode(CombineFunction.SUM);
SearchSourceBuilder searchSourceBuilder = searchBuilder(queryBuilder, null, from, size);
return addAggregation(searchSourceBuilder);
}
private static SearchSourceBuilder buildTableSearchBuilder(String query, int from, int size) {
QueryStringQueryBuilder queryStringBuilder =
QueryBuilders.queryStringQuery(query)
.fields(TableIndex.getFields())
.type(MultiMatchQueryBuilder.Type.MOST_FIELDS)
.fuzziness(Fuzziness.AUTO)
.tieBreaker(0.9f);
FieldValueFactorFunctionBuilder boostScoreBuilder =

24
openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java
View File

@ -322,6 +322,8 @@ public class OpenSearchClient implements SearchClient {
"search_service_index",
"metadata_service_index" -> buildServiceSearchBuilder(
request.getQuery(), request.getFrom(), request.getSize());
case "all", "dataAsset" -> buildSearchAcrossIndexesBuilder(
request.getQuery(), request.getFrom(), request.getSize());
default -> buildAggregateSearchBuilder(
request.getQuery(), request.getFrom(), request.getSize());
};
@ -731,6 +733,28 @@ public class OpenSearchClient implements SearchClient {
return addAggregation(searchSourceBuilder);
}
private static SearchSourceBuilder buildSearchAcrossIndexesBuilder(
String query, int from, int size) {
QueryStringQueryBuilder queryStringBuilder =
QueryBuilders.queryStringQuery(query)
.fields(SearchIndex.getAllFields())
.type(MultiMatchQueryBuilder.Type.MOST_FIELDS)
.fuzziness(Fuzziness.AUTO);
FieldValueFactorFunctionBuilder boostScoreBuilder =
ScoreFunctionBuilders.fieldValueFactorFunction("usageSummary.weeklyStats.count")
.missing(0)
.factor(0.2f);
FunctionScoreQueryBuilder.FilterFunctionBuilder[] functions =
new FunctionScoreQueryBuilder.FilterFunctionBuilder[] {
new FunctionScoreQueryBuilder.FilterFunctionBuilder(boostScoreBuilder)
};
FunctionScoreQueryBuilder queryBuilder =
QueryBuilders.functionScoreQuery(queryStringBuilder, functions);
queryBuilder.boostMode(CombineFunction.SUM);
SearchSourceBuilder searchSourceBuilder = searchBuilder(queryBuilder, null, from, size);
return addAggregation(searchSourceBuilder);
}
private static SearchSourceBuilder buildTableSearchBuilder(String query, int from, int size) {
QueryStringQueryBuilder queryStringBuilder =
QueryBuilders.queryStringQuery(query)

4
openmetadata-service/src/main/resources/elasticsearch/indexMapping.json
View File

@ -111,13 +111,13 @@
"indexName": "glossary_term_search_index",
"indexMappingFile": "/elasticsearch/%s/glossary_term_index_mapping.json",
"alias": "glossaryTerm",
"parentAliases": ["all", "glossary"]
"parentAliases": ["all", "glossary", "dataAsset"]
},
"tag": {
"indexName": "tag_search_index",
"indexMappingFile": "/elasticsearch/%s/tag_index_mapping.json",
"alias": "tag",
"parentAliases": ["classification","all"]
"parentAliases": ["classification","all", "dataAsset"]
},
"classification": {
"indexName": "classification_search_index",

56
openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/Explore.spec.js Normal file
View File

@ -0,0 +1,56 @@
/*
* Copyright 2024 Collate.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { interceptURL, verifyResponseStatusCode } from '../../common/common';
describe('Explore Page', () => {
before(() => {
cy.login();
});
it('should verify order of search results', () => {
const searchText = 'customer';
interceptURL(
'GET',
`api/v1/search/query?q=**&index=dataAsset**`,
'suggestAPI'
);
interceptURL(
'GET',
`api/v1/search/query?q=**&index=table_search_index**`,
'searchAPI'
);
cy.get('[data-testid="app-bar-item-explore"]').click();
cy.get('[data-testid="searchBox"]').clear();
cy.get('[data-testid="searchBox"]').type(searchText);
verifyResponseStatusCode('@suggestAPI', 200);
const linksArray = [];
cy.get('[data-testid="group-table_search_index"] button a').each(
($link) => {
linksArray.push($link.attr('href'));
}
);
cy.get('[data-testid="searchBox"]').type('{enter}');
verifyResponseStatusCode('@searchAPI', 200);
cy.wrap(linksArray).each((link, index) => {
cy.get(`[data-testid="entity-link"]`)
.eq(index)
.should('have.attr', 'href', link);
});
});
});

6
openmetadata-ui/src/main/resources/ui/src/components/AppBar/Suggestions.tsx
View File

@ -136,14 +136,14 @@ const Suggestions = ({
}
return (
<>
<div data-testid={`group-${searchIndex}`}>
{getGroupLabel(searchIndex)}
{suggestions.map((suggestion: SearchSuggestions[number]) => {
return getSuggestionElement(suggestion, searchIndex, () =>
setIsOpen(false)
);
})}
</>
</div>
);
};
@ -205,7 +205,7 @@ const Suggestions = ({
'',
'',
'',
searchCriteria ?? SearchIndex.ALL
searchCriteria ?? SearchIndex.DATA_ASSET
);
if (res.data) {

28
openmetadata-ui/src/main/resources/ui/src/constants/explore.constants.ts
View File

@ -12,8 +12,6 @@
*/
import { SortingField } from '../components/Explore/SortingDropDown';
import { EntityType } from '../enums/entity.enum';
import { SearchIndex } from '../enums/search.enum';
import i18n from '../utils/i18next/LocalUtil';
export const INITIAL_SORT_FIELD = 'updatedAt';
@ -59,29 +57,3 @@ export const COMMON_FILTERS_FOR_DIFFERENT_TABS = [
'owner.displayName',
'tags.tagFQN',
];
export const EntityTypeSearchIndexMapping: Record<string, SearchIndex> = {
[EntityType.ALL]: SearchIndex.ALL,
[EntityType.TABLE]: SearchIndex.TABLE,
[EntityType.PIPELINE]: SearchIndex.PIPELINE,
[EntityType.DASHBOARD]: SearchIndex.DASHBOARD,
[EntityType.MLMODEL]: SearchIndex.MLMODEL,
[EntityType.TOPIC]: SearchIndex.TOPIC,
[EntityType.CONTAINER]: SearchIndex.CONTAINER,
[EntityType.TAG]: SearchIndex.TAG,
[EntityType.GLOSSARY_TERM]: SearchIndex.GLOSSARY,
[EntityType.STORED_PROCEDURE]: SearchIndex.STORED_PROCEDURE,
[EntityType.DASHBOARD_DATA_MODEL]: SearchIndex.DASHBOARD_DATA_MODEL,
[EntityType.SEARCH_INDEX]: SearchIndex.SEARCH_INDEX,
[EntityType.DATABASE_SERVICE]: SearchIndex.DATABASE_SERVICE,
[EntityType.MESSAGING_SERVICE]: SearchIndex.MESSAGING_SERVICE,
[EntityType.DASHBOARD_SERVICE]: SearchIndex.DASHBOARD_SERVICE,
[EntityType.PIPELINE_SERVICE]: SearchIndex.PIPELINE_SERVICE,
[EntityType.MLMODEL_SERVICE]: SearchIndex.ML_MODEL_SERVICE,
[EntityType.STORAGE_SERVICE]: SearchIndex.STORAGE_SERVICE,
[EntityType.SEARCH_SERVICE]: SearchIndex.SEARCH_SERVICE,
[EntityType.DOMAIN]: SearchIndex.DOMAIN,
[EntityType.DATA_PRODUCT]: SearchIndex.DATA_PRODUCT,
[EntityType.DATABASE]: SearchIndex.DATABASE,
[EntityType.DATABASE_SCHEMA]: SearchIndex.DATABASE_SCHEMA,
};

3
openmetadata-ui/src/main/resources/ui/src/pages/ExplorePage/ExplorePageV1.component.tsx
View File

@ -36,7 +36,6 @@ import { useTourProvider } from '../../components/TourProvider/TourProvider';
import { getExplorePath, PAGE_SIZE } from '../../constants/constants';
import {
COMMON_FILTERS_FOR_DIFFERENT_TABS,
EntityTypeSearchIndexMapping,
INITIAL_SORT_FIELD,
} from '../../constants/explore.constants';
import {
@ -63,6 +62,8 @@ import {
const ExplorePageV1: FunctionComponent = () => {
const tabsInfo = searchClassBase.getTabsInfo();
const EntityTypeSearchIndexMapping =
searchClassBase.getEntityTypeSearchIndexMapping();
const location = useLocation();
const history = useHistory();
const { isTourOpen } = useTourProvider();

73
openmetadata-ui/src/main/resources/ui/src/utils/Explore.utils.test.ts
View File

@ -11,12 +11,7 @@
* limitations under the License.
*/
import { ExploreQuickFilterField } from '../components/Explore/ExplorePage.interface';
import { EntityType } from '../enums/entity.enum';
import { SearchIndex } from '../enums/search.enum';
import {
getQuickFilterQuery,
getSearchIndexFromEntityType,
} from './Explore.utils';
import { getQuickFilterQuery } from './Explore.utils';
describe('Explore Utils', () => {
it('should return undefined if data is empty', () => {
@ -73,70 +68,4 @@ describe('Explore Utils', () => {
expect(result).toEqual(expectedQuery);
});
it('getSearchIndexFromEntityType should return the correct search index for each entity type', () => {
expect(getSearchIndexFromEntityType(EntityType.ALL)).toEqual(
SearchIndex.ALL
);
expect(getSearchIndexFromEntityType(EntityType.TABLE)).toEqual(
SearchIndex.TABLE
);
expect(getSearchIndexFromEntityType(EntityType.PIPELINE)).toEqual(
SearchIndex.PIPELINE
);
expect(getSearchIndexFromEntityType(EntityType.DASHBOARD)).toEqual(
SearchIndex.DASHBOARD
);
expect(getSearchIndexFromEntityType(EntityType.MLMODEL)).toEqual(
SearchIndex.MLMODEL
);
expect(getSearchIndexFromEntityType(EntityType.TOPIC)).toEqual(
SearchIndex.TOPIC
);
expect(getSearchIndexFromEntityType(EntityType.CONTAINER)).toEqual(
SearchIndex.CONTAINER
);
expect(getSearchIndexFromEntityType(EntityType.TAG)).toEqual(
SearchIndex.TAG
);
expect(getSearchIndexFromEntityType(EntityType.GLOSSARY_TERM)).toEqual(
SearchIndex.GLOSSARY
);
expect(getSearchIndexFromEntityType(EntityType.STORED_PROCEDURE)).toEqual(
SearchIndex.STORED_PROCEDURE
);
expect(
getSearchIndexFromEntityType(EntityType.DASHBOARD_DATA_MODEL)
).toEqual(SearchIndex.DASHBOARD_DATA_MODEL);
expect(getSearchIndexFromEntityType(EntityType.SEARCH_INDEX)).toEqual(
SearchIndex.SEARCH_INDEX
);
expect(getSearchIndexFromEntityType(EntityType.DATABASE_SERVICE)).toEqual(
SearchIndex.DATABASE_SERVICE
);
expect(getSearchIndexFromEntityType(EntityType.MESSAGING_SERVICE)).toEqual(
SearchIndex.MESSAGING_SERVICE
);
expect(getSearchIndexFromEntityType(EntityType.DASHBOARD_SERVICE)).toEqual(
SearchIndex.DASHBOARD_SERVICE
);
expect(getSearchIndexFromEntityType(EntityType.PIPELINE_SERVICE)).toEqual(
SearchIndex.PIPELINE_SERVICE
);
expect(getSearchIndexFromEntityType(EntityType.MLMODEL_SERVICE)).toEqual(
SearchIndex.ML_MODEL_SERVICE
);
expect(getSearchIndexFromEntityType(EntityType.STORAGE_SERVICE)).toEqual(
SearchIndex.STORAGE_SERVICE
);
expect(getSearchIndexFromEntityType(EntityType.SEARCH_SERVICE)).toEqual(
SearchIndex.SEARCH_SERVICE
);
expect(getSearchIndexFromEntityType(EntityType.DOMAIN)).toEqual(
SearchIndex.DOMAIN
);
expect(getSearchIndexFromEntityType(EntityType.DATA_PRODUCT)).toEqual(
SearchIndex.DATA_PRODUCT
);
});
});

6
openmetadata-ui/src/main/resources/ui/src/utils/Explore.utils.ts
View File

@ -18,8 +18,6 @@ import {
SearchHitCounts,
} from '../components/Explore/ExplorePage.interface';
import { SearchDropdownOption } from '../components/SearchDropdown/SearchDropdown.interface';
import { EntityTypeSearchIndexMapping } from '../constants/explore.constants';
import { EntityType } from '../enums/entity.enum';
import { Aggregations } from '../interface/search.interface';
import {
QueryFieldInterface,
@ -149,7 +147,3 @@ export const getQuickFilterQuery = (data: ExploreQuickFilterField[]) => {
return quickFilterQuery;
};
export const getSearchIndexFromEntityType = (entityType: EntityType) => {
return EntityTypeSearchIndexMapping[entityType];
};

16
openmetadata-ui/src/main/resources/ui/src/utils/GlossaryUtils.test.ts
View File

@ -36,6 +36,22 @@ describe('Glossary Utils', () => {
],
},
},
{
bool: {
must_not: [
{
term: {
entityType: 'glossaryTerm',
},
},
{
term: {
entityType: 'tag',
},
},
],
},
},
],
},
},

16
openmetadata-ui/src/main/resources/ui/src/utils/GlossaryUtils.ts
View File

@ -113,6 +113,22 @@ export const getQueryFilterToExcludeTerm = (fqn: string) => ({
],
},
},
{
bool: {
must_not: [
{
term: {
entityType: 'glossaryTerm',
},
},
{
term: {
entityType: 'tag',
},
},
],
},
},
],
},
},

86
openmetadata-ui/src/main/resources/ui/src/utils/SearchClassBase.test.ts Normal file
View File

@ -0,0 +1,86 @@
/*
* Copyright 2024 Collate.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { EntityType } from '../enums/entity.enum';
import { SearchIndex } from '../enums/search.enum';
import { SearchClassBase } from './SearchClassBase';
describe('SearchClassBase', () => {
let searchClassBase: SearchClassBase;
beforeEach(() => {
searchClassBase = new SearchClassBase();
});
it('should return the correct search index for each entity type', () => {
const searchIndexMapping =
searchClassBase.getEntityTypeSearchIndexMapping();
expect(searchIndexMapping[EntityType.ALL]).toEqual(SearchIndex.ALL);
expect(searchIndexMapping[EntityType.TABLE]).toEqual(SearchIndex.TABLE);
expect(searchIndexMapping[EntityType.PIPELINE]).toEqual(
SearchIndex.PIPELINE
);
expect(searchIndexMapping[EntityType.DASHBOARD]).toEqual(
SearchIndex.DASHBOARD
);
expect(searchIndexMapping[EntityType.MLMODEL]).toEqual(SearchIndex.MLMODEL);
expect(searchIndexMapping[EntityType.TOPIC]).toEqual(SearchIndex.TOPIC);
expect(searchIndexMapping[EntityType.CONTAINER]).toEqual(
SearchIndex.CONTAINER
);
expect(searchIndexMapping[EntityType.TAG]).toEqual(SearchIndex.TAG);
expect(searchIndexMapping[EntityType.GLOSSARY_TERM]).toEqual(
SearchIndex.GLOSSARY
);
expect(searchIndexMapping[EntityType.STORED_PROCEDURE]).toEqual(
SearchIndex.STORED_PROCEDURE
);
expect(searchIndexMapping[EntityType.DASHBOARD_DATA_MODEL]).toEqual(
SearchIndex.DASHBOARD_DATA_MODEL
);
expect(searchIndexMapping[EntityType.SEARCH_INDEX]).toEqual(
SearchIndex.SEARCH_INDEX
);
expect(searchIndexMapping[EntityType.DATABASE_SERVICE]).toEqual(
SearchIndex.DATABASE_SERVICE
);
expect(searchIndexMapping[EntityType.MESSAGING_SERVICE]).toEqual(
SearchIndex.MESSAGING_SERVICE
);
expect(searchIndexMapping[EntityType.DASHBOARD_SERVICE]).toEqual(
SearchIndex.DASHBOARD_SERVICE
);
expect(searchIndexMapping[EntityType.PIPELINE_SERVICE]).toEqual(
SearchIndex.PIPELINE_SERVICE
);
expect(searchIndexMapping[EntityType.MLMODEL_SERVICE]).toEqual(
SearchIndex.ML_MODEL_SERVICE
);
expect(searchIndexMapping[EntityType.STORAGE_SERVICE]).toEqual(
SearchIndex.STORAGE_SERVICE
);
expect(searchIndexMapping[EntityType.SEARCH_SERVICE]).toEqual(
SearchIndex.SEARCH_SERVICE
);
expect(searchIndexMapping[EntityType.DOMAIN]).toEqual(SearchIndex.DOMAIN);
expect(searchIndexMapping[EntityType.DATA_PRODUCT]).toEqual(
SearchIndex.DATA_PRODUCT
);
expect(searchIndexMapping[EntityType.DATABASE]).toEqual(
SearchIndex.DATABASE
);
expect(searchIndexMapping[EntityType.DATABASE_SCHEMA]).toEqual(
SearchIndex.DATABASE_SCHEMA
);
});
});

28
openmetadata-ui/src/main/resources/ui/src/utils/SearchClassBase.ts
View File

@ -47,6 +47,34 @@ import i18n from './i18next/LocalUtil';
import { getServiceIcon } from './TableUtils';
class SearchClassBase {
public getEntityTypeSearchIndexMapping(): Record<string, SearchIndex> {
return {
[EntityType.ALL]: SearchIndex.ALL,
[EntityType.TABLE]: SearchIndex.TABLE,
[EntityType.PIPELINE]: SearchIndex.PIPELINE,
[EntityType.DASHBOARD]: SearchIndex.DASHBOARD,
[EntityType.MLMODEL]: SearchIndex.MLMODEL,
[EntityType.TOPIC]: SearchIndex.TOPIC,
[EntityType.CONTAINER]: SearchIndex.CONTAINER,
[EntityType.TAG]: SearchIndex.TAG,
[EntityType.GLOSSARY_TERM]: SearchIndex.GLOSSARY,
[EntityType.STORED_PROCEDURE]: SearchIndex.STORED_PROCEDURE,
[EntityType.DASHBOARD_DATA_MODEL]: SearchIndex.DASHBOARD_DATA_MODEL,
[EntityType.SEARCH_INDEX]: SearchIndex.SEARCH_INDEX,
[EntityType.DATABASE_SERVICE]: SearchIndex.DATABASE_SERVICE,
[EntityType.MESSAGING_SERVICE]: SearchIndex.MESSAGING_SERVICE,
[EntityType.DASHBOARD_SERVICE]: SearchIndex.DASHBOARD_SERVICE,
[EntityType.PIPELINE_SERVICE]: SearchIndex.PIPELINE_SERVICE,
[EntityType.MLMODEL_SERVICE]: SearchIndex.ML_MODEL_SERVICE,
[EntityType.STORAGE_SERVICE]: SearchIndex.STORAGE_SERVICE,
[EntityType.SEARCH_SERVICE]: SearchIndex.SEARCH_SERVICE,
[EntityType.DOMAIN]: SearchIndex.DOMAIN,
[EntityType.DATA_PRODUCT]: SearchIndex.DATA_PRODUCT,
[EntityType.DATABASE]: SearchIndex.DATABASE,
[EntityType.DATABASE_SCHEMA]: SearchIndex.DATABASE_SCHEMA,
};
}
public getTabsInfo() {
return {
[SearchIndex.TABLE]: {