Fix #14038: Inconsistent search results between auto-suggest box and … (#14747)

* Fix #14038: Inconsistent search results between auto-suggest box and actual search results

* use dataAsset instead of all alias

---------

Co-authored-by: karanh37 <karanh37@gmail.com>

openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java

@@ -330,6 +330,8 @@ public class ElasticSearchClient implements SearchClient {
               "search_service_index",
               "metadata_service_index" -> buildServiceSearchBuilder(
               request.getQuery(), request.getFrom(), request.getSize());
+          case "all", "dataAsset" -> buildSearchAcrossIndexesBuilder(
+              request.getQuery(), request.getFrom(), request.getSize());
           default -> buildAggregateSearchBuilder(
               request.getQuery(), request.getFrom(), request.getSize());
         };
@@ -717,11 +719,34 @@ public class ElasticSearchClient implements SearchClient {
     return addAggregation(searchSourceBuilder);
   }
 
+  private static SearchSourceBuilder buildSearchAcrossIndexesBuilder(
+      String query, int from, int size) {
+    QueryStringQueryBuilder queryStringBuilder =
+        QueryBuilders.queryStringQuery(query)
+            .fields(SearchIndex.getAllFields())
+            .type(MultiMatchQueryBuilder.Type.MOST_FIELDS)
+            .fuzziness(Fuzziness.AUTO);
+    FieldValueFactorFunctionBuilder boostScoreBuilder =
+        ScoreFunctionBuilders.fieldValueFactorFunction("usageSummary.weeklyStats.count")
+            .missing(0)
+            .factor(0.2f);
+    FunctionScoreQueryBuilder.FilterFunctionBuilder[] functions =
+        new FunctionScoreQueryBuilder.FilterFunctionBuilder[] {
+          new FunctionScoreQueryBuilder.FilterFunctionBuilder(boostScoreBuilder)
+        };
+    FunctionScoreQueryBuilder queryBuilder =
+        QueryBuilders.functionScoreQuery(queryStringBuilder, functions);
+    queryBuilder.boostMode(CombineFunction.SUM);
+    SearchSourceBuilder searchSourceBuilder = searchBuilder(queryBuilder, null, from, size);
+    return addAggregation(searchSourceBuilder);
+  }
+
   private static SearchSourceBuilder buildTableSearchBuilder(String query, int from, int size) {
     QueryStringQueryBuilder queryStringBuilder =
         QueryBuilders.queryStringQuery(query)
             .fields(TableIndex.getFields())
             .type(MultiMatchQueryBuilder.Type.MOST_FIELDS)
+            .fuzziness(Fuzziness.AUTO)
             .tieBreaker(0.9f);
 
     FieldValueFactorFunctionBuilder boostScoreBuilder =

openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java

@@ -322,6 +322,8 @@ public class OpenSearchClient implements SearchClient {
               "search_service_index",
               "metadata_service_index" -> buildServiceSearchBuilder(
               request.getQuery(), request.getFrom(), request.getSize());
+          case "all", "dataAsset" -> buildSearchAcrossIndexesBuilder(
+              request.getQuery(), request.getFrom(), request.getSize());
           default -> buildAggregateSearchBuilder(
               request.getQuery(), request.getFrom(), request.getSize());
         };
@@ -731,6 +733,28 @@ public class OpenSearchClient implements SearchClient {
     return addAggregation(searchSourceBuilder);
   }
 
+  private static SearchSourceBuilder buildSearchAcrossIndexesBuilder(
+      String query, int from, int size) {
+    QueryStringQueryBuilder queryStringBuilder =
+        QueryBuilders.queryStringQuery(query)
+            .fields(SearchIndex.getAllFields())
+            .type(MultiMatchQueryBuilder.Type.MOST_FIELDS)
+            .fuzziness(Fuzziness.AUTO);
+    FieldValueFactorFunctionBuilder boostScoreBuilder =
+        ScoreFunctionBuilders.fieldValueFactorFunction("usageSummary.weeklyStats.count")
+            .missing(0)
+            .factor(0.2f);
+    FunctionScoreQueryBuilder.FilterFunctionBuilder[] functions =
+        new FunctionScoreQueryBuilder.FilterFunctionBuilder[] {
+          new FunctionScoreQueryBuilder.FilterFunctionBuilder(boostScoreBuilder)
+        };
+    FunctionScoreQueryBuilder queryBuilder =
+        QueryBuilders.functionScoreQuery(queryStringBuilder, functions);
+    queryBuilder.boostMode(CombineFunction.SUM);
+    SearchSourceBuilder searchSourceBuilder = searchBuilder(queryBuilder, null, from, size);
+    return addAggregation(searchSourceBuilder);
+  }
+
   private static SearchSourceBuilder buildTableSearchBuilder(String query, int from, int size) {
     QueryStringQueryBuilder queryStringBuilder =
         QueryBuilders.queryStringQuery(query)

openmetadata-service/src/main/resources/elasticsearch/indexMapping.json

@@ -111,13 +111,13 @@
     "indexName": "glossary_term_search_index",
     "indexMappingFile": "/elasticsearch/%s/glossary_term_index_mapping.json",
     "alias": "glossaryTerm",
-    "parentAliases": ["all", "glossary"]
+    "parentAliases": ["all", "glossary", "dataAsset"]
   },
   "tag": {
     "indexName": "tag_search_index",
     "indexMappingFile": "/elasticsearch/%s/tag_index_mapping.json",
     "alias": "tag",
-    "parentAliases": ["classification","all"]
+    "parentAliases": ["classification","all", "dataAsset"]
   },
   "classification": {
     "indexName": "classification_search_index",

openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/Explore.spec.js

@@ -0,0 +1,56 @@
+/*
+ *  Copyright 2024 Collate.
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+import { interceptURL, verifyResponseStatusCode } from '../../common/common';
+
+describe('Explore Page', () => {
+  before(() => {
+    cy.login();
+  });
+
+  it('should verify order of search results', () => {
+    const searchText = 'customer';
+    interceptURL(
+      'GET',
+      `api/v1/search/query?q=**&index=dataAsset**`,
+      'suggestAPI'
+    );
+
+    interceptURL(
+      'GET',
+      `api/v1/search/query?q=**&index=table_search_index**`,
+      'searchAPI'
+    );
+
+    cy.get('[data-testid="app-bar-item-explore"]').click();
+    cy.get('[data-testid="searchBox"]').clear();
+    cy.get('[data-testid="searchBox"]').type(searchText);
+
+    verifyResponseStatusCode('@suggestAPI', 200);
+
+    const linksArray = [];
+    cy.get('[data-testid="group-table_search_index"] button a').each(
+      ($link) => {
+        linksArray.push($link.attr('href'));
+      }
+    );
+
+    cy.get('[data-testid="searchBox"]').type('{enter}');
+    verifyResponseStatusCode('@searchAPI', 200);
+
+    cy.wrap(linksArray).each((link, index) => {
+      cy.get(`[data-testid="entity-link"]`)
+        .eq(index)
+        .should('have.attr', 'href', link);
+    });
+  });
+});

openmetadata-ui/src/main/resources/ui/src/components/AppBar/Suggestions.tsx

@@ -136,14 +136,14 @@ const Suggestions = ({
     }
 
     return (
-      <>
+      <div data-testid={`group-${searchIndex}`}>
         {getGroupLabel(searchIndex)}
         {suggestions.map((suggestion: SearchSuggestions[number]) => {
           return getSuggestionElement(suggestion, searchIndex, () =>
             setIsOpen(false)
           );
         })}
-      </>
+      </div>
     );
   };
 
@@ -205,7 +205,7 @@ const Suggestions = ({
         '',
         '',
         '',
-        searchCriteria ?? SearchIndex.ALL
+        searchCriteria ?? SearchIndex.DATA_ASSET
       );
 
       if (res.data) {

openmetadata-ui/src/main/resources/ui/src/constants/explore.constants.ts

@@ -12,8 +12,6 @@
  */
 
 import { SortingField } from '../components/Explore/SortingDropDown';
-import { EntityType } from '../enums/entity.enum';
-import { SearchIndex } from '../enums/search.enum';
 import i18n from '../utils/i18next/LocalUtil';
 
 export const INITIAL_SORT_FIELD = 'updatedAt';
@@ -59,29 +57,3 @@ export const COMMON_FILTERS_FOR_DIFFERENT_TABS = [
   'owner.displayName',
   'tags.tagFQN',
 ];
-
-export const EntityTypeSearchIndexMapping: Record<string, SearchIndex> = {
-  [EntityType.ALL]: SearchIndex.ALL,
-  [EntityType.TABLE]: SearchIndex.TABLE,
-  [EntityType.PIPELINE]: SearchIndex.PIPELINE,
-  [EntityType.DASHBOARD]: SearchIndex.DASHBOARD,
-  [EntityType.MLMODEL]: SearchIndex.MLMODEL,
-  [EntityType.TOPIC]: SearchIndex.TOPIC,
-  [EntityType.CONTAINER]: SearchIndex.CONTAINER,
-  [EntityType.TAG]: SearchIndex.TAG,
-  [EntityType.GLOSSARY_TERM]: SearchIndex.GLOSSARY,
-  [EntityType.STORED_PROCEDURE]: SearchIndex.STORED_PROCEDURE,
-  [EntityType.DASHBOARD_DATA_MODEL]: SearchIndex.DASHBOARD_DATA_MODEL,
-  [EntityType.SEARCH_INDEX]: SearchIndex.SEARCH_INDEX,
-  [EntityType.DATABASE_SERVICE]: SearchIndex.DATABASE_SERVICE,
-  [EntityType.MESSAGING_SERVICE]: SearchIndex.MESSAGING_SERVICE,
-  [EntityType.DASHBOARD_SERVICE]: SearchIndex.DASHBOARD_SERVICE,
-  [EntityType.PIPELINE_SERVICE]: SearchIndex.PIPELINE_SERVICE,
-  [EntityType.MLMODEL_SERVICE]: SearchIndex.ML_MODEL_SERVICE,
-  [EntityType.STORAGE_SERVICE]: SearchIndex.STORAGE_SERVICE,
-  [EntityType.SEARCH_SERVICE]: SearchIndex.SEARCH_SERVICE,
-  [EntityType.DOMAIN]: SearchIndex.DOMAIN,
-  [EntityType.DATA_PRODUCT]: SearchIndex.DATA_PRODUCT,
-  [EntityType.DATABASE]: SearchIndex.DATABASE,
-  [EntityType.DATABASE_SCHEMA]: SearchIndex.DATABASE_SCHEMA,
-};

openmetadata-ui/src/main/resources/ui/src/pages/ExplorePage/ExplorePageV1.component.tsx

@@ -36,7 +36,6 @@ import { useTourProvider } from '../../components/TourProvider/TourProvider';
 import { getExplorePath, PAGE_SIZE } from '../../constants/constants';
 import {
   COMMON_FILTERS_FOR_DIFFERENT_TABS,
-  EntityTypeSearchIndexMapping,
   INITIAL_SORT_FIELD,
 } from '../../constants/explore.constants';
 import {
@@ -63,6 +62,8 @@ import {
 
 const ExplorePageV1: FunctionComponent = () => {
   const tabsInfo = searchClassBase.getTabsInfo();
+  const EntityTypeSearchIndexMapping =
+    searchClassBase.getEntityTypeSearchIndexMapping();
   const location = useLocation();
   const history = useHistory();
   const { isTourOpen } = useTourProvider();

openmetadata-ui/src/main/resources/ui/src/utils/Explore.utils.test.ts

@@ -11,12 +11,7 @@
  *  limitations under the License.
  */
 import { ExploreQuickFilterField } from '../components/Explore/ExplorePage.interface';
-import { EntityType } from '../enums/entity.enum';
+import { getQuickFilterQuery } from './Explore.utils';
-import { SearchIndex } from '../enums/search.enum';
-import {
-  getQuickFilterQuery,
-  getSearchIndexFromEntityType,
-} from './Explore.utils';
 
 describe('Explore Utils', () => {
   it('should return undefined if data is empty', () => {
@@ -73,70 +68,4 @@ describe('Explore Utils', () => {
 
     expect(result).toEqual(expectedQuery);
   });
-
-  it('getSearchIndexFromEntityType should return the correct search index for each entity type', () => {
-    expect(getSearchIndexFromEntityType(EntityType.ALL)).toEqual(
-      SearchIndex.ALL
-    );
-    expect(getSearchIndexFromEntityType(EntityType.TABLE)).toEqual(
-      SearchIndex.TABLE
-    );
-    expect(getSearchIndexFromEntityType(EntityType.PIPELINE)).toEqual(
-      SearchIndex.PIPELINE
-    );
-    expect(getSearchIndexFromEntityType(EntityType.DASHBOARD)).toEqual(
-      SearchIndex.DASHBOARD
-    );
-    expect(getSearchIndexFromEntityType(EntityType.MLMODEL)).toEqual(
-      SearchIndex.MLMODEL
-    );
-    expect(getSearchIndexFromEntityType(EntityType.TOPIC)).toEqual(
-      SearchIndex.TOPIC
-    );
-    expect(getSearchIndexFromEntityType(EntityType.CONTAINER)).toEqual(
-      SearchIndex.CONTAINER
-    );
-    expect(getSearchIndexFromEntityType(EntityType.TAG)).toEqual(
-      SearchIndex.TAG
-    );
-    expect(getSearchIndexFromEntityType(EntityType.GLOSSARY_TERM)).toEqual(
-      SearchIndex.GLOSSARY
-    );
-    expect(getSearchIndexFromEntityType(EntityType.STORED_PROCEDURE)).toEqual(
-      SearchIndex.STORED_PROCEDURE
-    );
-    expect(
-      getSearchIndexFromEntityType(EntityType.DASHBOARD_DATA_MODEL)
-    ).toEqual(SearchIndex.DASHBOARD_DATA_MODEL);
-    expect(getSearchIndexFromEntityType(EntityType.SEARCH_INDEX)).toEqual(
-      SearchIndex.SEARCH_INDEX
-    );
-    expect(getSearchIndexFromEntityType(EntityType.DATABASE_SERVICE)).toEqual(
-      SearchIndex.DATABASE_SERVICE
-    );
-    expect(getSearchIndexFromEntityType(EntityType.MESSAGING_SERVICE)).toEqual(
-      SearchIndex.MESSAGING_SERVICE
-    );
-    expect(getSearchIndexFromEntityType(EntityType.DASHBOARD_SERVICE)).toEqual(
-      SearchIndex.DASHBOARD_SERVICE
-    );
-    expect(getSearchIndexFromEntityType(EntityType.PIPELINE_SERVICE)).toEqual(
-      SearchIndex.PIPELINE_SERVICE
-    );
-    expect(getSearchIndexFromEntityType(EntityType.MLMODEL_SERVICE)).toEqual(
-      SearchIndex.ML_MODEL_SERVICE
-    );
-    expect(getSearchIndexFromEntityType(EntityType.STORAGE_SERVICE)).toEqual(
-      SearchIndex.STORAGE_SERVICE
-    );
-    expect(getSearchIndexFromEntityType(EntityType.SEARCH_SERVICE)).toEqual(
-      SearchIndex.SEARCH_SERVICE
-    );
-    expect(getSearchIndexFromEntityType(EntityType.DOMAIN)).toEqual(
-      SearchIndex.DOMAIN
-    );
-    expect(getSearchIndexFromEntityType(EntityType.DATA_PRODUCT)).toEqual(
-      SearchIndex.DATA_PRODUCT
-    );
-  });
 });

openmetadata-ui/src/main/resources/ui/src/utils/Explore.utils.ts

@@ -18,8 +18,6 @@ import {
   SearchHitCounts,
 } from '../components/Explore/ExplorePage.interface';
 import { SearchDropdownOption } from '../components/SearchDropdown/SearchDropdown.interface';
-import { EntityTypeSearchIndexMapping } from '../constants/explore.constants';
-import { EntityType } from '../enums/entity.enum';
 import { Aggregations } from '../interface/search.interface';
 import {
   QueryFieldInterface,
@@ -149,7 +147,3 @@ export const getQuickFilterQuery = (data: ExploreQuickFilterField[]) => {
 
   return quickFilterQuery;
 };
-
-export const getSearchIndexFromEntityType = (entityType: EntityType) => {
-  return EntityTypeSearchIndexMapping[entityType];
-};

openmetadata-ui/src/main/resources/ui/src/utils/GlossaryUtils.test.ts

@@ -36,6 +36,22 @@ describe('Glossary Utils', () => {
                 ],
               },
             },
+            {
+              bool: {
+                must_not: [
+                  {
+                    term: {
+                      entityType: 'glossaryTerm',
+                    },
+                  },
+                  {
+                    term: {
+                      entityType: 'tag',
+                    },
+                  },
+                ],
+              },
+            },
           ],
         },
       },

openmetadata-ui/src/main/resources/ui/src/utils/GlossaryUtils.ts

@@ -113,6 +113,22 @@ export const getQueryFilterToExcludeTerm = (fqn: string) => ({
             ],
           },
         },
+        {
+          bool: {
+            must_not: [
+              {
+                term: {
+                  entityType: 'glossaryTerm',
+                },
+              },
+              {
+                term: {
+                  entityType: 'tag',
+                },
+              },
+            ],
+          },
+        },
       ],
     },
   },

openmetadata-ui/src/main/resources/ui/src/utils/SearchClassBase.test.ts

@@ -0,0 +1,86 @@
+/*
+ *  Copyright 2024 Collate.
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+import { EntityType } from '../enums/entity.enum';
+import { SearchIndex } from '../enums/search.enum';
+import { SearchClassBase } from './SearchClassBase';
+
+describe('SearchClassBase', () => {
+  let searchClassBase: SearchClassBase;
+
+  beforeEach(() => {
+    searchClassBase = new SearchClassBase();
+  });
+
+  it('should return the correct search index for each entity type', () => {
+    const searchIndexMapping =
+      searchClassBase.getEntityTypeSearchIndexMapping();
+
+    expect(searchIndexMapping[EntityType.ALL]).toEqual(SearchIndex.ALL);
+    expect(searchIndexMapping[EntityType.TABLE]).toEqual(SearchIndex.TABLE);
+    expect(searchIndexMapping[EntityType.PIPELINE]).toEqual(
+      SearchIndex.PIPELINE
+    );
+    expect(searchIndexMapping[EntityType.DASHBOARD]).toEqual(
+      SearchIndex.DASHBOARD
+    );
+    expect(searchIndexMapping[EntityType.MLMODEL]).toEqual(SearchIndex.MLMODEL);
+    expect(searchIndexMapping[EntityType.TOPIC]).toEqual(SearchIndex.TOPIC);
+    expect(searchIndexMapping[EntityType.CONTAINER]).toEqual(
+      SearchIndex.CONTAINER
+    );
+    expect(searchIndexMapping[EntityType.TAG]).toEqual(SearchIndex.TAG);
+    expect(searchIndexMapping[EntityType.GLOSSARY_TERM]).toEqual(
+      SearchIndex.GLOSSARY
+    );
+    expect(searchIndexMapping[EntityType.STORED_PROCEDURE]).toEqual(
+      SearchIndex.STORED_PROCEDURE
+    );
+    expect(searchIndexMapping[EntityType.DASHBOARD_DATA_MODEL]).toEqual(
+      SearchIndex.DASHBOARD_DATA_MODEL
+    );
+    expect(searchIndexMapping[EntityType.SEARCH_INDEX]).toEqual(
+      SearchIndex.SEARCH_INDEX
+    );
+    expect(searchIndexMapping[EntityType.DATABASE_SERVICE]).toEqual(
+      SearchIndex.DATABASE_SERVICE
+    );
+    expect(searchIndexMapping[EntityType.MESSAGING_SERVICE]).toEqual(
+      SearchIndex.MESSAGING_SERVICE
+    );
+    expect(searchIndexMapping[EntityType.DASHBOARD_SERVICE]).toEqual(
+      SearchIndex.DASHBOARD_SERVICE
+    );
+    expect(searchIndexMapping[EntityType.PIPELINE_SERVICE]).toEqual(
+      SearchIndex.PIPELINE_SERVICE
+    );
+    expect(searchIndexMapping[EntityType.MLMODEL_SERVICE]).toEqual(
+      SearchIndex.ML_MODEL_SERVICE
+    );
+    expect(searchIndexMapping[EntityType.STORAGE_SERVICE]).toEqual(
+      SearchIndex.STORAGE_SERVICE
+    );
+    expect(searchIndexMapping[EntityType.SEARCH_SERVICE]).toEqual(
+      SearchIndex.SEARCH_SERVICE
+    );
+    expect(searchIndexMapping[EntityType.DOMAIN]).toEqual(SearchIndex.DOMAIN);
+    expect(searchIndexMapping[EntityType.DATA_PRODUCT]).toEqual(
+      SearchIndex.DATA_PRODUCT
+    );
+    expect(searchIndexMapping[EntityType.DATABASE]).toEqual(
+      SearchIndex.DATABASE
+    );
+    expect(searchIndexMapping[EntityType.DATABASE_SCHEMA]).toEqual(
+      SearchIndex.DATABASE_SCHEMA
+    );
+  });
+});

openmetadata-ui/src/main/resources/ui/src/utils/SearchClassBase.ts

@@ -47,6 +47,34 @@ import i18n from './i18next/LocalUtil';
 import { getServiceIcon } from './TableUtils';
 
 class SearchClassBase {
+  public getEntityTypeSearchIndexMapping(): Record<string, SearchIndex> {
+    return {
+      [EntityType.ALL]: SearchIndex.ALL,
+      [EntityType.TABLE]: SearchIndex.TABLE,
+      [EntityType.PIPELINE]: SearchIndex.PIPELINE,
+      [EntityType.DASHBOARD]: SearchIndex.DASHBOARD,
+      [EntityType.MLMODEL]: SearchIndex.MLMODEL,
+      [EntityType.TOPIC]: SearchIndex.TOPIC,
+      [EntityType.CONTAINER]: SearchIndex.CONTAINER,
+      [EntityType.TAG]: SearchIndex.TAG,
+      [EntityType.GLOSSARY_TERM]: SearchIndex.GLOSSARY,
+      [EntityType.STORED_PROCEDURE]: SearchIndex.STORED_PROCEDURE,
+      [EntityType.DASHBOARD_DATA_MODEL]: SearchIndex.DASHBOARD_DATA_MODEL,
+      [EntityType.SEARCH_INDEX]: SearchIndex.SEARCH_INDEX,
+      [EntityType.DATABASE_SERVICE]: SearchIndex.DATABASE_SERVICE,
+      [EntityType.MESSAGING_SERVICE]: SearchIndex.MESSAGING_SERVICE,
+      [EntityType.DASHBOARD_SERVICE]: SearchIndex.DASHBOARD_SERVICE,
+      [EntityType.PIPELINE_SERVICE]: SearchIndex.PIPELINE_SERVICE,
+      [EntityType.MLMODEL_SERVICE]: SearchIndex.ML_MODEL_SERVICE,
+      [EntityType.STORAGE_SERVICE]: SearchIndex.STORAGE_SERVICE,
+      [EntityType.SEARCH_SERVICE]: SearchIndex.SEARCH_SERVICE,
+      [EntityType.DOMAIN]: SearchIndex.DOMAIN,
+      [EntityType.DATA_PRODUCT]: SearchIndex.DATA_PRODUCT,
+      [EntityType.DATABASE]: SearchIndex.DATABASE,
+      [EntityType.DATABASE_SCHEMA]: SearchIndex.DATABASE_SCHEMA,
+    };
+  }
+
   public getTabsInfo() {
     return {
       [SearchIndex.TABLE]: {