UI: Handle deleted flag in /aggregate API to restore dropdown search results (#23558)

* add searchSettings aggregation in searchSettings.json

* refactor: update ExploreQuickFilters and ExploreUtils to include showDeleted and deleted parameters

* feat: enhance ExploreDiscovery tests to handle deleted assets visibility based on showDeleted toggle

* Revert schemaChanges.sql files to match main

---------

Co-authored-by: Aniket Katkar <aniketkatkar97@gmail.com>

openmetadata-ui/src/main/resources/ui/playwright/e2e/Flow/ExploreDiscovery.spec.ts

@@ -11,33 +11,58 @@
  *  limitations under the License.
  */
 import test, { expect } from '@playwright/test';
+import { SidebarItem } from '../../constant/sidebar';
+import { Domain } from '../../support/domain/Domain';
 import { TableClass } from '../../support/entity/TableClass';
+import { UserClass } from '../../support/user/UserClass';
 import { createNewPage, redirectToHomePage } from '../../utils/common';
+import {
+  getEncodedFqn,
+  waitForAllLoadersToDisappear,
+} from '../../utils/entity';
 import { getJsonTreeObject } from '../../utils/exploreDiscovery';
+import { sidebarClick } from '../../utils/sidebar';
 
 // use the admin user to login
 test.use({ storageState: 'playwright/.auth/admin.json' });
 
 const table = new TableClass();
 const table1 = new TableClass();
+const user = new UserClass();
+const domain = new Domain();
 
 test.describe('Explore Assets Discovery', () => {
   test.beforeAll(async ({ browser }) => {
     const { apiContext, afterAction } = await createNewPage(browser);
 
+    await user.create(apiContext);
+    await domain.create(apiContext);
     await table.create(apiContext);
     await table1.create(apiContext);
-    await table.delete(apiContext, false);
+    await table.patch({
-    // await table1.delete(apiContext, false);
+      apiContext,
-
+      patchData: [
-    await afterAction();
+        {
+          op: 'add',
+          value: {
+            type: 'user',
+            id: user.responseData.id,
+          },
+          path: '/owners/0',
+        },
+        {
+          op: 'add',
+          path: '/domains/0',
+          value: {
+            id: domain.responseData.id,
+            type: 'domain',
+            name: domain.responseData.name,
+            displayName: domain.responseData.displayName,
+          },
+        },
+      ],
     });
-
+    await table.delete(apiContext, false);
-  test.afterAll(async ({ browser }) => {
-    const { apiContext, afterAction } = await createNewPage(browser);
-
-    await table.delete(apiContext);
-    await table1.delete(apiContext);
 
     await afterAction();
   });
@@ -229,4 +254,144 @@ test.describe('Explore Assets Discovery', () => {
       page.locator('.ant-popover-inner-content').textContent()
     ).not.toContain(table1.entityResponseData.name);
   });
+
+  test('Should not display domain and owner of deleted asset in suggestions when showDeleted is off', async ({
+    page,
+  }) => {
+    await sidebarClick(page, SidebarItem.EXPLORE);
+    await page.waitForLoadState('networkidle');
+    await waitForAllLoadersToDisappear(page);
+
+    // The user should not be visible in the owners filter when the deleted switch is off
+    await page.click('[data-testid="search-dropdown-Owners"]');
+    const searchResOwner = page.waitForResponse(
+      `/api/v1/search/aggregate?index=dataAsset&field=owners.displayName.keyword*deleted=false*`
+    );
+
+    await page.fill(
+      '[data-testid="search-input"]',
+      user.responseData.displayName
+    );
+    await searchResOwner;
+
+    await waitForAllLoadersToDisappear(page);
+
+    await expect(
+      page
+        .getByTestId('drop-down-menu')
+        .getByTestId(user.responseData.displayName)
+    ).not.toBeAttached();
+
+    await page.getByTestId('close-btn').click();
+
+    // The domain should not be visible in the domains filter when the deleted switch is off
+    await page.click('[data-testid="search-dropdown-Domains"]');
+
+    const searchResDomain = page.waitForResponse(
+      `/api/v1/search/aggregate?index=dataAsset&field=domains.displayName.keyword*deleted=false*`
+    );
+
+    await page.fill(
+      '[data-testid="search-input"]',
+      domain.responseData.displayName
+    );
+    await searchResDomain;
+
+    await waitForAllLoadersToDisappear(page);
+
+    await expect(
+      page
+        .getByTestId('drop-down-menu')
+        .getByTestId(domain.responseData.displayName)
+    ).not.toBeAttached();
+
+    await page.getByTestId('close-btn').click();
+  });
+
+  test('Should display domain and owner of deleted asset in suggestions when showDeleted is on', async ({
+    page,
+  }) => {
+    await sidebarClick(page, SidebarItem.EXPLORE);
+    await page.waitForLoadState('networkidle');
+    await waitForAllLoadersToDisappear(page);
+
+    // Click on the show deleted toggle button
+    await page.getByTestId('show-deleted').click();
+
+    await page.waitForLoadState('networkidle');
+    await waitForAllLoadersToDisappear(page);
+
+    // The user should be visible in the owners filter when the deleted switch is on
+    const ownerSearchText = user.responseData.displayName.toLowerCase();
+    await page.click('[data-testid="search-dropdown-Owners"]');
+
+    const searchResOwner = page.waitForResponse(
+      `/api/v1/search/aggregate?index=dataAsset&field=owners.displayName.keyword*deleted=true*`
+    );
+
+    await page.fill('[data-testid="search-input"]', ownerSearchText);
+    await searchResOwner;
+
+    await waitForAllLoadersToDisappear(page);
+
+    await expect(
+      page.getByTestId('drop-down-menu').getByTestId(ownerSearchText)
+    ).toBeAttached();
+
+    await page
+      .getByTestId('drop-down-menu')
+      .getByTestId(ownerSearchText)
+      .click();
+
+    const fetchWithOwner = page.waitForResponse(
+      `/api/v1/search/query?*deleted=true*owners.displayName.keyword*${ownerSearchText}*`
+    );
+    await page.getByTestId('update-btn').click();
+    await fetchWithOwner;
+
+    await page.waitForLoadState('networkidle');
+    await waitForAllLoadersToDisappear(page);
+
+    // The domain should be visible in the domains filter when the deleted switch is on
+    const domainSearchText = domain.responseData.displayName.toLowerCase();
+    await page.click('[data-testid="search-dropdown-Domains"]');
+
+    const searchResDomain = page.waitForResponse(
+      `/api/v1/search/aggregate?index=dataAsset&field=domains.displayName.keyword*deleted=true*`
+    );
+
+    await page.fill('[data-testid="search-input"]', domainSearchText);
+    await searchResDomain;
+
+    await waitForAllLoadersToDisappear(page);
+
+    await expect(
+      page.getByTestId('drop-down-menu').getByTestId(domainSearchText)
+    ).toBeAttached();
+
+    await page
+      .getByTestId('drop-down-menu')
+      .getByTestId(domainSearchText)
+      .click();
+
+    const fetchWithDomain = page.waitForResponse(
+      `/api/v1/search/query?*deleted=true*domains.displayName.keyword*${getEncodedFqn(
+        domainSearchText,
+        true
+      )}*`
+    );
+    await page.getByTestId('update-btn').click();
+    await fetchWithDomain;
+
+    await page.waitForLoadState('networkidle');
+    await waitForAllLoadersToDisappear(page);
+
+    // Only the table option should be visible for the data assets filter when the deleted switch is on
+    // with the owner and domain filter applied
+    await page.click('[data-testid="search-dropdown-Data Assets"]');
+
+    await expect(
+      page.getByTestId('drop-down-menu').getByTestId('table')
+    ).toBeAttached();
+  });
 });

openmetadata-ui/src/main/resources/ui/src/components/Explore/ExploreQuickFilters.tsx

@@ -94,7 +94,8 @@ const ExploreQuickFilters: FC<ExploreQuickFiltersProps> = ({
           key,
           '',
           JSON.stringify(combinedQueryFilter),
-          independent
+          independent,
+          showDeleted
         ),
         key === TIER_FQN_KEY
           ? getTags({ parent: 'Tier', limit: 50 })
@@ -159,7 +160,8 @@ const ExploreQuickFilters: FC<ExploreQuickFiltersProps> = ({
           key,
           value,
           JSON.stringify(combinedQueryFilter),
-          independent
+          independent,
+          showDeleted
         );
 
         const buckets = res.data.aggregations[`sterms#${key}`].buckets;

openmetadata-ui/src/main/resources/ui/src/rest/miscAPI.ts

@@ -168,7 +168,8 @@ export const getAggregateFieldOptions = (
   field: string,
   value: string,
   q: string,
-  sourceFields?: string
+  sourceFields?: string,
+  deleted = false
 ) => {
   const withWildCardValue = value
     ? `.*${escapeESReservedCharacters(value)}.*`
@@ -179,6 +180,7 @@ export const getAggregateFieldOptions = (
     value: withWildCardValue,
     q,
     sourceFields,
+    deleted,
   };
 
   return APIClient.get<SearchResponse<ExploreSearchIndex>>(

openmetadata-ui/src/main/resources/ui/src/utils/ExploreUtils.tsx

@@ -200,41 +200,6 @@ export const extractTermKeys = (objects: QueryFieldInterface[]): string[] => {
   return termKeys;
 };
 
-export const getSubLevelHierarchyKey = (
-  isDatabaseHierarchy = false,
-  filterField?: ExploreQuickFilterField[],
-  key?: EntityFields,
-  value?: string
-) => {
-  const queryFilter = {
-    query: { bool: {} },
-  };
-
-  if ((key && value) || filterField) {
-    (queryFilter.query.bool as EsBoolQuery).must = isUndefined(filterField)
-      ? { term: { [key ?? '']: value } }
-      : getExploreQueryFilterMust(filterField);
-  }
-
-  const bucketMapping = isDatabaseHierarchy
-    ? {
-        [EntityFields.SERVICE_TYPE]: EntityFields.SERVICE,
-        [EntityFields.SERVICE]: EntityFields.DATABASE_DISPLAY_NAME,
-        [EntityFields.DATABASE_DISPLAY_NAME]:
-          EntityFields.DATABASE_SCHEMA_DISPLAY_NAME,
-        [EntityFields.DATABASE_SCHEMA_DISPLAY_NAME]: EntityFields.ENTITY_TYPE,
-      }
-    : {
-        [EntityFields.SERVICE_TYPE]: EntityFields.SERVICE,
-        [EntityFields.SERVICE]: EntityFields.ENTITY_TYPE,
-      };
-
-  return {
-    bucket: bucketMapping[key as DatabaseFields] ?? EntityFields.SERVICE_TYPE,
-    queryFilter,
-  };
-};
-
 export const getExploreQueryFilterMust = (data: ExploreQuickFilterField[]) => {
   const must = [] as Array<QueryFieldInterface>;
 
@@ -281,6 +246,41 @@ export const getExploreQueryFilterMust = (data: ExploreQuickFilterField[]) => {
   return must;
 };
 
+export const getSubLevelHierarchyKey = (
+  isDatabaseHierarchy = false,
+  filterField?: ExploreQuickFilterField[],
+  key?: EntityFields,
+  value?: string
+) => {
+  const queryFilter = {
+    query: { bool: {} },
+  };
+
+  if ((key && value) || filterField) {
+    (queryFilter.query.bool as EsBoolQuery).must = isUndefined(filterField)
+      ? { term: { [key ?? '']: value } }
+      : getExploreQueryFilterMust(filterField);
+  }
+
+  const bucketMapping = isDatabaseHierarchy
+    ? {
+        [EntityFields.SERVICE_TYPE]: EntityFields.SERVICE,
+        [EntityFields.SERVICE]: EntityFields.DATABASE_DISPLAY_NAME,
+        [EntityFields.DATABASE_DISPLAY_NAME]:
+          EntityFields.DATABASE_SCHEMA_DISPLAY_NAME,
+        [EntityFields.DATABASE_SCHEMA_DISPLAY_NAME]: EntityFields.ENTITY_TYPE,
+      }
+    : {
+        [EntityFields.SERVICE_TYPE]: EntityFields.SERVICE,
+        [EntityFields.SERVICE]: EntityFields.ENTITY_TYPE,
+      };
+
+  return {
+    bucket: bucketMapping[key as DatabaseFields] ?? EntityFields.SERVICE_TYPE,
+    queryFilter,
+  };
+};
+
 export const updateTreeData = (
   list: ExploreTreeNode[],
   key: React.Key,
@@ -360,11 +360,12 @@ export const getAggregationOptions = async (
   key: string,
   value: string,
   filter: string,
-  isIndependent: boolean
+  isIndependent: boolean,
+  deleted = false
 ) => {
   return isIndependent
     ? postAggregateFieldOptions(index, key, value, filter)
-    : getAggregateFieldOptions(index, key, value, filter);
+    : getAggregateFieldOptions(index, key, value, filter, undefined, deleted);
 };
 
 export const updateTreeDataWithCounts = (
@@ -412,7 +413,7 @@ export const isElasticsearchError = (error: unknown): boolean => {
     return false;
   }
 
-  const data = axiosError.response.data as Record<string, any>;
+  const data = axiosError.response.data as Record<string, unknown>;
   const message = data.message as string;
 
   return (