ui: move queries from ES to REST API (#10947)

* initial commit for moving from ES to REST * updated filters * miner fix * fixed failing unit test * fixed failing cypress
2025-08-31 20:51:26 +00:00 · 2023-04-10 10:21:27 +05:30 · 2023-04-10 10:21:27 +05:30 · d79974dba1
commit d79974dba1
parent 445ac90ef9
7 changed files with 148 additions and 69 deletions
--- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/AddNewService/postgres.spec.js
+++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/AddNewService/postgres.spec.js
@ -161,11 +161,7 @@ describe('Postgres Ingestion', () => {
    );
    visitEntityDetailsPage(tableName, serviceName, 'tables');
    verifyResponseStatusCode('@entityDetailsPage', 200);
-    interceptURL(
-      'GET',
-      '/api/v1/search/query?q=&index=query_search_index*',
-      'queriesTab'
-    );
+    interceptURL('GET', '/api/v1/queries?*', 'queriesTab');
    cy.get('[data-testid="Queries"]').should('be.visible').trigger('click');
    verifyResponseStatusCode('@queriesTab', 200);
    // Validate that the triggered query is visible in the queries container
--- a/openmetadata-ui/src/main/resources/ui/src/components/TableQueries/QueryFilters/QueryFilters.component.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/TableQueries/QueryFilters/QueryFilters.component.tsx
@ -15,6 +15,7 @@ import { Space, Typography } from 'antd';
 import SearchDropdown from 'components/SearchDropdown/SearchDropdown';
 import { SearchDropdownOption } from 'components/SearchDropdown/SearchDropdown.interface';
 import { WILD_CARD_CHAR } from 'constants/char.constants';
+import { INITIAL_PAGING_VALUE } from 'constants/constants';
 import { QUERY_PAGE_FILTER } from 'constants/Query.constant';
 import { PROMISE_STATE } from 'enums/common.enum';
 import { debounce, isEmpty } from 'lodash';
@ -59,7 +60,6 @@ const QueryFilters = ({ onFilterChange }: QueryFiltersProps) => {
  ) => {
    setSelectedFilter((pre) => {
      const updatedFilter = { ...pre, [searchKey]: value };
-
      onFilterChange(updatedFilter);

      return updatedFilter;
@ -111,13 +111,13 @@ const QueryFilters = ({ onFilterChange }: QueryFiltersProps) => {
    }
  };

-  const debounceOnUserSearch = debounce(onUserSearch, 400);
-  const debounceOnTeamSearch = debounce(onTeamSearch, 400);
+  const debounceOnUserSearch = debounce(onUserSearch, 500);
+  const debounceOnTeamSearch = debounce(onTeamSearch, 500);

  const getInitialUserAndTeam = () => {
    const promise = [
-      getSearchedUsers(WILD_CARD_CHAR, 1),
-      getSearchedTeams(WILD_CARD_CHAR, 1),
+      getSearchedUsers(WILD_CARD_CHAR, INITIAL_PAGING_VALUE),
+      getSearchedTeams(WILD_CARD_CHAR, INITIAL_PAGING_VALUE),
    ];
    Promise.allSettled(promise)
      .then((res) => {
--- a/openmetadata-ui/src/main/resources/ui/src/components/TableQueries/TableQueries.interface.ts
+++ b/openmetadata-ui/src/main/resources/ui/src/components/TableQueries/TableQueries.interface.ts
@ -62,10 +62,13 @@ export interface QueryFiltersProps {
  onFilterChange: (value: QueryFiltersType) => void;
 }

-export type QuerySearchParams = QueryFiltersType & {
-  page: string;
-  tableId: string;
-  query: string;
+export type QuerySearchParams = {
+  queryFrom?: number;
+  after?: string;
+  tableId?: string;
+  query?: string;
+  user?: SearchDropdownOption[];
+  team?: SearchDropdownOption[];
 };

 export type QuerySearchShouldFilterType = {
--- a/openmetadata-ui/src/main/resources/ui/src/components/TableQueries/TableQueries.test.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/TableQueries/TableQueries.test.tsx
@ -19,8 +19,8 @@ import {
 } from '@testing-library/react';
 import React from 'react';
 import { MemoryRouter } from 'react-router-dom';
-import { searchQuery } from 'rest/searchAPI';
-import { MOCK_QUERIES_ES_DATA } from '../../mocks/Queries.mock';
+import { getQueriesList } from 'rest/queryAPI';
+import { MOCK_QUERIES, MOCK_QUERIES_ES_DATA } from '../../mocks/Queries.mock';
 import TableQueries from './TableQueries';
 import { TableQueriesProp } from './TableQueries.interface';

@ -49,6 +49,14 @@ jest.mock('rest/searchAPI', () => ({
    })
  ),
 }));
+jest.mock('rest/queryAPI', () => ({
+  ...jest.requireActual('rest/queryAPI'),
+  getQueriesList: jest
+    .fn()
+    .mockImplementation(() =>
+      Promise.resolve({ data: MOCK_QUERIES, paging: { total: 2 } })
+    ),
+}));
 jest.mock('components/PermissionProvider/PermissionProvider', () => ({
  usePermissionProvider: jest
    .fn()
@ -89,9 +97,10 @@ describe('Test TableQueries Component', () => {
  });

  it('Error placeholder should display if there is no data', async () => {
-    (searchQuery as jest.Mock).mockImplementationOnce(() =>
+    (getQueriesList as jest.Mock).mockImplementationOnce(() =>
      Promise.resolve({
-        hits: { hits: [], total: { value: 0 } },
+        data: [],
+        paging: { total: 0 },
      })
    );
    render(<TableQueries {...mockTableQueriesProp} />, {
@ -105,9 +114,10 @@ describe('Test TableQueries Component', () => {
  });

  it('If paging count is more than 10, pagination should be visible', async () => {
-    (searchQuery as jest.Mock).mockImplementationOnce(() =>
+    (getQueriesList as jest.Mock).mockImplementationOnce(() =>
      Promise.resolve({
-        hits: { hits: [], total: { value: 11 } },
+        data: MOCK_QUERIES,
+        paging: { total: 11 },
      })
    );
    render(<TableQueries {...mockTableQueriesProp} />, {
--- a/openmetadata-ui/src/main/resources/ui/src/components/TableQueries/TableQueries.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/TableQueries/TableQueries.tsx
@ -38,10 +38,20 @@ import Qs from 'qs';
 import React, { FC, useEffect, useMemo, useState } from 'react';
 import { useTranslation } from 'react-i18next';
 import { useHistory, useLocation } from 'react-router-dom';
-import { getQueryById, patchQueries, updateQueryVote } from 'rest/queryAPI';
+import {
+  getQueriesList,
+  getQueryById,
+  ListQueriesParams,
+  patchQueries,
+  updateQueryVote,
+} from 'rest/queryAPI';
 import { searchQuery } from 'rest/searchAPI';
 import { DEFAULT_ENTITY_PERMISSION } from 'utils/PermissionsUtils';
-import { createQueryFilter, parseSearchParams } from 'utils/Query/QueryUtils';
+import {
+  createQueryFilter,
+  parseSearchParams,
+  stringifySearchParams,
+} from 'utils/Query/QueryUtils';
 import { showErrorToast } from '../../utils/ToastUtils';
 import ErrorPlaceHolder from '../common/error-with-placeholder/ErrorPlaceHolder';
 import Loader from '../Loader/Loader';
@ -65,10 +75,13 @@ const TableQueries: FC<TableQueriesProp> = ({
  const { searchParams, selectedFilters } = useMemo(() => {
    const searchData = parseSearchParams(location.search);

-    const selectedFilters = {
-      user: searchData.user || [],
-      team: searchData.team || [],
-    };
+    const selectedFilters =
+      searchData.user || searchData.team
+        ? {
+            user: searchData.user || [],
+            team: searchData.team || [],
+          }
+        : undefined;

    return {
      searchParams: searchData,
@ -87,13 +100,19 @@ const TableQueries: FC<TableQueriesProp> = ({
    DEFAULT_ENTITY_PERMISSION
  );
  const [currentPage, setCurrentPage] = useState(
-    Number(searchParams.page) || INITIAL_PAGING_VALUE
+    Number(searchParams.queryFrom) || INITIAL_PAGING_VALUE
  );
-  const [appliedFilter, setAppliedFilter] =
-    useState<QueryFiltersType>(selectedFilters);
+  const [appliedFilter, setAppliedFilter] = useState<
+    QueryFiltersType | undefined
+  >(selectedFilters);

  const { getEntityPermission } = usePermissionProvider();

+  const isNumberBasedPaging = useMemo(
+    () => Boolean(appliedFilter?.team.length || appliedFilter?.user.length),
+    [appliedFilter]
+  );
+
  const fetchResourcePermission = async () => {
    if (isUndefined(selectedQuery)) {
      return;
@ -165,6 +184,45 @@ const TableQueries: FC<TableQueriesProp> = ({
      showErrorToast(error as AxiosError);
    }
  };
+  const fetchTableQuery = async (
+    params?: ListQueriesParams,
+    activePage?: number
+  ) => {
+    setIsLoading((pre) => ({ ...pre, query: true }));
+    try {
+      const queries = await getQueriesList({
+        ...params,
+        limit: PAGE_SIZE,
+        entityId: tableId,
+        fields: 'owner,votes,tags,queryUsedIn',
+      });
+      if (queries.data.length === 0) {
+        setIsError((pre) => ({ ...pre, page: true }));
+      } else {
+        setTableQueries(queries);
+        const selectedQueryData = searchParams.query
+          ? queries.data.find((query) => query.id === searchParams.query) ||
+            queries.data[0]
+          : queries.data[0];
+        setSelectedQuery(selectedQueryData);
+
+        history.push({
+          search: stringifySearchParams({
+            tableId,
+            after: params?.after,
+            query: selectedQueryData.id,
+            queryFrom: activePage,
+          }),
+        });
+      }
+    } catch (error) {
+      showErrorToast(error as AxiosError);
+      setIsError((pre) => ({ ...pre, page: true }));
+    } finally {
+      setIsLoading((pre) => ({ ...pre, query: false }));
+    }
+  };
+
  const fetchFilterData = async (value?: QueryFiltersType, page?: number) => {
    setIsLoading((pre) => ({ ...pre, query: true }));
    const allFilter = flatMap(value);
@ -196,13 +254,13 @@ const TableQueries: FC<TableQueriesProp> = ({
        : queries[0];

      setSelectedQuery(selectedQueryData);
+
      history.push({
-        search: Qs.stringify({
-          ...searchParams,
+        search: stringifySearchParams({
          ...value,
-          page: pageNumber,
          tableId,
          query: selectedQueryData.id,
+          queryFrom: pageNumber,
        }),
      });
      if (queries.length === 0) {
@ -215,10 +273,14 @@ const TableQueries: FC<TableQueriesProp> = ({
    }
  };

-  const pagingHandler = (cursorType: string | number) => {
+  const pagingHandler = (cursorType: string | number, activePage?: number) => {
+    const { paging } = tableQueries;
    if (isNumber(cursorType)) {
      setCurrentPage(cursorType);
      fetchFilterData(appliedFilter, cursorType);
+    } else {
+      fetchTableQuery({ [cursorType]: paging[cursorType] }, activePage);
+      activePage && setCurrentPage(activePage);
    }
  };

@ -238,20 +300,27 @@ const TableQueries: FC<TableQueriesProp> = ({
  useEffect(() => {
    setIsLoading((pre) => ({ ...pre, page: true }));
    if (tableId && !isTableDeleted) {
-      fetchFilterData(selectedFilters, Number(searchParams.page)).finally(
-        () => {
-          setIsLoading((pre) => ({ ...pre, page: false }));
-        }
-      );
+      const initialFetch = selectedFilters
+        ? fetchFilterData(selectedFilters, Number(searchParams.queryFrom))
+        : fetchTableQuery({ after: searchParams?.after });
+      initialFetch.finally(() => {
+        setIsLoading((pre) => ({ ...pre, page: false }));
+      });
    } else {
      setIsLoading((pre) => ({ ...pre, page: false }));
    }
  }, [tableId]);

  const onOwnerFilterChange = (value: QueryFiltersType) => {
+    const { team, user } = value;
+
    setIsError((pre) => ({ ...pre, search: false }));
    setAppliedFilter(value);
-    fetchFilterData(value, INITIAL_PAGING_VALUE);
+    if (team.length || user.length) {
+      fetchFilterData(value, INITIAL_PAGING_VALUE);
+    } else {
+      fetchTableQuery();
+    }
  };

  if (isLoading.page) {
@ -265,6 +334,30 @@ const TableQueries: FC<TableQueriesProp> = ({
    );
  }

+  const queryTabBody = isError.search ? (
+    <Col className="flex-center font-medium" data-testid="no-queries" span={24}>
+      <ErrorPlaceHolder
+        heading={t('label.query-lowercase-plural')}
+        type={ERROR_PLACEHOLDER_TYPE.VIEW}
+      />
+    </Col>
+  ) : (
+    tableQueries.data.map((query) => (
+      <Col key={query.id} span={24}>
+        <QueryCard
+          isExpanded={false}
+          permission={queryPermissions}
+          query={query}
+          selectedId={selectedQuery?.id}
+          tableId={tableId}
+          onQuerySelection={handleSelectedQuery}
+          onQueryUpdate={handleQueryUpdate}
+          onUpdateVote={updateVote}
+        />
+      </Col>
+    ))
+  );
+
  return (
    <Row className="h-full" id="tablequeries">
      <Col span={18}>
@ -276,39 +369,13 @@ const TableQueries: FC<TableQueriesProp> = ({
            <QueryFilters onFilterChange={onOwnerFilterChange} />
          </Col>

-          {isLoading.query ? (
-            <Loader />
-          ) : isError.search ? (
-            <Col
-              className="flex-center font-medium"
-              data-testid="no-queries"
-              span={24}>
-              <ErrorPlaceHolder
-                heading={t('label.query-lowercase-plural')}
-                type={ERROR_PLACEHOLDER_TYPE.VIEW}
-              />
-            </Col>
-          ) : (
-            tableQueries.data.map((query) => (
-              <Col key={query.id} span={24}>
-                <QueryCard
-                  isExpanded={false}
-                  permission={queryPermissions}
-                  query={query}
-                  selectedId={selectedQuery?.id}
-                  tableId={tableId}
-                  onQuerySelection={handleSelectedQuery}
-                  onQueryUpdate={handleQueryUpdate}
-                  onUpdateVote={updateVote}
-                />
-              </Col>
-            ))
-          )}
+          {isLoading.query ? <Loader /> : queryTabBody}
+
          <Col span={24}>
            {tableQueries.paging.total > PAGE_SIZE && (
              <NextPrevious
-                isNumberBased
                currentPage={currentPage}
+                isNumberBased={isNumberBasedPaging}
                pageSize={PAGE_SIZE}
                paging={tableQueries.paging}
                pagingHandler={pagingHandler}
--- a/openmetadata-ui/src/main/resources/ui/src/constants/Query.constant.ts
+++ b/openmetadata-ui/src/main/resources/ui/src/constants/Query.constant.ts
@ -13,7 +13,7 @@

 export const QUERY_USED_BY_TABLE_VIEW_CAP = 3;
 export const QUERY_LINE_HEIGHT = 4;
-export const QUERY_DATE_FORMAT = "'On' MMMM do 'at' h:mma 'UTC'ZZ"; // eg: On March 6th at 6:20pm UTC+1
+export const QUERY_DATE_FORMAT = "'On' MMMM dd 'at' h:mma 'UTC'ZZ"; // eg: On March 6th at 6:20pm UTC+1

 export const QUERY_PAGE_LOADING_STATE = {
  page: true,
--- a/openmetadata-ui/src/main/resources/ui/src/utils/Query/QueryUtils.ts
+++ b/openmetadata-ui/src/main/resources/ui/src/utils/Query/QueryUtils.ts
@ -48,3 +48,6 @@ export const parseSearchParams = (param: string) => {
    // need to typecast into QuerySearchParams as Qs.parse returns as "Qs.ParsedQs" Type object
  ) as unknown as QuerySearchParams;
 };
+export const stringifySearchParams = (param: QuerySearchParams) => {
+  return Qs.stringify(param);
+};