Explore page aggr changes (#60)

* Explore page aggr changes * Added progressive filter updates for aggregations * Fixed aggregation update on filter change * Fixed exceptions * Fixed exception
2026-01-07 21:16:45 +00:00 · 2021-08-10 10:10:08 +05:30 · 2021-08-10 10:10:08 +05:30 · 51ae16d813
commit 51ae16d813
parent 588ee163b8
12 changed files with 213 additions and 123 deletions
--- a/catalog-rest-service/src/main/resources/ui/src/components/common/facetfilter/FacetFilter.tsx
+++ b/catalog-rest-service/src/main/resources/ui/src/components/common/facetfilter/FacetFilter.tsx
@ -143,7 +143,6 @@ FacetFilter.propTypes = {
  onSelectHandler: PropTypes.func.isRequired,
  filters: PropTypes.shape({
    tags: PropTypes.array.isRequired,
-    service: PropTypes.array.isRequired,
    'service type': PropTypes.array.isRequired,
    tier: PropTypes.array.isRequired,
  }).isRequired,
--- a/catalog-rest-service/src/main/resources/ui/src/components/common/facetfilter/FilterContainer.tsx
+++ b/catalog-rest-service/src/main/resources/ui/src/components/common/facetfilter/FilterContainer.tsx
@ -26,14 +26,13 @@ const FilterContainer: FunctionComponent<FilterContainerProp> = ({
  type,
 }: FilterContainerProp) => {
  return (
-    <div
-      className={`filter-group tw-mb-2 ${count > 0 ? '' : 'tw-text-gray-500'}`}>
+    <div className="filter-group tw-mb-2">
      <input
        checked={isSelected}
        className="mr-1"
-        disabled={count > 0 ? false : true}
+        // disabled={count > 0 ? false : true}
        type="checkbox"
-        onClick={() => {
+        onChange={() => {
          onSelect(!isSelected, name, type);
        }}
      />
--- a/catalog-rest-service/src/main/resources/ui/src/components/common/table-data-card/TableDataCard.tsx
+++ b/catalog-rest-service/src/main/resources/ui/src/components/common/table-data-card/TableDataCard.tsx
@ -28,7 +28,7 @@ type Props = {
  tableType?: string;
  tier?: string;
  usage?: number;
-  service?: string;
+  serviceType?: string;
  fullyQualifiedName: string;
  tags?: string[];
 };
@ -40,7 +40,7 @@ const TableDataCard: FunctionComponent<Props> = ({
  tableType,
  tier = 'No Tier',
  usage,
-  service,
+  serviceType,
  fullyQualifiedName,
  tags,
 }: Props) => {
@ -48,7 +48,7 @@ const TableDataCard: FunctionComponent<Props> = ({
  const badgeName = getBadgeName(tableType);
  const OtherDetails = [
    { key: 'Owner', value: owner },
-    { key: 'Service', value: service },
+    { key: 'Service Type', value: serviceType },
    { key: 'Usage', value: percentile },
    { key: 'Tier', value: tier },
  ];
--- a/catalog-rest-service/src/main/resources/ui/src/components/searched-data/SearchedData.tsx
+++ b/catalog-rest-service/src/main/resources/ui/src/components/searched-data/SearchedData.tsx
@ -52,50 +52,58 @@ const SearchedData: React.FC<SearchedDataProp> = ({
 }) => {
  return (
    <>
-      {isLoading ? (
-        <Loader />
-      ) : totalValue > 0 || showOnboardingTemplate ? (
-        <PageContainer leftPanelContent={fetchLeftPanel && fetchLeftPanel()}>
-          <div className="container-fluid" data-testid="fluid-container">
-            {children}
-            {showResultCount && searchText ? (
-              <div className="tw-mb-1">{pluralize(totalValue, 'result')}</div>
-            ) : null}
-            {data.length > 0 ? (
-              <div className="tw-grid tw-grid-rows-1 tw-grid-cols-1">
-                {data.map((table, index) => (
-                  <div className="tw-mb-3" key={index}>
-                    <TableDataCard
-                      description={table.description}
-                      fullyQualifiedName={table.fullyQualifiedName}
-                      name={table.name}
-                      owner={table.tableEntity.owner?.name}
-                      service={table.service || '--'}
-                      tableType={table.tableType}
-                      tags={table.tags}
-                      tier={table.tier?.split('.')[1]}
-                      usage={table.weeklyStats}
-                    />
-                  </div>
-                ))}
+      <PageContainer leftPanelContent={fetchLeftPanel && fetchLeftPanel()}>
+        <div className="container-fluid" data-testid="fluid-container">
+          {isLoading ? (
+            <Loader />
+          ) : (
+            <>
+              {totalValue > 0 || showOnboardingTemplate ? (
+                <>
+                  {children}
+                  {showResultCount && searchText ? (
+                    <div className="tw-mb-1">
+                      {pluralize(totalValue, 'result')}
+                    </div>
+                  ) : null}
+                  {data.length > 0 ? (
+                    <div className="tw-grid tw-grid-rows-1 tw-grid-cols-1">
+                      {data.map((table, index) => (
+                        <div className="tw-mb-3" key={index}>
+                          <TableDataCard
+                            description={table.description}
+                            fullyQualifiedName={table.fullyQualifiedName}
+                            name={table.name}
+                            owner={table.tableEntity.owner?.name}
+                            serviceType={table.serviceType || '--'}
+                            tableType={table.tableType}
+                            tags={table.tags}
+                            tier={table.tier?.split('.')[1]}
+                            usage={table.weeklyStats}
+                          />
+                        </div>
+                      ))}

-                {totalValue > 0 && data.length > 0 && (
-                  <Pagination
-                    currentPage={currentPage}
-                    paginate={paginate}
-                    sizePerPage={PAGE_SIZE}
-                    totalNumberOfValues={totalValue}
-                  />
-                )}
-              </div>
-            ) : (
-              <Onboarding />
-            )}
-          </div>
-        </PageContainer>
-      ) : (
-        <ErrorPlaceHolder />
-      )}
+                      {totalValue > 0 && data.length > 0 && (
+                        <Pagination
+                          currentPage={currentPage}
+                          paginate={paginate}
+                          sizePerPage={PAGE_SIZE}
+                          totalNumberOfValues={totalValue}
+                        />
+                      )}
+                    </div>
+                  ) : (
+                    <Onboarding />
+                  )}
+                </>
+              ) : (
+                <ErrorPlaceHolder />
+              )}
+            </>
+          )}
+        </div>
+      </PageContainer>
    </>
  );
 };
--- a/catalog-rest-service/src/main/resources/ui/src/interface/types.d.ts
+++ b/catalog-rest-service/src/main/resources/ui/src/interface/types.d.ts
@ -23,7 +23,7 @@ declare module 'Models' {
  };
  export type FilterObject = {
    tags: Array<string>;
-    service: Array<string>;
+    // service: Array<string>;
    'service type': Array<string>;
    tier: Array<string>;
  };
@ -197,6 +197,7 @@ declare module 'Models' {
    dailyStats: number;
    weeklyStats: number;
    service?: string;
+    serviceType?: string;
    tier: string;
  };

@ -277,7 +278,7 @@ declare module 'Models' {
        };
        hits: Array<SearchHit>;
      };
-      aggregations: Aggregation;
+      aggregations: Record<string, Sterm>;
    };
  };
 }
--- a/catalog-rest-service/src/main/resources/ui/src/pages/explore/explore.constants.ts
+++ b/catalog-rest-service/src/main/resources/ui/src/pages/explore/explore.constants.ts
@ -15,6 +15,7 @@
  * limitations under the License.
 */

+import { lowerCase } from 'lodash';
 import { AggregationType, Bucket } from 'Models';
 import { tiers } from '../../constants/constants';

@ -34,18 +35,21 @@ export const getBucketList = (buckets: Array<Bucket>) => {
 };

 export const getAggrWithDefaultValue = (
-  aggregations: Array<AggregationType>
+  aggregations: Array<AggregationType>,
+  visibleAgg: Array<string> = []
 ) => {
  const aggregation = aggregations.find(
    (aggregation) => aggregation.title === 'Tier'
  );

+  const allowedAgg = visibleAgg.map((item) => lowerCase(item));
+
  if (aggregation) {
    const index = aggregations.indexOf(aggregation);
    aggregations[index].buckets = getBucketList(aggregations[index].buckets);
-
-    return aggregations;
-  } else {
-    return aggregations;
  }
+
+  return !allowedAgg.length
+    ? aggregations
+    : aggregations.filter((item) => allowedAgg.includes(lowerCase(item.title)));
 };
--- a/catalog-rest-service/src/main/resources/ui/src/pages/explore/index.tsx
+++ b/catalog-rest-service/src/main/resources/ui/src/pages/explore/index.tsx
@ -16,13 +16,14 @@
 */

 import { AxiosError } from 'axios';
+import { cloneDeep } from 'lodash';
 import {
  AggregationType,
  FilterObject,
  FormatedTableData,
  SearchResponse,
 } from 'Models';
-import React, { useEffect, useState } from 'react';
+import React, { useCallback, useEffect, useRef, useState } from 'react';
 import { useLocation, useParams } from 'react-router-dom';
 import { searchData } from '../../axiosAPIs/miscAPI';
 import Error from '../../components/common/error/Error';
@ -36,7 +37,7 @@ import { getFilterString } from '../../utils/FilterUtils';
 import { getAggrWithDefaultValue } from './explore.constants';
 import { Params } from './explore.interface';

-const visibleFilters = ['tags', 'service', 'service type', 'tier'];
+const visibleFilters = ['tags', 'service type', 'tier'];

 const getQueryParam = (urlSearchQuery = ''): FilterObject => {
  const arrSearchQuery = urlSearchQuery
@ -60,7 +61,7 @@ const ExplorePage: React.FC = (): React.ReactElement => {
  const location = useLocation();

  const filterObject: FilterObject = {
-    ...{ tags: [], service: [], 'service type': [], tier: [] },
+    ...{ tags: [], 'service type': [], tier: [] },
    ...getQueryParam(location.search),
  };
  const showToast = useToastContext();
@ -70,10 +71,12 @@ const ExplorePage: React.FC = (): React.ReactElement => {
  const [filters, setFilters] = useState<FilterObject>(filterObject);
  const [currentPage, setCurrentPage] = useState<number>(1);
  const [totalNumberOfValue, setTotalNumberOfValues] = useState<number>(0);
-  const [aggregations, setAggregation] = useState<Array<AggregationType>>([]);
+  const [aggregations, setAggregations] = useState<Array<AggregationType>>([]);
  const [isLoading, setIsLoading] = useState<boolean>(true);
  const [searchTag, setSearchTag] = useState<string>(location.search);
  const [error, setError] = useState<string>('');
+  const isMounting = useRef(true);
+
  const handleSelectedFilter = (
    checked: boolean,
    selectedFilter: string,
@ -105,21 +108,105 @@ const ExplorePage: React.FC = (): React.ReactElement => {
    setCurrentPage(pageNumber);
  };

-  const fetchTableData = () => {
+  const updateAggregationCount = useCallback(
+    (newAggregations: Array<AggregationType>) => {
+      const oldAggs = cloneDeep(aggregations);
+      for (const newAgg of newAggregations) {
+        for (const oldAgg of oldAggs) {
+          if (newAgg.title === oldAgg.title) {
+            for (const oldBucket of oldAgg.buckets) {
+              let docCount = 0;
+              for (const newBucket of newAgg.buckets) {
+                if (newBucket.key === oldBucket.key) {
+                  docCount = newBucket.doc_count;
+
+                  break;
+                }
+              }
+              // eslint-disable-next-line @typescript-eslint/camelcase
+              oldBucket.doc_count = docCount;
+            }
+          }
+        }
+      }
+      setAggregations(oldAggs);
+    },
+    [aggregations, filters]
+  );
+
+  const updateSearchResults = (res: SearchResponse) => {
+    const hits = res.data.hits.hits;
+    if (hits.length > 0) {
+      setTotalNumberOfValues(res.data.hits.total.value);
+      setData(formatDataResponse(hits));
+    } else {
+      setData([]);
+      setTotalNumberOfValues(0);
+    }
+  };
+
+  const fetchTableData = (forceSetAgg: boolean) => {
    setIsLoading(true);
-    searchData(searchText, currentPage, PAGE_SIZE, getFilterString(filters))
-      .then((res: SearchResponse) => {
-        const hits = res.data.hits.hits;
-        if (hits.length > 0) {
-          setTotalNumberOfValues(res.data.hits.total.value);
-          setData(formatDataResponse(hits));
-          setIsLoading(false);
-        } else {
-          setData([]);
-          setTotalNumberOfValues(0);
+
+    const searchResults = searchData(
+      searchText,
+      currentPage,
+      PAGE_SIZE,
+      getFilterString(filters)
+    );
+    const serviceTypeAgg = searchData(
+      searchText,
+      currentPage,
+      PAGE_SIZE,
+      getFilterString(filters, ['service type'])
+    );
+    const tierAgg = searchData(
+      searchText,
+      currentPage,
+      PAGE_SIZE,
+      getFilterString(filters, ['tier'])
+    );
+    const tagAgg = searchData(
+      searchText,
+      currentPage,
+      PAGE_SIZE,
+      getFilterString(filters, ['tags'])
+    );
+
+    Promise.all([searchResults, serviceTypeAgg, tierAgg, tagAgg])
+      .then(
+        ([
+          resSearchResults,
+          resAggServiceType,
+          resAggTier,
+          resAggTag,
+        ]: Array<SearchResponse>) => {
+          updateSearchResults(resSearchResults);
+          if (forceSetAgg) {
+            setAggregations(
+              resSearchResults.data.hits.hits.length > 0
+                ? getAggregationList(resSearchResults.data.aggregations)
+                : []
+            );
+          } else {
+            const aggServiceType = getAggregationList(
+              resAggServiceType.data.aggregations,
+              'service type'
+            );
+            const aggTier = getAggregationList(
+              resAggTier.data.aggregations,
+              'tier'
+            );
+            const aggTag = getAggregationList(
+              resAggTag.data.aggregations,
+              'tags'
+            );
+
+            updateAggregationCount([...aggServiceType, ...aggTier, ...aggTag]);
+          }
          setIsLoading(false);
        }
-      })
+      )
      .catch((err: AxiosError) => {
        setError(ERROR404);
        showToast({
@ -131,26 +218,6 @@ const ExplorePage: React.FC = (): React.ReactElement => {
      });
  };

-  const fetchAggregationData = () => {
-    setIsLoading(true);
-    searchData('*', 1, PAGE_SIZE, '')
-      .then((res: SearchResponse) => {
-        const hits = res.data.hits.hits;
-        if (hits.length > 0) {
-          setAggregation(getAggregationList(res.data.aggregations));
-        } else {
-          setAggregation([]);
-        }
-      })
-      .catch((err: AxiosError) => {
-        // setError(ERROR404);
-        showToast({
-          variant: 'error',
-          body: err.response?.data?.responseMessage ?? ERROR500,
-        });
-      });
-  };
-
  const getFacetedFilter = () => {
    const facetFilters: FilterObject = filterObject;
    for (const key in filters) {
@ -175,28 +242,24 @@ const ExplorePage: React.FC = (): React.ReactElement => {
  }, [searchText, filters]);

  useEffect(() => {
-    fetchAggregationData();
-  }, []);
+    fetchTableData(true);
+  }, [searchText]);

  useEffect(() => {
-    fetchTableData();
-  }, [searchText, currentPage, filters]);
-
-  useEffect(() => {
-    if (location.search) {
-      setFilters({
-        ...filterObject,
-        ...getQueryParam(location.search),
-      });
-    } else {
-      setFilters(filterObject);
+    if (!isMounting.current) {
+      fetchTableData(false);
    }
-  }, [location.search]);
+  }, [currentPage, filters]);
+
+  // alwyas Keep this useEffect at the end...
+  useEffect(() => {
+    isMounting.current = false;
+  }, []);

  const fetchLeftPanel = () => {
    return (
      <FacetFilter
-        aggregations={getAggrWithDefaultValue(aggregations)}
+        aggregations={getAggrWithDefaultValue(aggregations, visibleFilters)}
        filters={getFacetedFilter()}
        onSelectHandler={handleSelectedFilter}
      />
--- a/catalog-rest-service/src/main/resources/ui/src/pages/my-data/index.tsx
+++ b/catalog-rest-service/src/main/resources/ui/src/pages/my-data/index.tsx
@ -45,7 +45,7 @@ const MyDataPage: React.FC = (): React.ReactElement => {
  const fetchTableData = () => {
    setIsLoading(true);
    searchData(
-      `*`,
+      '',
      currentPage,
      PAGE_SIZE,
      filter ? `${filter}:${getCurrentUserId()}` : ''
--- a/catalog-rest-service/src/main/resources/ui/src/utils/APIUtils.js
+++ b/catalog-rest-service/src/main/resources/ui/src/utils/APIUtils.js
@ -14,6 +14,7 @@ export const formatDataResponse = (hits) => {
    newData.tags = hit._source.tags;
    newData.dailyStats = hit._source.daily_stats;
    newData.service = hit._source.service;
+    newData.serviceType = hit._source.service_type;
    newData.tableEntity = hit._source.table_entity;
    newData.tier = hit._source.tier;

--- a/catalog-rest-service/src/main/resources/ui/src/utils/AggregationUtils.js
+++ b/catalog-rest-service/src/main/resources/ui/src/utils/AggregationUtils.js
@ -1,12 +0,0 @@
-export const getAggregationList = (aggregation) => {
-  const aggrEntriesArr = Object.entries(aggregation);
-  const aggregationList = [];
-  aggrEntriesArr.forEach((aggr) => {
-    aggregationList.push({
-      title: aggr[0].substring(aggr[0].indexOf('#') + 1),
-      buckets: aggr[1].buckets,
-    });
-  });
-
-  return aggregationList;
-};
--- a/catalog-rest-service/src/main/resources/ui/src/utils/AggregationUtils.ts
+++ b/catalog-rest-service/src/main/resources/ui/src/utils/AggregationUtils.ts
@ -0,0 +1,24 @@
+import { lowerCase } from 'lodash';
+import { AggregationType, Sterm } from 'Models';
+
+export const getAggregationList = (
+  aggregation: Record<string, Sterm>,
+  aggregationType = ''
+): Array<AggregationType> => {
+  const aggrEntriesArr = Object.entries(aggregation);
+  const aggregationList: Array<AggregationType> = [];
+  aggrEntriesArr.forEach((aggr) => {
+    const aggrTitle = aggr[0].substring(aggr[0].indexOf('#') + 1);
+    if (
+      !aggregationType ||
+      lowerCase(aggrTitle) === lowerCase(aggregationType)
+    ) {
+      aggregationList.push({
+        title: aggrTitle,
+        buckets: aggr[1].buckets,
+      });
+    }
+  });
+
+  return aggregationList;
+};
--- a/catalog-rest-service/src/main/resources/ui/src/utils/FilterUtils.js
+++ b/catalog-rest-service/src/main/resources/ui/src/utils/FilterUtils.js
@ -1,6 +1,9 @@
-export const getFilterString = (filters) => {
+export const getFilterString = (filters, excludeFilters = []) => {
  const modifiedFilters = {};
  for (const key in filters) {
+    if (excludeFilters.includes(key)) {
+      continue;
+    }
    const modifiedFilter = [];
    const filter = filters[key];
    filter.forEach((value) => {