feat(ui/homepage): add infinite scroll list component and implement in asset collection module (#14371)

Co-authored-by: Chris Collins <chriscollins3456@gmail.com>

datahub-web-react/src/alchemy-components/components/InfiniteScrollList/InfiniteScroll.stories.tsx

@@ -0,0 +1,141 @@
+import { BADGE } from '@geometricpanda/storybook-addon-badges';
+import type { Meta, StoryObj } from '@storybook/react';
+import React from 'react';
+import styled from 'styled-components';
+
+import { InfiniteScrollList } from '@components/components/InfiniteScrollList/InfiniteScrollList';
+import type { InfiniteScrollListProps } from '@components/components/InfiniteScrollList/types';
+
+// Sample type for list items
+type User = {
+    id: number;
+    name: string;
+};
+
+// Sample data set
+const USER_DATA: User[] = Array.from({ length: 50 }, (_, i) => ({
+    id: i + 1,
+    name: `User ${i + 1}`,
+}));
+
+// Simulated async fetch function with pagination support
+const mockFetchUsers = (start: number, count: number): Promise<User[]> => {
+    return new Promise((resolve) => {
+        setTimeout(() => {
+            resolve(USER_DATA.slice(start, start + count));
+        }, 250);
+    });
+};
+
+const ScrollContainer = styled.div`
+    height: 300px;
+    width: 300px;
+    overflow-y: auto;
+    border: 1px solid #ddd;
+    padding: 8px;
+    box-sizing: border-box;
+    display: flex;
+    flex-direction: column;
+    align-items: center;
+`;
+
+// Auto Docs
+const meta = {
+    title: 'Lists & Tables / InfiniteScrollList',
+    component: InfiniteScrollList,
+    parameters: {
+        layout: 'centered',
+        badges: [BADGE.STABLE, 'readyForDesignReview'],
+        docs: {
+            subtitle: 'Used to render an infinite scroll list',
+        },
+    },
+    argTypes: {
+        fetchData: {
+            description: 'Function to asynchronously fetch page data, given start index and count',
+            control: false,
+        },
+        renderItem: {
+            description: 'Render function for each list item',
+            control: false,
+        },
+        pageSize: {
+            description: 'Number of items to fetch per page',
+            control: { type: 'number' },
+            defaultValue: 10,
+        },
+        totalItemCount: {
+            description: 'Optional total number of items available',
+            control: { type: 'number', min: 0 },
+        },
+        emptyState: {
+            description: 'Component/UI to show when no items are returned',
+            control: false,
+        },
+    },
+    args: {
+        pageSize: 10,
+        totalItemCount: USER_DATA.length,
+        fetchData: mockFetchUsers,
+        renderItem: (user: User) => (
+            <div key={user.id} style={{ padding: 8, borderBottom: '1px solid #eee' }}>
+                {user.name}
+            </div>
+        ),
+        emptyState: <div>No users found.</div>,
+    },
+} satisfies Meta<InfiniteScrollListProps<User>>;
+
+export default meta;
+
+// Stories
+
+type Story = StoryObj<typeof meta>;
+
+export const sandbox: Story = {
+    tags: ['dev'],
+
+    render: (args) => (
+        <ScrollContainer>
+            <InfiniteScrollList {...args} />
+        </ScrollContainer>
+    ),
+};
+
+export const emptyData: Story = {
+    args: {
+        fetchData: () => Promise.resolve([]),
+        totalItemCount: 0,
+    },
+    render: (args) => (
+        <ScrollContainer>
+            <InfiniteScrollList {...args} />
+        </ScrollContainer>
+    ),
+};
+
+export const smallPageSize: Story = {
+    args: {
+        pageSize: 3,
+    },
+    render: (args) => (
+        <ScrollContainer>
+            <InfiniteScrollList {...args} />
+        </ScrollContainer>
+    ),
+};
+
+export const customRenderItem: Story = {
+    render: (args) => (
+        <ScrollContainer>
+            <InfiniteScrollList
+                {...args}
+                renderItem={(user) => (
+                    <div key={user.id} style={{ padding: 8, borderBottom: '1px solid #ccc', fontWeight: 'bold' }}>
+                        {user.name}
+                    </div>
+                )}
+            />
+        </ScrollContainer>
+    ),
+};

datahub-web-react/src/alchemy-components/components/InfiniteScrollList/InfiniteScrollList.tsx

@@ -0,0 +1,30 @@
+import React from 'react';
+
+import { ObserverContainer } from '@components/components/InfiniteScrollList/components';
+import { InfiniteScrollListProps } from '@components/components/InfiniteScrollList/types';
+import { useInfiniteScroll } from '@components/components/InfiniteScrollList/useInfiniteScroll';
+import { Loader } from '@components/components/Loader';
+
+export function InfiniteScrollList<T>({
+    fetchData,
+    renderItem,
+    pageSize,
+    emptyState,
+    totalItemCount,
+    showLoader = true,
+}: InfiniteScrollListProps<T>) {
+    const { items, loading, observerRef, hasMore } = useInfiniteScroll({
+        fetchData,
+        pageSize,
+        totalItemCount,
+    });
+
+    return (
+        <>
+            {items.length === 0 && !loading && emptyState}
+            {items.map((item) => renderItem(item))}
+            {hasMore && <ObserverContainer ref={observerRef} />}
+            {items.length > 0 && showLoader && loading && <Loader size="sm" alignItems="center" />}
+        </>
+    );
+}

datahub-web-react/src/alchemy-components/components/InfiniteScrollList/__tests__/useInfiniteScroll.test.ts

@@ -0,0 +1,154 @@
+import { act } from '@testing-library/react';
+import { renderHook } from '@testing-library/react-hooks';
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+
+import { useInfiniteScroll } from '@components/components/InfiniteScrollList/useInfiniteScroll';
+
+const flushPromises = () => new Promise(setImmediate);
+
+const mockIntersectionObserver = vi.fn((callback) => {
+    const observerInstance = {
+        observe: (element: Element) => {
+            callback([{ isIntersecting: true, target: element }], observerInstance);
+        },
+        unobserve: vi.fn(),
+        disconnect: vi.fn(),
+    };
+    return observerInstance;
+});
+global.IntersectionObserver = mockIntersectionObserver as any;
+
+describe('useInfiniteScroll hook', () => {
+    const pageSize = 3;
+
+    function createFetchDataMock(dataBatches: unknown[][]) {
+        let callCount = 0;
+        return vi.fn().mockImplementation(() => {
+            const result = dataBatches[callCount] || [];
+            callCount++;
+            return Promise.resolve(result);
+        });
+    }
+
+    beforeEach(() => {
+        vi.clearAllMocks();
+    });
+
+    it('loads initial data on mount', async () => {
+        const dataBatches = [[1, 2, 3]];
+        const fetchData = createFetchDataMock(dataBatches);
+
+        const { result, waitForNextUpdate } = renderHook(() => useInfiniteScroll({ fetchData, pageSize }));
+
+        expect(result.current.loading).toBe(true);
+        expect(result.current.items).toEqual([]);
+
+        await waitForNextUpdate();
+
+        expect(fetchData).toHaveBeenCalledWith(0, pageSize);
+        expect(result.current.items).toEqual([1, 2, 3]);
+        expect(result.current.loading).toBe(false);
+        expect(result.current.hasMore).toBe(true);
+    });
+
+    it('sets hasMore=false when less than pageSize items are fetched and no totalItemCount provided', async () => {
+        const dataBatches = [[1, 2]];
+        const fetchData = createFetchDataMock(dataBatches);
+
+        const { result, waitForNextUpdate } = renderHook(() => useInfiniteScroll({ fetchData, pageSize }));
+
+        await waitForNextUpdate();
+
+        expect(result.current.items.length).toBe(2);
+        expect(result.current.hasMore).toBe(false);
+    });
+
+    it('sets hasMore based on totalItemCount when provided', async () => {
+        const totalItemCount = 5;
+        const dataBatches = [
+            [1, 2, 3],
+            [4, 5],
+        ];
+        const fetchData = createFetchDataMock(dataBatches);
+
+        const { result, waitForNextUpdate } = renderHook(() =>
+            useInfiniteScroll({ fetchData, pageSize, totalItemCount }),
+        );
+
+        act(() => {
+            result.current.observerRef.current = document.createElement('div');
+        });
+
+        await waitForNextUpdate();
+
+        expect(result.current.items.length).toBe(5);
+        expect(result.current.hasMore).toBe(false);
+    });
+
+    it('does not load more data if already loading or hasMore is false', async () => {
+        const dataBatches = [[1, 2, 3]];
+        const fetchData = createFetchDataMock(dataBatches);
+
+        const { result, waitForNextUpdate } = renderHook(() => useInfiniteScroll({ fetchData, pageSize }));
+
+        await waitForNextUpdate();
+
+        expect(result.current.loading).toBe(false);
+        expect(result.current.hasMore).toBe(true);
+
+        act(() => {
+            result.current.observerRef.current = document.createElement('div');
+        });
+
+        await flushPromises();
+
+        expect(fetchData).toHaveBeenCalledTimes(1);
+        expect(fetchData).toHaveBeenCalledTimes(1);
+    });
+
+    it('cleans up IntersectionObserver on unmount', async () => {
+        const dataBatches = [[1, 2, 3]];
+        const fetchData = createFetchDataMock(dataBatches);
+
+        const unobserveMock = vi.fn();
+        const disconnectMock = vi.fn();
+        const observeMock = vi.fn(function (this: any, element: Element) {
+            this.callback([{ isIntersecting: true, target: element }], this);
+        });
+
+        const mockIO = vi.fn(function (this: any, callback: IntersectionObserverCallback) {
+            this.callback = callback;
+            return {
+                observe: observeMock.bind(this),
+                unobserve: unobserveMock,
+                disconnect: disconnectMock,
+            };
+        });
+
+        global.IntersectionObserver = mockIO as any;
+
+        const { unmount, result, waitForNextUpdate } = renderHook(() => useInfiniteScroll({ fetchData, pageSize }));
+
+        act(() => {
+            result.current.observerRef.current = document.createElement('div');
+        });
+
+        await waitForNextUpdate();
+
+        expect(observeMock).toHaveBeenCalled();
+
+        unmount();
+
+        expect(unobserveMock).toHaveBeenCalled();
+        expect(disconnectMock).toHaveBeenCalled();
+    });
+
+    it('exposes observerRef ref object', () => {
+        const fetchData = vi.fn(() => Promise.resolve([]));
+        const { result } = renderHook(() => useInfiniteScroll({ fetchData }));
+
+        expect(result.current.observerRef).toBeDefined();
+        expect(typeof result.current.observerRef).toBe('object');
+        expect(result.current.observerRef.current).toBeNull();
+    });
+});

datahub-web-react/src/alchemy-components/components/InfiniteScrollList/components.ts

@@ -0,0 +1,6 @@
+import styled from 'styled-components';
+
+export const ObserverContainer = styled.div`
+    height: 1px;
+    margintop: 1px;
+`;

datahub-web-react/src/alchemy-components/components/InfiniteScrollList/index.ts

@@ -0,0 +1 @@
+export { InfiniteScrollList } from './InfiniteScrollList';

datahub-web-react/src/alchemy-components/components/InfiniteScrollList/types.ts

@@ -0,0 +1,8 @@
+export interface InfiniteScrollListProps<T> {
+    fetchData: (start: number, count: number) => Promise<T[]>;
+    renderItem: (item: T) => React.ReactNode;
+    pageSize?: number;
+    emptyState?: React.ReactNode;
+    totalItemCount?: number;
+    showLoader?: boolean;
+}

datahub-web-react/src/alchemy-components/components/InfiniteScrollList/useInfiniteScroll.ts

@@ -0,0 +1,70 @@
+import { useCallback, useEffect, useRef, useState } from 'react';
+
+interface Props<T> {
+    fetchData: (start: number, count: number) => Promise<T[]>;
+    pageSize?: number;
+    totalItemCount?: number;
+}
+
+export function useInfiniteScroll<T>({ fetchData, pageSize = 10, totalItemCount }: Props<T>) {
+    const [items, setItems] = useState<T[]>([]);
+    const [loading, setLoading] = useState(false);
+    const [hasMore, setHasMore] = useState(true);
+    const start = useRef(0);
+
+    // Ref element to be observed by IntersectionObserver
+    const observerRef = useRef<HTMLDivElement | null>(null);
+
+    // Function to fetch the next data batch, invoked initially and when observer comes into view
+    const loadMore = useCallback(() => {
+        if (loading || !hasMore) return;
+
+        setLoading(true);
+
+        fetchData(start.current, pageSize)
+            .then((newItems) => {
+                // Append newly fetched items to current list
+                setItems((prev) => [...prev, ...newItems]);
+                // Advance the start index by the number of new items fetched
+                start.current += newItems.length;
+                // Update hasMore depending on totalItemCount or inferred from batch size
+                if (totalItemCount !== undefined) {
+                    setHasMore(start.current < totalItemCount);
+                } else {
+                    setHasMore(newItems.length === pageSize);
+                }
+            })
+            .finally(() => {
+                setLoading(false);
+            });
+    }, [fetchData, loading, hasMore, pageSize, totalItemCount]);
+
+    // Initial load
+    useEffect(() => {
+        if (items.length === 0 && hasMore && !loading) {
+            loadMore();
+        }
+        // eslint-disable-next-line react-hooks/exhaustive-deps
+    }, [hasMore]);
+
+    // Intersection Observer
+    useEffect(() => {
+        if (!observerRef.current || !hasMore) return undefined;
+
+        const observer = new IntersectionObserver((entries) => {
+            if (entries[0].isIntersecting) {
+                loadMore();
+            }
+        });
+
+        const currentObserverRef = observerRef.current;
+        observer.observe(currentObserverRef);
+
+        return () => {
+            observer.unobserve(currentObserverRef);
+            observer.disconnect();
+        };
+    }, [loadMore, hasMore]);
+
+    return { items, loading, observerRef, hasMore };
+}

datahub-web-react/src/alchemy-components/index.ts

@@ -43,3 +43,4 @@ export * from './components/Timeline';
 export * from './components/Tooltip';
 export * from './components/Utils';
 export * from './components/WhiskerChart';
+export * from './components/InfiniteScrollList';

datahub-web-react/src/app/homeV3/modules/assetCollection/AssetCollectionModule.tsx

@@ -1,40 +1,80 @@
-import React from 'react';
+import { InfiniteScrollList } from '@components';
+import React, { useCallback, useMemo } from 'react';
 
 import EmptyContent from '@app/homeV3/module/components/EmptyContent';
 import EntityItem from '@app/homeV3/module/components/EntityItem';
 import LargeModule from '@app/homeV3/module/components/LargeModule';
 import { ModuleProps } from '@app/homeV3/module/types';
-import { useGetEntities } from '@app/sharedV2/useGetEntities';
 
+import { useGetSearchResultsForMultipleQuery } from '@graphql/search.generated';
 import { DataHubPageModuleType, Entity } from '@types';
 
+const DEFAULT_PAGE_SIZE = 6;
+
 const AssetCollectionModule = (props: ModuleProps) => {
-    const assetUrns =
+    const assetUrns = useMemo(
+        () =>
             props.module.properties.params.assetCollectionParams?.assetUrns.filter(
                 (urn): urn is string => typeof urn === 'string',
-        ) || [];
+            ) || [],
+        [props.module.properties.params.assetCollectionParams?.assetUrns],
+    );
 
-    const { entities, loading } = useGetEntities(assetUrns, true);
+    const { loading, refetch } = useGetSearchResultsForMultipleQuery({
+        variables: {
+            input: {
+                start: 0,
+                count: DEFAULT_PAGE_SIZE,
+                query: '*',
+                filters: [{ field: 'urn', values: assetUrns }],
+            },
+        },
+        skip: assetUrns.length === 0,
+    });
+
+    const fetchEntities = useCallback(
+        async (start: number, count: number): Promise<Entity[]> => {
+            if (assetUrns.length === 0) return [];
+            // urn slicing is done at the front-end to maintain the order of assets to show with pagination
+            const urnSlice = assetUrns.slice(start, start + count);
+            const result = await refetch({
+                input: {
+                    start: 0, // Using start as 0 every time because sliced urns are sent
+                    count: urnSlice.length,
+                    query: '*',
+                    filters: [{ field: 'urn', values: urnSlice }],
+                },
+            });
+
+            const results =
+                result.data?.searchAcrossEntities?.searchResults
+                    ?.map((res) => res.entity)
+                    .filter((entity): entity is Entity => !!entity) || [];
+
+            const urnToEntity = new Map(results.map((e) => [e.urn, e]));
+            return urnSlice.map((urn) => urnToEntity.get(urn)).filter((entity): entity is Entity => !!entity);
+        },
+        [assetUrns, refetch],
+    );
 
     return (
         <LargeModule {...props} loading={loading}>
-            {entities?.length === 0 ? (
+            <InfiniteScrollList<Entity>
+                key={assetUrns.join(',')}
+                fetchData={fetchEntities}
+                renderItem={(entity) => (
+                    <EntityItem entity={entity} key={entity?.urn} moduleType={DataHubPageModuleType.AssetCollection} />
+                )}
+                pageSize={DEFAULT_PAGE_SIZE}
+                emptyState={
                     <EmptyContent
                         icon="Stack"
                         title="No Assets"
                         description="Edit the module and add assets to see them in this list"
                     />
-            ) : (
-                entities
-                    .filter((entity): entity is Entity => entity !== null)
-                    .map((entity) => (
-                        <EntityItem
-                            entity={entity}
-                            key={entity?.urn}
-                            moduleType={DataHubPageModuleType.AssetCollection}
+                }
+                totalItemCount={assetUrns.length}
             />
-                    ))
-            )}
         </LargeModule>
     );
 };