refactor: replace Container with List, update DatasetCard z-index, and implement useDatasetList for data fetching

web/app/(commonLayout)/datasets/page.tsx

@@ -1,7 +1,7 @@
-import Container from '../../components/datasets/list/container'
+import List from '../../components/datasets/list'
 
-const AppList = async () => {
-  return <Container />
+const DatasetList = async () => {
+  return <List />
 }
 
-export default AppList
+export default DatasetList

web/app/components/datasets/list/container.tsx

@@ -1,146 +0,0 @@
-'use client'
-
-// Libraries
-import { useEffect, useMemo, useRef, useState } from 'react'
-import { useRouter } from 'next/navigation'
-import { useTranslation } from 'react-i18next'
-import { useBoolean, useDebounceFn } from 'ahooks'
-import { useQuery } from '@tanstack/react-query'
-
-// Components
-import ExternalAPIPanel from '../external-api/external-api-panel'
-import Datasets from './datasets'
-import DatasetFooter from './dataset-footer'
-import ApiServer from '../../develop/ApiServer'
-import Doc from './doc'
-import SegmentedControl from '@/app/components/base/segmented-control'
-import { RiBook2Line, RiTerminalBoxLine } from '@remixicon/react'
-import TagManagementModal from '@/app/components/base/tag-management'
-import TagFilter from '@/app/components/base/tag-management/filter'
-import Button from '@/app/components/base/button'
-import Input from '@/app/components/base/input'
-import { ApiConnectionMod } from '@/app/components/base/icons/src/vender/solid/development'
-import CheckboxWithLabel from '@/app/components/datasets/create/website/base/checkbox-with-label'
-
-// Services
-import { fetchDatasetApiBaseUrl } from '@/service/datasets'
-
-// Hooks
-import { useTabSearchParams } from '@/hooks/use-tab-searchparams'
-import { useStore as useTagStore } from '@/app/components/base/tag-management/store'
-import { useAppContext } from '@/context/app-context'
-import { useExternalApiPanel } from '@/context/external-api-panel-context'
-
-const Container = () => {
-  const { t } = useTranslation()
-  const router = useRouter()
-  const { currentWorkspace, isCurrentWorkspaceOwner } = useAppContext()
-  const showTagManagementModal = useTagStore(s => s.showTagManagementModal)
-  const { showExternalApiPanel, setShowExternalApiPanel } = useExternalApiPanel()
-  const [includeAll, { toggle: toggleIncludeAll }] = useBoolean(false)
-
-  document.title = `${t('dataset.knowledge')} - Dify`
-
-  const options = useMemo(() => {
-    return [
-      { value: 'dataset', text: t('dataset.datasets'), Icon: RiBook2Line },
-      ...(currentWorkspace.role === 'dataset_operator' ? [] : [{
-        value: 'api', text: t('dataset.datasetsApi'), Icon: RiTerminalBoxLine,
-      }]),
-    ]
-  }, [currentWorkspace.role, t])
-
-  const [activeTab, setActiveTab] = useTabSearchParams({
-    defaultTab: 'dataset',
-  })
-  const containerRef = useRef<HTMLDivElement>(null)
-  const { data } = useQuery(
-    {
-      queryKey: ['datasetApiBaseInfo'],
-      queryFn: () => fetchDatasetApiBaseUrl('/datasets/api-base-info'),
-      enabled: activeTab !== 'dataset',
-    },
-  )
-
-  const [keywords, setKeywords] = useState('')
-  const [searchKeywords, setSearchKeywords] = useState('')
-  const { run: handleSearch } = useDebounceFn(() => {
-    setSearchKeywords(keywords)
-  }, { wait: 500 })
-  const handleKeywordsChange = (value: string) => {
-    setKeywords(value)
-    handleSearch()
-  }
-  const [tagFilterValue, setTagFilterValue] = useState<string[]>([])
-  const [tagIDs, setTagIDs] = useState<string[]>([])
-  const { run: handleTagsUpdate } = useDebounceFn(() => {
-    setTagIDs(tagFilterValue)
-  }, { wait: 500 })
-  const handleTagsChange = (value: string[]) => {
-    setTagFilterValue(value)
-    handleTagsUpdate()
-  }
-
-  useEffect(() => {
-    if (currentWorkspace.role === 'normal')
-      return router.replace('/apps')
-  }, [currentWorkspace, router])
-
-  return (
-    <div ref={containerRef} className='scroll-container relative flex grow flex-col overflow-y-auto bg-background-body'>
-      <div className='sticky top-0 z-10 flex items-center justify-between gap-x-1 bg-background-body px-12 pb-2 pt-4'>
-        <SegmentedControl
-          value={activeTab}
-          onChange={newActiveTab => setActiveTab(newActiveTab as string)}
-          options={options}
-          size='large'
-          activeClassName='text-text-primary'
-        />
-        {activeTab === 'dataset' && (
-          <div className='flex items-center justify-center gap-2'>
-            {isCurrentWorkspaceOwner && <CheckboxWithLabel
-              isChecked={includeAll}
-              onChange={toggleIncludeAll}
-              label={t('dataset.allKnowledge')}
-              labelClassName='system-md-regular text-text-secondary'
-              className='mr-2'
-              tooltip={t('dataset.allKnowledgeDescription') as string}
-            />}
-            <TagFilter type='knowledge' value={tagFilterValue} onChange={handleTagsChange} />
-            <Input
-              showLeftIcon
-              showClearIcon
-              wrapperClassName='w-[200px]'
-              value={keywords}
-              onChange={e => handleKeywordsChange(e.target.value)}
-              onClear={() => handleKeywordsChange('')}
-            />
-            <div className='h-4 w-[1px] bg-divider-regular' />
-            <Button
-              className='shadows-shadow-xs gap-0.5'
-              onClick={() => setShowExternalApiPanel(true)}
-            >
-              <ApiConnectionMod className='h-4 w-4 text-components-button-secondary-text' />
-              <div className='system-sm-medium flex items-center justify-center gap-1 px-0.5 text-components-button-secondary-text'>{t('dataset.externalAPIPanelTitle')}</div>
-            </Button>
-          </div>
-        )}
-        {activeTab === 'api' && data && <ApiServer apiBaseUrl={data.api_base_url || ''} />}
-      </div>
-      {activeTab === 'dataset' && (
-        <>
-          <Datasets containerRef={containerRef} tags={tagIDs} keywords={searchKeywords} includeAll={includeAll} />
-          <DatasetFooter />
-          {showTagManagementModal && (
-            <TagManagementModal type='knowledge' show={showTagManagementModal} />
-          )}
-        </>
-      )}
-      {activeTab === 'api' && data && <Doc apiBaseUrl={data.api_base_url || ''} />}
-
-      {showExternalApiPanel && <ExternalAPIPanel onClose={() => setShowExternalApiPanel(false)} />}
-    </div>
-  )
-}
-
-export default Container

web/app/components/datasets/list/dataset-card/index.tsx

@@ -133,7 +133,7 @@ const DatasetCard = ({
               background={iconInfo.icon_type === 'image' ? undefined : iconInfo.icon_background}
               imageUrl={iconInfo.icon_type === 'image' ? iconInfo.icon_url : undefined}
             />
-            <div className='absolute -bottom-1 -right-1 z-10'>
+            <div className='absolute -bottom-1 -right-1 z-[5]'>
               <Icon className='size-4' />
             </div>
           </div>
@@ -189,7 +189,7 @@ const DatasetCard = ({
           {/* Tag Mask */}
           <div
             className={cn(
-              'absolute right-0 top-0 z-10 h-full w-20 bg-tag-selector-mask-bg group-hover:bg-tag-selector-mask-hover-bg',
+              'absolute right-0 top-0 z-[5] h-full w-20 bg-tag-selector-mask-bg group-hover:bg-tag-selector-mask-hover-bg',
               isHoveringTagSelector && 'hidden',
             )}
           />
@@ -202,49 +202,49 @@ const DatasetCard = ({
         >
           <Tooltip popupContent={documentCountTooltip} >
             <div className='flex items-center gap-x-1'>
-              <RiFileTextFill className='size-3 text-text-quaternary'/>
-                <span className='system-xs-medium'>{documentCount}</span>
+              <RiFileTextFill className='size-3 text-text-quaternary' />
+              <span className='system-xs-medium'>{documentCount}</span>
             </div>
           </Tooltip>
           {!isExternalProvider && (
             <Tooltip popupContent={`${dataset.app_count} ${t('dataset.appCount')}`}>
               <div className='flex items-center gap-x-1'>
-                <RiRobot2Fill className='size-3 text-text-quaternary'/>
-                  <span className='system-xs-medium'>{dataset.app_count}</span>
+                <RiRobot2Fill className='size-3 text-text-quaternary' />
+                <span className='system-xs-medium'>{dataset.app_count}</span>
               </div>
             </Tooltip>
           )}
           <span className='system-xs-regular text-divider-deep'>/</span>
           <span className='system-xs-regular'>{`${t('dataset.updated')} ${formatTimeFromNow(dataset.updated_at)}`}</span>
         </div>
-        <div className='absolute right-2 top-2 z-20 hidden group-hover:block'>
-            <CustomPopover
-              htmlContent={
-                <Operations
-                  showDelete={!isCurrentWorkspaceDatasetOperator}
-                  openRenameModal={() => {
-                    setShowRenameModal(true)
-                  }}
-                  detectIsUsedByApp={detectIsUsedByApp}
-                />
-              }
-              className={'z-20 min-w-[186px]'}
-              popupClassName={'rounded-xl bg-none shadow-none ring-0 min-w-[186px]'}
-              position='br'
-              trigger='click'
-              btnElement={
-                <div className='flex size-8 items-center justify-center rounded-[10px] hover:bg-state-base-hover'>
-                  <RiMoreFill className='h-5 w-5 text-text-tertiary' />
-                </div>
-              }
-              btnClassName={open =>
-                cn(
-                  'size-9 cursor-pointer justify-center rounded-[10px] border-[0.5px] border-components-actionbar-border bg-components-actionbar-bg p-0 shadow-lg shadow-shadow-shadow-5 ring-[2px] ring-inset ring-components-actionbar-bg hover:border-components-actionbar-border',
-                  open ? 'border-components-actionbar-border bg-state-base-hover' : '',
-                )
-              }
-            />
-          </div>
+        <div className='absolute right-2 top-2 z-[5] hidden group-hover:block'>
+          <CustomPopover
+            htmlContent={
+              <Operations
+                showDelete={!isCurrentWorkspaceDatasetOperator}
+                openRenameModal={() => {
+                  setShowRenameModal(true)
+                }}
+                detectIsUsedByApp={detectIsUsedByApp}
+              />
+            }
+            className={'z-20 min-w-[186px]'}
+            popupClassName={'rounded-xl bg-none shadow-none ring-0 min-w-[186px]'}
+            position='br'
+            trigger='click'
+            btnElement={
+              <div className='flex size-8 items-center justify-center rounded-[10px] hover:bg-state-base-hover'>
+                <RiMoreFill className='h-5 w-5 text-text-tertiary' />
+              </div>
+            }
+            btnClassName={open =>
+              cn(
+                'size-9 cursor-pointer justify-center rounded-[10px] border-[0.5px] border-components-actionbar-border bg-components-actionbar-bg p-0 shadow-lg shadow-shadow-shadow-5 ring-[2px] ring-inset ring-components-actionbar-bg hover:border-components-actionbar-border',
+                open ? 'border-components-actionbar-border bg-state-base-hover' : '',
+              )
+            }
+          />
+        </div>
       </div>
       {showRenameModal && (
         <RenameDatasetModal

web/app/components/datasets/list/datasets.tsx

@@ -1,39 +1,11 @@
 'use client'
 
-import { useEffect, useMemo, useRef } from 'react'
-import useSWRInfinite from 'swr/infinite'
-import { debounce } from 'lodash-es'
+import { useEffect, useRef } from 'react'
 import { useTranslation } from 'react-i18next'
 import NewDatasetCard from './new-dataset-card'
 import DatasetCard from './dataset-card'
-import type { DataSetListResponse, FetchDatasetsParams } from '@/models/datasets'
-import { fetchDatasets } from '@/service/datasets'
-import { useAppContext } from '@/context/app-context'
-
-const getKey = (
-  pageIndex: number,
-  previousPageData: DataSetListResponse,
-  tags: string[],
-  keyword: string,
-  includeAll: boolean,
-) => {
-  if (!pageIndex || previousPageData.has_more) {
-    const params: FetchDatasetsParams = {
-      url: 'datasets',
-      params: {
-        page: pageIndex + 1,
-        limit: 30,
-        include_all: includeAll,
-      },
-    }
-    if (tags.length)
-      params.params.tag_ids = tags
-    if (keyword)
-      params.params.keyword = keyword
-    return params
-  }
-  return null
-}
+import { useSelector as useAppContextWithSelector } from '@/context/app-context'
+import { useDatasetList, useResetDatasetList } from '@/service/knowledge/use-dataset'
 
 type Props = {
   containerRef: React.RefObject<HTMLDivElement>
@@ -43,53 +15,57 @@ type Props = {
 }
 
 const Datasets = ({
-  containerRef,
   tags,
   keywords,
   includeAll,
 }: Props) => {
   const { t } = useTranslation()
-  const { isCurrentWorkspaceEditor } = useAppContext()
-  const { data, isLoading, setSize, mutate } = useSWRInfinite(
-    (pageIndex: number, previousPageData: DataSetListResponse) => getKey(pageIndex, previousPageData, tags, keywords, includeAll),
-    fetchDatasets,
-    { revalidateFirstPage: false, revalidateAll: true },
-  )
+  const isCurrentWorkspaceEditor = useAppContextWithSelector(state => state.isCurrentWorkspaceEditor)
+  const {
+    data,
+    fetchNextPage,
+    hasNextPage,
+    isFetching,
+  } = useDatasetList({
+    initialPage: 1,
+    tag_ids: tags,
+    limit: 20,
+    include_all: includeAll,
+    keyword: keywords,
+  })
+  const resetDatasetList = useResetDatasetList()
   const loadingStateRef = useRef(false)
-  const anchorRef = useRef<HTMLAnchorElement>(null)
+  const anchorRef = useRef<HTMLDivElement>(null)
+  const observerRef = useRef<IntersectionObserver>()
 
   useEffect(() => {
-    loadingStateRef.current = isLoading
+    loadingStateRef.current = isFetching
     document.title = `${t('dataset.knowledge')} - Dify`
-  }, [isLoading, t])
-
-  const onScroll = useMemo(() => {
-    return debounce(() => {
-      if (!loadingStateRef.current && containerRef.current && anchorRef.current) {
-        const { scrollTop, clientHeight } = containerRef.current
-        const anchorOffset = anchorRef.current.offsetTop
-        if (anchorOffset - scrollTop - clientHeight < 100)
-          setSize(size => size + 1)
-      }
-    }, 50)
-  }, [containerRef, setSize])
+  }, [isFetching, t])
 
   useEffect(() => {
-    const currentContainer = containerRef.current
-    currentContainer?.addEventListener('scroll', onScroll)
-    return () => {
-      currentContainer?.removeEventListener('scroll', onScroll)
-      onScroll.cancel()
+    if (anchorRef.current) {
+      observerRef.current = new IntersectionObserver((entries) => {
+        if (entries[0].isIntersecting && hasNextPage)
+          fetchNextPage()
+      }, {
+        rootMargin: '100px',
+      })
+      observerRef.current.observe(anchorRef.current)
     }
-  }, [onScroll, containerRef])
+    return () => observerRef.current?.disconnect()
+  }, [anchorRef, data, hasNextPage, fetchNextPage])
 
   return (
-    <nav className='grid shrink-0 grow grid-cols-1 content-start gap-3 px-12 pt-2 sm:grid-cols-2 md:grid-cols-3 lg:grid-cols-4'>
-      { isCurrentWorkspaceEditor && <NewDatasetCard ref={anchorRef} /> }
-      {data?.map(({ data: datasets }) => datasets.map(dataset => (
-        <DatasetCard key={dataset.id} dataset={dataset} onSuccess={mutate} />),
-      ))}
-    </nav>
+    <>
+      <nav className='grid shrink-0 grow grid-cols-1 content-start gap-3 px-12 pt-2 sm:grid-cols-2 md:grid-cols-3 lg:grid-cols-4'>
+        {isCurrentWorkspaceEditor && <NewDatasetCard />}
+        {data?.pages.map(({ data: datasets }) => datasets.map(dataset => (
+          <DatasetCard key={dataset.id} dataset={dataset} onSuccess={resetDatasetList} />),
+        ))}
+      </nav>
+      <div ref={anchorRef} className='h-0' />
+    </>
   )
 }
 

web/app/components/datasets/list/index.tsx

@@ -88,7 +88,7 @@ const List = () => {
 
   return (
     <div ref={containerRef} className='scroll-container relative flex grow flex-col overflow-y-auto bg-background-body'>
-      <div className='sticky top-0 z-30 flex items-center justify-between gap-x-1 bg-background-body px-12 pb-2 pt-4'>
+      <div className='sticky top-0 z-10 flex items-center justify-between gap-x-1 bg-background-body px-12 pb-2 pt-4'>
         <SegmentedControl
           value={activeTab}
           onChange={newActiveTab => setActiveTab(newActiveTab as string)}

web/app/components/datasets/list/new-dataset-card/index.tsx

@@ -9,21 +9,14 @@ import {
 import Link from './link'
 import { ApiConnectionMod } from '@/app/components/base/icons/src/vender/solid/development'
 
-type CreateAppCardProps = {
-  ref: React.RefObject<HTMLAnchorElement>
-}
-
-const CreateAppCard = ({
-  ref,
-  ..._
-}: CreateAppCardProps) => {
+const CreateAppCard = () => {
   const { t } = useTranslation()
 
   return (
     <div className='flex h-[166px] flex-col gap-y-0.5 rounded-xl bg-background-default-dimmed'>
       <div className='flex grow flex-col items-center justify-center p-2'>
         <Link href={`${basePath}/datasets/create-from-pipeline`} Icon={RiFunctionAddLine} text={t('dataset.createFromPipeline')} />
-        <Link ref={ref} href={`${basePath}/datasets/create`} Icon={RiAddLine} text={t('dataset.createDataset')} />
+        <Link href={`${basePath}/datasets/create`} Icon={RiAddLine} text={t('dataset.createDataset')} />
       </div>
       <div className='border-t-[0.5px] border-divider-subtle p-2'>
         <Link href={`${basePath}/datasets/connect`} Icon={ApiConnectionMod} text={t('dataset.connectDataset')} />

web/models/datasets.ts

@@ -175,6 +175,14 @@ export type FetchDatasetsParams = {
   }
 }
 
+export type DatasetListRequest = {
+  initialPage: number
+  tag_ids: string[]
+  limit: number
+  include_all: boolean
+  keyword: string
+}
+
 export type DataSetListResponse = {
   data: DataSet[]
   has_more: boolean

web/service/knowledge/use-dataset.ts

@@ -0,0 +1,32 @@
+import { useInfiniteQuery } from '@tanstack/react-query'
+import type { DataSetListResponse, DatasetListRequest } from '@/models/datasets'
+import { get } from '../base'
+import { useReset } from '../use-base'
+import qs from 'qs'
+
+const NAME_SPACE = 'dataset'
+
+const DatasetListKey = [NAME_SPACE, 'list']
+
+export const useDatasetList = (params: DatasetListRequest) => {
+  const { initialPage, tag_ids, limit, include_all, keyword } = params
+  return useInfiniteQuery({
+    queryKey: [...DatasetListKey, initialPage, tag_ids, limit, include_all, keyword],
+    queryFn: ({ pageParam = 1 }) => {
+      const urlParams = qs.stringify({
+        tag_ids,
+        limit,
+        include_all,
+        keyword,
+        page: pageParam,
+      }, { indices: false })
+      return get<DataSetListResponse>(`/datasets?${urlParams}`)
+    },
+    getNextPageParam: lastPage => lastPage.has_more ? lastPage.page + 1 : null,
+    initialPageParam: initialPage,
+  })
+}
+
+export const useResetDatasetList = () => {
+  return useReset([...DatasetListKey])
+}