Feat: Batch operations on documents in a dataset #3221 (#7352)

### What problem does this PR solve?

Feat: Batch operations on documents in a dataset #3221

### Type of change


- [x] New Feature (non-breaking change which adds functionality)

web/src/app.tsx

@@ -1,13 +1,15 @@
+import { Toaster as Sonner } from '@/components/ui/sonner';
+import { Toaster } from '@/components/ui/toaster';
 import i18n from '@/locales/config';
 import { QueryClient, QueryClientProvider } from '@tanstack/react-query';
 import { ReactQueryDevtools } from '@tanstack/react-query-devtools';
 import { App, ConfigProvider, ConfigProviderProps, theme } from 'antd';
 import pt_BR from 'antd/lib/locale/pt_BR';
+import deDE from 'antd/locale/de_DE';
 import enUS from 'antd/locale/en_US';
 import vi_VN from 'antd/locale/vi_VN';
 import zhCN from 'antd/locale/zh_CN';
 import zh_HK from 'antd/locale/zh_HK';
-import deDE from 'antd/locale/de_DE';
 import dayjs from 'dayjs';
 import advancedFormat from 'dayjs/plugin/advancedFormat';
 import customParseFormat from 'dayjs/plugin/customParseFormat';
@@ -67,6 +69,8 @@ function Root({ children }: React.PropsWithChildren) {
         locale={locale}
       >
         <App>{children}</App>
+        <Sonner position={'top-right'} expand richColors closeButton></Sonner>
+        <Toaster />
       </ConfigProvider>
       <ReactQueryDevtools buttonPosition={'top-left'} />
     </>

web/src/components/list-filter-bar/filter-popover.tsx

@@ -4,7 +4,7 @@ import {
   PopoverTrigger,
 } from '@/components/ui/popover';
 import { zodResolver } from '@hookform/resolvers/zod';
-import { PropsWithChildren, useCallback, useEffect } from 'react';
+import { PropsWithChildren, useCallback, useEffect, useState } from 'react';
 import { useForm } from 'react-hook-form';
 import { ZodArray, ZodString, z } from 'zod';
 
@@ -24,12 +24,14 @@ export type CheckboxFormMultipleProps = {
   filters?: FilterCollection[];
   value?: FilterValue;
   onChange?: FilterChange;
+  setOpen(open: boolean): void;
 };
 
 function CheckboxFormMultiple({
   filters = [],
   value,
   onChange,
+  setOpen,
 }: CheckboxFormMultipleProps) {
   const fieldsDict = filters?.reduce<Record<string, Array<any>>>((pre, cur) => {
     pre[cur.field] = [];
@@ -53,14 +55,14 @@ function CheckboxFormMultiple({
   });
 
   function onSubmit(data: z.infer<typeof FormSchema>) {
-    console.log('🚀 ~ onSubmit ~ data:', data);
-    // setOwnerIds(data.items);
     onChange?.(data);
+    setOpen(false);
   }
 
   const onReset = useCallback(() => {
     onChange?.(fieldsDict);
-  }, [fieldsDict, onChange]);
+    setOpen(false);
+  }, [fieldsDict, onChange, setOpen]);
 
   useEffect(() => {
     form.reset(value);
@@ -148,14 +150,17 @@ export function FilterPopover({
   onChange,
   filters,
 }: PropsWithChildren & CheckboxFormMultipleProps) {
+  const [open, setOpen] = useState(false);
+
   return (
-    <Popover>
+    <Popover open={open} onOpenChange={setOpen}>
       <PopoverTrigger asChild>{children}</PopoverTrigger>
       <PopoverContent>
         <CheckboxFormMultiple
           onChange={onChange}
           value={value}
           filters={filters}
+          setOpen={setOpen}
         ></CheckboxFormMultiple>
       </PopoverContent>
     </Popover>

web/src/hooks/logic-hooks/use-row-selection.ts

@@ -0,0 +1,29 @@
+import { RowSelectionState } from '@tanstack/react-table';
+import { isEmpty } from 'lodash';
+import { useMemo, useState } from 'react';
+
+export function useRowSelection() {
+  const [rowSelection, setRowSelection] = useState<RowSelectionState>({});
+
+  return {
+    rowSelection,
+    setRowSelection,
+    rowSelectionIsEmpty: isEmpty(rowSelection),
+  };
+}
+
+export type UseRowSelectionType = ReturnType<typeof useRowSelection>;
+
+export function useSelectedIds<T extends Array<{ id: string }>>(
+  rowSelection: RowSelectionState,
+  list: T,
+) {
+  const selectedIds = useMemo(() => {
+    const indexes = Object.keys(rowSelection);
+    return list
+      .filter((x, idx) => indexes.some((y) => Number(y) === idx))
+      .map((x) => x.id);
+  }, [list, rowSelection]);
+
+  return { selectedIds };
+}

web/src/layouts/index.tsx

@@ -4,9 +4,6 @@ import { Outlet } from 'umi';
 import '../locales/config';
 import Header from './components/header';
 
-import { Toaster as Sonner } from '@/components/ui/sonner';
-import { Toaster } from '@/components/ui/toaster';
-
 import styles from './index.less';
 
 const { Content } = Layout;
@@ -32,8 +29,6 @@ const App: React.FC = () => {
         >
           <Outlet />
         </Content>
-        <Toaster />
-        <Sonner position={'top-right'} expand richColors closeButton></Sonner>
       </Layout>
     </Layout>
   );

web/src/pages/dataset/dataset/dataset-table.tsx

@@ -24,6 +24,7 @@ import {
   TableHeader,
   TableRow,
 } from '@/components/ui/table';
+import { UseRowSelectionType } from '@/hooks/logic-hooks/use-row-selection';
 import { useFetchDocumentList } from '@/hooks/use-document-request';
 import { getExtension } from '@/utils/document-util';
 import { useMemo } from 'react';
@@ -36,12 +37,15 @@ import { useSaveMeta } from './use-save-meta';
 export type DatasetTableProps = Pick<
   ReturnType<typeof useFetchDocumentList>,
   'documents' | 'setPagination' | 'pagination'
->;
+> &
+  Pick<UseRowSelectionType, 'rowSelection' | 'setRowSelection'>;
 
 export function DatasetTable({
   documents,
   pagination,
   setPagination,
+  rowSelection,
+  setRowSelection,
 }: DatasetTableProps) {
   const [sorting, setSorting] = React.useState<SortingState>([]);
   const [columnFilters, setColumnFilters] = React.useState<ColumnFiltersState>(
@@ -49,7 +53,6 @@ export function DatasetTable({
   );
   const [columnVisibility, setColumnVisibility] =
     React.useState<VisibilityState>({});
-  const [rowSelection, setRowSelection] = React.useState({});
 
   const {
     changeParserLoading,

web/src/pages/dataset/dataset/index.tsx

@@ -10,6 +10,7 @@ import {
   DropdownMenuSeparator,
   DropdownMenuTrigger,
 } from '@/components/ui/dropdown-menu';
+import { useRowSelection } from '@/hooks/logic-hooks/use-row-selection';
 import { useFetchDocumentList } from '@/hooks/use-document-request';
 import { Upload } from 'lucide-react';
 import { useTranslation } from 'react-i18next';
@@ -28,7 +29,7 @@ export default function Dataset() {
     onDocumentUploadOk,
     documentUploadLoading,
   } = useHandleUploadDocument();
-  const { list } = useBulkOperateDataset();
+
   const {
     searchString,
     documents,
@@ -48,6 +49,15 @@ export default function Dataset() {
     showCreateModal,
   } = useCreateEmptyDocument();
 
+  const { rowSelection, rowSelectionIsEmpty, setRowSelection } =
+    useRowSelection();
+
+  const { list } = useBulkOperateDataset({
+    documents,
+    rowSelection,
+    setRowSelection,
+  });
+
   return (
     <section className="p-8">
       <ListFilterBar
@@ -76,11 +86,13 @@ export default function Dataset() {
           </DropdownMenuContent>
         </DropdownMenu>
       </ListFilterBar>
-      <BulkOperateBar list={list}></BulkOperateBar>
+      {rowSelectionIsEmpty || <BulkOperateBar list={list}></BulkOperateBar>}
       <DatasetTable
         documents={documents}
         pagination={pagination}
         setPagination={setPagination}
+        rowSelection={rowSelection}
+        setRowSelection={setRowSelection}
       ></DatasetTable>
       {documentUploadVisible && (
         <FileUploadDialog

web/src/pages/dataset/dataset/use-bulk-operate-dataset.tsx

@@ -1,39 +1,131 @@
+import {
+  UseRowSelectionType,
+  useSelectedIds,
+} from '@/hooks/logic-hooks/use-row-selection';
+import {
+  useRemoveDocument,
+  useRunDocument,
+  useSetDocumentStatus,
+} from '@/hooks/use-document-request';
+import { IDocumentInfo } from '@/interfaces/database/document';
 import { Ban, CircleCheck, CircleX, Play, Trash2 } from 'lucide-react';
+import { useCallback } from 'react';
 import { useTranslation } from 'react-i18next';
+import { toast } from 'sonner';
+import { DocumentType, RunningStatus } from './constant';
 
-export function useBulkOperateDataset() {
+export function useBulkOperateDataset({
+  rowSelection,
+  setRowSelection,
+  documents,
+}: Pick<UseRowSelectionType, 'rowSelection' | 'setRowSelection'> & {
+  documents: IDocumentInfo[];
+}) {
   const { t } = useTranslation();
+  const { selectedIds: selectedRowKeys } = useSelectedIds(
+    rowSelection,
+    documents,
+  );
+
+  const { runDocumentByIds } = useRunDocument();
+  const { setDocumentStatus } = useSetDocumentStatus();
+  const { removeDocument } = useRemoveDocument();
+
+  const runDocument = useCallback(
+    (run: number) => {
+      const nonVirtualKeys = selectedRowKeys.filter(
+        (x) =>
+          !documents.some((y) => x === y.id && y.type === DocumentType.Virtual),
+      );
+
+      if (nonVirtualKeys.length === 0) {
+        toast.error(t('Please select a non-empty file list'));
+        return;
+      }
+      runDocumentByIds({
+        documentIds: nonVirtualKeys,
+        run,
+        shouldDelete: false,
+      });
+    },
+    [documents, runDocumentByIds, selectedRowKeys, t],
+  );
+
+  const handleRunClick = useCallback(() => {
+    runDocument(1);
+  }, [runDocument]);
+
+  const handleCancelClick = useCallback(() => {
+    runDocument(2);
+  }, [runDocument]);
+
+  const onChangeStatus = useCallback(
+    (enabled: boolean) => {
+      selectedRowKeys.forEach((id) => {
+        setDocumentStatus({ status: enabled, documentId: id });
+      });
+    },
+    [selectedRowKeys, setDocumentStatus],
+  );
+
+  const handleEnableClick = useCallback(() => {
+    onChangeStatus(true);
+  }, [onChangeStatus]);
+
+  const handleDisableClick = useCallback(() => {
+    onChangeStatus(false);
+  }, [onChangeStatus]);
+
+  const handleDelete = useCallback(() => {
+    const deletedKeys = selectedRowKeys.filter(
+      (x) =>
+        !documents
+          .filter((y) => y.run === RunningStatus.RUNNING)
+          .some((y) => y.id === x),
+    );
+    if (deletedKeys.length === 0) {
+      toast.error(t('theDocumentBeingParsedCannotBeDeleted'));
+      return;
+    }
+
+    return removeDocument(deletedKeys);
+  }, [selectedRowKeys, removeDocument, documents, t]);
 
   const list = [
     {
       id: 'enabled',
       label: t('knowledgeDetails.enabled'),
       icon: <CircleCheck />,
-      onClick: () => {},
+      onClick: handleEnableClick,
     },
     {
       id: 'disabled',
       label: t('knowledgeDetails.disabled'),
       icon: <Ban />,
-      onClick: () => {},
+      onClick: handleDisableClick,
     },
     {
       id: 'run',
       label: t('knowledgeDetails.run'),
       icon: <Play />,
-      onClick: () => {},
+      onClick: handleRunClick,
     },
     {
       id: 'cancel',
       label: t('knowledgeDetails.cancel'),
       icon: <CircleX />,
-      onClick: () => {},
+      onClick: handleCancelClick,
     },
     {
       id: 'delete',
       label: t('common.delete'),
       icon: <Trash2 />,
-      onClick: () => {},
+      onClick: async () => {
+        const code = await handleDelete();
+        if (code === 0) {
+          setRowSelection({});
+        }
+      },
     },
   ];
 

web/src/pages/dataset/sidebar/index.tsx

@@ -1,8 +1,10 @@
+import { Avatar, AvatarFallback, AvatarImage } from '@/components/ui/avatar';
 import { Button } from '@/components/ui/button';
 import { useSecondPathName } from '@/hooks/route-hook';
 import { useFetchKnowledgeBaseConfiguration } from '@/hooks/use-knowledge-request';
 import { cn } from '@/lib/utils';
 import { Routes } from '@/routes';
+import { formatDate } from '@/utils/date';
 import { Banknote, LayoutGrid, User } from 'lucide-react';
 import { useHandleMenuClick } from './hooks';
 
@@ -16,15 +18,6 @@ const items = [
   { icon: Banknote, label: 'Settings', key: Routes.DatasetSetting },
 ];
 
-const dataset = {
-  id: 1,
-  title: 'Legal knowledge base',
-  files: '1,242 files',
-  size: '152 MB',
-  created: '12.02.2024',
-  image: 'https://github.com/shadcn.png',
-};
-
 export function SideBar() {
   const pathName = useSecondPathName();
   const { handleMenuClick } = useHandleMenuClick();
@@ -33,16 +26,18 @@ export function SideBar() {
   return (
     <aside className="w-60 relative border-r ">
       <div className="p-6 space-y-2 border-b">
-        <div
+        <Avatar className="size-20 rounded-lg">
-          className="w-[70px] h-[70px] rounded-xl bg-cover"
+          <AvatarImage src={data.avatar} />
-          style={{ backgroundImage: `url(${dataset.image})` }}
+          <AvatarFallback className="rounded-lg">CN</AvatarFallback>
-        />
+        </Avatar>
 
         <h3 className="text-lg font-semibold mb-2">{data.name}</h3>
         <div className="text-sm opacity-80">
-          {dataset.files} | {dataset.size}
+          {data.doc_num} files | {data.chunk_num} chunks
+        </div>
+        <div className="text-sm opacity-80">
+          Created {formatDate(data.create_time)}
         </div>
-        <div className="text-sm opacity-80">Created {dataset.created}</div>
       </div>
       <div className="mt-4">
         {items.map((item, itemIdx) => {

web/src/pages/datasets/datasets-filter-popover.tsx

@@ -1,149 +0,0 @@
-import {
-  Popover,
-  PopoverContent,
-  PopoverTrigger,
-} from '@/components/ui/popover';
-import { zodResolver } from '@hookform/resolvers/zod';
-import { PropsWithChildren, useCallback, useEffect } from 'react';
-import { useForm } from 'react-hook-form';
-import { z } from 'zod';
-
-import { Button } from '@/components/ui/button';
-import { Checkbox } from '@/components/ui/checkbox';
-import {
-  Form,
-  FormControl,
-  FormField,
-  FormItem,
-  FormLabel,
-  FormMessage,
-} from '@/components/ui/form';
-import { useFetchNextKnowledgeListByPage } from '@/hooks/use-knowledge-request';
-import { useSelectOwners } from './use-select-owners';
-
-const FormSchema = z.object({
-  items: z.array(z.string()).refine((value) => value.some((item) => item), {
-    message: 'You have to select at least one item.',
-  }),
-});
-
-type CheckboxReactHookFormMultipleProps = Pick<
-  ReturnType<typeof useFetchNextKnowledgeListByPage>,
-  'setOwnerIds' | 'ownerIds'
->;
-
-function CheckboxReactHookFormMultiple({
-  setOwnerIds,
-  ownerIds,
-}: CheckboxReactHookFormMultipleProps) {
-  const owners = useSelectOwners();
-
-  const form = useForm<z.infer<typeof FormSchema>>({
-    resolver: zodResolver(FormSchema),
-    defaultValues: {
-      items: [],
-    },
-  });
-
-  function onSubmit(data: z.infer<typeof FormSchema>) {
-    setOwnerIds(data.items);
-  }
-
-  const onReset = useCallback(() => {
-    setOwnerIds([]);
-  }, [setOwnerIds]);
-
-  useEffect(() => {
-    form.setValue('items', ownerIds);
-  }, [form, ownerIds]);
-
-  return (
-    <Form {...form}>
-      <form
-        onSubmit={form.handleSubmit(onSubmit)}
-        className="space-y-8"
-        onReset={() => form.reset()}
-      >
-        <FormField
-          control={form.control}
-          name="items"
-          render={() => (
-            <FormItem>
-              <div className="mb-4">
-                <FormLabel className="text-base">Owner</FormLabel>
-              </div>
-              {owners.map((item) => (
-                <FormField
-                  key={item.id}
-                  control={form.control}
-                  name="items"
-                  render={({ field }) => {
-                    return (
-                      <div className="flex items-center justify-between">
-                        <FormItem
-                          key={item.id}
-                          className="flex flex-row  space-x-3 space-y-0 items-center"
-                        >
-                          <FormControl>
-                            <Checkbox
-                              checked={field.value?.includes(item.id)}
-                              onCheckedChange={(checked) => {
-                                return checked
-                                  ? field.onChange([...field.value, item.id])
-                                  : field.onChange(
-                                      field.value?.filter(
-                                        (value) => value !== item.id,
-                                      ),
-                                    );
-                              }}
-                            />
-                          </FormControl>
-                          <FormLabel className="text-lg">
-                            {item.label}
-                          </FormLabel>
-                        </FormItem>
-                        <span className=" text-sm">{item.count}</span>
-                      </div>
-                    );
-                  }}
-                />
-              ))}
-              <FormMessage />
-            </FormItem>
-          )}
-        />
-        <div className="flex justify-between">
-          <Button
-            type="button"
-            variant={'outline'}
-            size={'sm'}
-            onClick={onReset}
-          >
-            Clear
-          </Button>
-          <Button type="submit" size={'sm'}>
-            Submit
-          </Button>
-        </div>
-      </form>
-    </Form>
-  );
-}
-
-export function DatasetsFilterPopover({
-  children,
-  setOwnerIds,
-  ownerIds,
-}: PropsWithChildren & CheckboxReactHookFormMultipleProps) {
-  return (
-    <Popover>
-      <PopoverTrigger asChild>{children}</PopoverTrigger>
-      <PopoverContent>
-        <CheckboxReactHookFormMultiple
-          setOwnerIds={setOwnerIds}
-          ownerIds={ownerIds}
-        ></CheckboxReactHookFormMultiple>
-      </PopoverContent>
-    </Popover>
-  );
-}

web/src/pages/files/files-table.tsx

@@ -3,8 +3,6 @@
 import {
   ColumnDef,
   ColumnFiltersState,
-  OnChangeFn,
-  RowSelectionState,
   SortingState,
   VisibilityState,
   flexRender,
@@ -35,6 +33,7 @@ import {
   TooltipContent,
   TooltipTrigger,
 } from '@/components/ui/tooltip';
+import { UseRowSelectionType } from '@/hooks/logic-hooks/use-row-selection';
 import { useFetchFileList } from '@/hooks/use-file-request';
 import { IFile } from '@/interfaces/database/file-manager';
 import { cn } from '@/lib/utils';
@@ -52,10 +51,9 @@ import { useNavigateToOtherFolder } from './use-navigate-to-folder';
 type FilesTableProps = Pick<
   ReturnType<typeof useFetchFileList>,
   'files' | 'loading' | 'pagination' | 'setPagination' | 'total'
-> & {
+> &
-  rowSelection: RowSelectionState;
+  Pick<UseRowSelectionType, 'rowSelection' | 'setRowSelection'> &
-  setRowSelection: OnChangeFn<RowSelectionState>;
+  UseMoveDocumentShowType;
-} & UseMoveDocumentShowType;
 
 export function FilesTable({
   files,

web/src/pages/files/index.tsx

@@ -9,11 +9,9 @@ import {
   DropdownMenuSeparator,
   DropdownMenuTrigger,
 } from '@/components/ui/dropdown-menu';
+import { useRowSelection } from '@/hooks/logic-hooks/use-row-selection';
 import { useFetchFileList } from '@/hooks/use-file-request';
-import { RowSelectionState } from '@tanstack/react-table';
-import { isEmpty } from 'lodash';
 import { Upload } from 'lucide-react';
-import { useState } from 'react';
 import { useTranslation } from 'react-i18next';
 import { CreateFolderDialog } from './create-folder-dialog';
 import { FileBreadcrumb } from './file-breadcrumb';
@@ -60,7 +58,8 @@ export default function Files() {
     moveFileLoading,
   } = useHandleMoveFile();
 
-  const [rowSelection, setRowSelection] = useState<RowSelectionState>({});
+  const { rowSelection, setRowSelection, rowSelectionIsEmpty } =
+    useRowSelection();
 
   const { list } = useBulkOperateFile({
     files,
@@ -101,7 +100,7 @@ export default function Files() {
           </DropdownMenuContent>
         </DropdownMenu>
       </ListFilterBar>
-      {!isEmpty(rowSelection) && <BulkOperateBar list={list}></BulkOperateBar>}
+      {!rowSelectionIsEmpty && <BulkOperateBar list={list}></BulkOperateBar>}
       <FilesTable
         files={files}
         total={total}

web/src/pages/files/use-bulk-operate-file.tsx

@@ -1,7 +1,7 @@
+import { useSelectedIds } from '@/hooks/logic-hooks/use-row-selection';
 import { IFile } from '@/interfaces/database/file-manager';
 import { OnChangeFn, RowSelectionState } from '@tanstack/react-table';
 import { FolderInput, Trash2 } from 'lucide-react';
-import { useMemo } from 'react';
 import { useTranslation } from 'react-i18next';
 import { useHandleDeleteFile } from './use-delete-file';
 import { UseMoveDocumentShowType } from './use-move-file';
@@ -18,12 +18,7 @@ export function useBulkOperateFile({
 } & UseMoveDocumentShowType) {
   const { t } = useTranslation();
 
-  const selectedIds = useMemo(() => {
+  const { selectedIds } = useSelectedIds(rowSelection, files);
-    const indexes = Object.keys(rowSelection);
-    return files
-      .filter((x, idx) => indexes.some((y) => Number(y) === idx))
-      .map((x) => x.id);
-  }, [files, rowSelection]);
 
   const { handleRemoveFile } = useHandleDeleteFile();