feat: test document chunks (#62)

web/src/interfaces/database/knowledge.ts

@@ -75,3 +75,30 @@ export interface IChunk {
   img_id: string;
   important_kwd: any[];
 }
+
+export interface ITestingChunk {
+  chunk_id: string;
+  content_ltks: string;
+  content_with_weight: string;
+  doc_id: string;
+  docnm_kwd: string;
+  img_id: string;
+  important_kwd: any[];
+  kb_id: string;
+  similarity: number;
+  term_similarity: number;
+  vector: number[];
+  vector_similarity: number;
+}
+
+export interface ITestingDocument {
+  count: number;
+  doc_id: string;
+  doc_name: string;
+}
+
+export interface ITestingResult {
+  chunks: ITestingChunk[];
+  doc_aggs: Record<string, number>;
+  total: number;
+}

web/src/pages/add-knowledge/components/knowledge-testing/index.tsx

@@ -1,14 +1,47 @@
-import { Flex } from 'antd';
+import { Flex, Form } from 'antd';
 import TestingControl from './testing-control';
 import TestingResult from './testing-result';
 
+import { useKnowledgeBaseId } from '@/hooks/knowledgeHook';
+import { useEffect } from 'react';
+import { useDispatch } from 'umi';
 import styles from './index.less';
 
 const KnowledgeTesting = () => {
+  const [form] = Form.useForm();
+
+  const dispatch = useDispatch();
+  const knowledgeBaseId = useKnowledgeBaseId();
+
+  const handleTesting = async () => {
+    const values = await form.validateFields();
+    console.info(values);
+    const similarity_threshold = values.similarity_threshold / 100;
+    const vector_similarity_weight = values.vector_similarity_weight / 100;
+    dispatch({
+      type: 'testingModel/testDocumentChunk',
+      payload: {
+        ...values,
+        similarity_threshold,
+        vector_similarity_weight,
+        kb_id: knowledgeBaseId,
+      },
+    });
+  };
+
+  useEffect(() => {
+    return () => {
+      dispatch({ type: 'testingModel/reset' });
+    };
+  }, [dispatch]);
+
   return (
     <Flex className={styles.testingWrapper} gap={16}>
-      <TestingControl></TestingControl>
+      <TestingControl
-      <TestingResult></TestingResult>
+        form={form}
+        handleTesting={handleTesting}
+      ></TestingControl>
+      <TestingResult handleTesting={handleTesting}></TestingResult>
     </Flex>
   );
 };

web/src/pages/add-knowledge/components/knowledge-testing/model.ts

@@ -0,0 +1,72 @@
+import { BaseState } from '@/interfaces/common';
+import {
+  ITestingChunk,
+  ITestingDocument,
+} from '@/interfaces/database/knowledge';
+import kbService from '@/services/kbService';
+import { DvaModel } from 'umi';
+
+export interface TestingModelState extends Pick<BaseState, 'pagination'> {
+  chunks: ITestingChunk[];
+  documents: ITestingDocument[];
+  total: number;
+  selectedDocumentIds: string[] | undefined;
+}
+
+const initialState = {
+  chunks: [],
+  documents: [],
+  total: 0,
+  pagination: {
+    current: 1,
+    pageSize: 10,
+  },
+  selectedDocumentIds: undefined,
+};
+
+const model: DvaModel<TestingModelState> = {
+  namespace: 'testingModel',
+  state: initialState,
+  reducers: {
+    setChunksAndDocuments(state, { payload }) {
+      return {
+        ...state,
+        ...payload,
+      };
+    },
+    setPagination(state, { payload }) {
+      return { ...state, pagination: { ...state.pagination, ...payload } };
+    },
+    setSelectedDocumentIds(state, { payload }) {
+      return { ...state, selectedDocumentIds: payload };
+    },
+    reset() {
+      return initialState;
+    },
+  },
+  effects: {
+    *testDocumentChunk({ payload = {} }, { call, put, select }) {
+      const { pagination, selectedDocumentIds }: TestingModelState =
+        yield select((state: any) => state.testingModel);
+
+      const { data } = yield call(kbService.retrieval_test, {
+        ...payload,
+        doc_ids: selectedDocumentIds,
+        page: pagination.current,
+        size: pagination.pageSize,
+      });
+      const { retcode, data: res } = data;
+      if (retcode === 0) {
+        yield put({
+          type: 'setChunksAndDocuments',
+          payload: {
+            chunks: res.chunks,
+            documents: res.doc_aggs,
+            total: res.total,
+          },
+        });
+      }
+    },
+  },
+};
+export default model;

web/src/pages/add-knowledge/components/knowledge-testing/testing-control/index.less

@@ -2,6 +2,9 @@
   width: 350px;
   background-color: white;
   padding: 30px 20px;
+  overflow: auto;
+  height: calc(100vh - 160px);
+
   .historyTitle {
     padding: 30px 0 20px;
   }

web/src/pages/add-knowledge/components/knowledge-testing/testing-control/index.tsx

@@ -3,6 +3,7 @@ import {
   Card,
   Divider,
   Flex,
+  Form,
   Input,
   Slider,
   SliderSingleProps,
@@ -11,50 +12,95 @@ import {
 } from 'antd';
 
 import { DeleteOutlined, HistoryOutlined } from '@ant-design/icons';
+import { FormInstance } from 'antd/lib';
 import styles from './index.less';
 
 const list = [1, 2, 3];
 
 const marks: SliderSingleProps['marks'] = {
-  0: '0°C',
+  0: '0',
-  26: '26°C',
+  100: '1',
-  37: '37°C',
-  100: {
-    style: {
-      color: '#f50',
-    },
-    label: <strong>100°C</strong>,
-  },
 };
 
-const TestingControl = () => {
+type FieldType = {
+  similarity_threshold?: number;
+  vector_similarity_weight?: number;
+  top_k?: number;
+  question: string;
+};
+
+const formatter = (value: number | undefined) => {
+  return typeof value === 'number' ? value / 100 : 0;
+};
+
+const tooltip = { formatter };
+
+interface IProps {
+  form: FormInstance;
+  handleTesting: () => Promise<any>;
+}
+
+const TestingControl = ({ form, handleTesting }: IProps) => {
+  const question = Form.useWatch('question', { form, preserve: true });
+
+  const buttonDisabled =
+    !question || (typeof question === 'string' && question.trim() === '');
+
   return (
     <section className={styles.testingControlWrapper}>
       <p>
         <b>Retrieval testing</b>
       </p>
-      <p>xxxx</p>
+      <p>Final step! After success, leave the rest to Infiniflow AI.</p>
       <Divider></Divider>
       <section>
-        <Slider range marks={marks} defaultValue={[26, 37]} />
+        <Form
-        <Slider range marks={marks} defaultValue={[26, 37]} />
+          name="testing"
-        <Card
+          layout="vertical"
-          size="small"
+          form={form}
-          title="Test text"
+          initialValues={{
-          extra={
+            similarity_threshold: 20,
-            <Button type="primary" ghost>
+            vector_similarity_weight: 30,
-              Semantic Search
+            top_k: 1024,
-            </Button>
+          }}
-          }
+        >
+          <Form.Item<FieldType>
+            label="Similarity threshold"
+            name={'similarity_threshold'}
+          >
+            <Slider marks={marks} defaultValue={0} tooltip={tooltip} />
+          </Form.Item>
+          <Form.Item<FieldType>
+            label="Vector similarity weight"
+            name={'vector_similarity_weight'}
+          >
+            <Slider marks={marks} defaultValue={0} tooltip={tooltip} />
+          </Form.Item>
+          <Form.Item<FieldType> label="Top k" name={'top_k'}>
+            <Slider marks={{ 0: 0, 2048: 2048 }} defaultValue={0} max={2048} />
+          </Form.Item>
+          <Card size="small" title="Test text">
+            <Form.Item<FieldType>
+              name={'question'}
+              rules={[
+                { required: true, message: 'Please input your question!' },
+              ]}
             >
               <Input.TextArea autoSize={{ minRows: 8 }}></Input.TextArea>
+            </Form.Item>
             <Flex justify={'space-between'}>
               <Tag>10/200</Tag>
-            <Button type="primary" size="small">
+              <Button
+                type="primary"
+                size="small"
+                onClick={handleTesting}
+                disabled={buttonDisabled}
+              >
                 Testing
               </Button>
             </Flex>
           </Card>
+        </Form>
       </section>
       <section>
         <p className={styles.historyTitle}>

web/src/pages/add-knowledge/components/knowledge-testing/testing-result/index.less

@@ -2,15 +2,35 @@
   flex: 1;
   background-color: white;
   padding: 30px 20px;
+  overflow: auto;
+  height: calc(100vh - 160px);
+  display: flex;
+  flex-direction: column;
+  justify-content: space-between;
 
   .selectFilesCollapse {
     :global(.ant-collapse-header) {
       padding-left: 22px;
     }
     margin-bottom: 32px;
+    overflow-y: auto;
   }
 
   .selectFilesTitle {
     padding-right: 10px;
   }
+
+  .similarityCircle {
+    width: 24px;
+    height: 24px;
+    border-radius: 50%;
+    background-color: rgba(244, 235, 255, 1);
+    font-size: 10px;
+    font-weight: normal;
+  }
+
+  .similarityText {
+    font-size: 12px;
+    font-weight: 500;
+  }
 }

web/src/pages/add-knowledge/components/knowledge-testing/testing-result/index.tsx

@@ -1,12 +1,55 @@
 import { ReactComponent as SelectedFilesCollapseIcon } from '@/assets/svg/selected-files-collapse.svg';
-import { Card, Collapse, Flex, Space } from 'antd';
+import { ITestingChunk } from '@/interfaces/database/knowledge';
+import { Card, Collapse, Flex, Pagination, PaginationProps, Space } from 'antd';
+import { useDispatch, useSelector } from 'umi';
+import { TestingModelState } from '../model';
+import styles from './index.less';
 import SelectFiles from './select-files';
 
-import styles from './index.less';
+const similarityList: Array<{ field: keyof ITestingChunk; label: string }> = [
+  { field: 'similarity', label: 'Hybrid Similarity' },
+  { field: 'term_similarity', label: 'Term Similarity' },
+  { field: 'vector_similarity', label: 'Vector Similarity' },
+];
 
-const list = [1, 2, 3, 4];
+const ChunkTitle = ({ item }: { item: ITestingChunk }) => {
+  return (
+    <Flex gap={10}>
+      {similarityList.map((x) => (
+        <Space key={x.field}>
+          <span className={styles.similarityCircle}>
+            {((item[x.field] as number) * 100).toFixed(2)}%
+          </span>
+          <span className={styles.similarityText}>Hybrid Similarity</span>
+        </Space>
+      ))}
+    </Flex>
+  );
+};
+
+interface IProps {
+  handleTesting: () => Promise<any>;
+}
+
+const TestingResult = ({ handleTesting }: IProps) => {
+  const {
+    documents,
+    chunks,
+    total,
+    pagination,
+    selectedDocumentIds,
+  }: TestingModelState = useSelector((state: any) => state.testingModel);
+  const dispatch = useDispatch();
+
+  const onChange: PaginationProps['onChange'] = (pageNumber, pageSize) => {
+    console.log('Page: ', pageNumber, pageSize);
+    dispatch({
+      type: 'testingModel/setPagination',
+      payload: { current: pageNumber, pageSize },
+    });
+    handleTesting();
+  };
 
-const TestingResult = () => {
   return (
     <section className={styles.testingResultWrapper}>
       <Collapse
@@ -23,7 +66,10 @@ const TestingResult = () => {
                 align="center"
                 className={styles.selectFilesTitle}
               >
-                <span>4/25 Files Selected</span>
+                <span>
+                  {selectedDocumentIds?.length ?? 0}/{documents.length} Files
+                  Selected
+                </span>
                 <Space size={52}>
                   <b>Hits</b>
                   <b>View</b>
@@ -32,21 +78,33 @@ const TestingResult = () => {
             ),
             children: (
               <div>
-                <SelectFiles></SelectFiles>
+                <SelectFiles handleTesting={handleTesting}></SelectFiles>
               </div>
             ),
           },
         ]}
       />
-      <Flex gap={'large'} vertical>
+      <Flex
-        {list.map((x) => (
+        gap={'large'}
-          <Card key={x} title="Default size card" extra={<a href="#">More</a>}>
+        vertical
-            <p>Card content</p>
+        flex={1}
-            <p>Card content</p>
+        className={styles.selectFilesCollapse}
-            <p>Card content</p>
+      >
+        {chunks.map((x) => (
+          <Card key={x.chunk_id} title={<ChunkTitle item={x}></ChunkTitle>}>
+            <div>{x.content_with_weight}</div>
           </Card>
         ))}
       </Flex>
+      <Pagination
+        size={'small'}
+        showQuickJumper
+        current={pagination.current}
+        pageSize={pagination.pageSize}
+        total={total}
+        showSizeChanger
+        onChange={onChange}
+      />
     </section>
   );
 };

web/src/pages/add-knowledge/components/knowledge-testing/testing-result/select-files.tsx

@@ -1,80 +1,71 @@
 import { ReactComponent as NavigationPointerIcon } from '@/assets/svg/navigation-pointer.svg';
+import { ITestingDocument } from '@/interfaces/database/knowledge';
+import { api_host } from '@/utils/api';
 import { Table, TableProps } from 'antd';
+import { useDispatch, useSelector } from 'umi';
 
-interface DataType {
+interface IProps {
-  key: string;
+  handleTesting: () => Promise<any>;
-  name: string;
-  hits: number;
-  address: string;
-  tags: string[];
 }
 
-const SelectFiles = () => {
+const SelectFiles = ({ handleTesting }: IProps) => {
-  const columns: TableProps<DataType>['columns'] = [
+  const documents: ITestingDocument[] = useSelector(
+    (state: any) => state.testingModel.documents,
+  );
+
+  const dispatch = useDispatch();
+
+  const columns: TableProps<ITestingDocument>['columns'] = [
     {
       title: 'Name',
-      dataIndex: 'name',
+      dataIndex: 'doc_name',
-      key: 'name',
+      key: 'doc_name',
       render: (text) => <p>{text}</p>,
     },
 
     {
       title: 'Hits',
-      dataIndex: 'hits',
+      dataIndex: 'count',
-      key: 'hits',
+      key: 'count',
       width: 80,
     },
     {
       title: 'View',
       key: 'view',
       width: 50,
-      render: () => <NavigationPointerIcon />,
+      render: (_, { doc_id }) => (
+        <a
+          href={`${api_host}/document/get/${doc_id}`}
+          target="_blank"
+          rel="noreferrer"
+        >
+          <NavigationPointerIcon />
+        </a>
+      ),
     },
   ];
 
   const rowSelection = {
-    onChange: (selectedRowKeys: React.Key[], selectedRows: DataType[]) => {
+    onChange: (selectedRowKeys: React.Key[]) => {
-      console.log(
+      dispatch({
-        `selectedRowKeys: ${selectedRowKeys}`,
+        type: 'testingModel/setSelectedDocumentIds',
-        'selectedRows: ',
+        payload: selectedRowKeys,
-        selectedRows,
+      });
-      );
+      handleTesting();
     },
-    getCheckboxProps: (record: DataType) => ({
+    getCheckboxProps: (record: ITestingDocument) => ({
-      disabled: record.name === 'Disabled User', // Column configuration not to be checked
+      disabled: record.doc_name === 'Disabled User', // Column configuration not to be checked
-      name: record.name,
+      name: record.doc_name,
     }),
   };
 
-  const data: DataType[] = [
-    {
-      key: '1',
-      name: 'John Brown',
-      hits: 32,
-      address: 'New York No. 1 Lake Park',
-      tags: ['nice', 'developer'],
-    },
-    {
-      key: '2',
-      name: 'Jim Green',
-      hits: 42,
-      address: 'London No. 1 Lake Park',
-      tags: ['loser'],
-    },
-    {
-      key: '3',
-      name: 'Joe Black',
-      hits: 32,
-      address: 'Sydney No. 1 Lake Park',
-      tags: ['cool', 'teacher'],
-    },
-  ];
   return (
     <Table
       columns={columns}
-      dataSource={data}
+      dataSource={documents}
       showHeader={false}
       rowSelection={rowSelection}
+      rowKey={'doc_id'}
     />
   );
 };

web/src/utils/api.ts

@@ -1,4 +1,4 @@
-let api_host = `http://223.111.148.200:9380/v1`;
+let api_host = `http://123.60.95.134:9380/v1`;
 
 export { api_host };