refactor: refactor document processing components and update translations

web/app/components/datasets/documents/create-from-pipeline/index.tsx

@@ -13,7 +13,6 @@ import FireCrawl from '@/app/components/rag-pipeline/components/panel/test-run/d
 import JinaReader from '@/app/components/rag-pipeline/components/panel/test-run/data-source/website/jina-reader'
 import WaterCrawl from '@/app/components/rag-pipeline/components/panel/test-run/data-source/website/water-crawl'
 import Actions from './data-source/actions'
-import DocumentProcessing from '@/app/components/rag-pipeline/components/panel/test-run/document-processing'
 import { useTranslation } from 'react-i18next'
 import type { Datasource } from '@/app/components/rag-pipeline/components/panel/test-run/types'
 import LeftHeader from './left-header'
@@ -25,6 +24,7 @@ import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-so
 import FilePreview from './preview/file-preview'
 import NotionPagePreview from './preview/notion-page-preview'
 import WebsitePreview from './preview/web-preview'
+import ProcessDocuments from './process-documents'
 
 const TestRunPanel = () => {
   const { t } = useTranslation()
@@ -229,7 +229,7 @@ const TestRunPanel = () => {
           }
           {
             currentStep === 2 && (
-              <DocumentProcessing
+              <ProcessDocuments
                 dataSourceNodeId={datasource?.nodeId || ''}
                 onProcess={handleProcess}
                 onBack={handleBackStep}

web/app/components/datasets/documents/create-from-pipeline/process-documents/actions.tsx

@@ -0,0 +1,34 @@
+import React from 'react'
+import Button from '@/app/components/base/button'
+import { useTranslation } from 'react-i18next'
+
+type ActionsProps = {
+  onBack: () => void
+  onProcess: () => void
+}
+
+const Actions = ({
+  onBack,
+  onProcess,
+}: ActionsProps) => {
+  const { t } = useTranslation()
+
+  return (
+    <div className='flex items-center justify-between'>
+      <Button
+        variant='secondary'
+        onClick={onBack}
+      >
+        {t('datasetPipeline.operations.dataSource')}
+      </Button>
+      <Button
+        variant='primary'
+        onClick={onProcess}
+      >
+        {t('datasetPipeline.operations.saveAndProcess')}
+      </Button>
+    </div>
+  )
+}
+
+export default React.memo(Actions)

web/app/components/datasets/documents/create-from-pipeline/process-documents/header.tsx

@@ -0,0 +1,33 @@
+import React from 'react'
+import Button from '@/app/components/base/button'
+import { useTranslation } from 'react-i18next'
+
+type HeaderProps = {
+  onReset: () => void
+  disableReset?: boolean
+  onPreview?: () => void
+}
+
+const Header = ({
+  onReset,
+  disableReset = true,
+  onPreview,
+}: HeaderProps) => {
+  const { t } = useTranslation()
+
+  return (
+    <div className='flex items-center gap-x-1 px-4 py-2'>
+      <div className='system-sm-semibold-uppercase grow text-text-secondary'>
+        {t('datasetPipeline.addDocuments.stepTwo.chunkSettings')}
+      </div>
+      <Button variant='ghost' disabled={disableReset} onClick={onReset}>
+        {t('common.operations.reset')}
+      </Button>
+      <Button variant='primary' onClick={onPreview}>
+        {t('common.operations.reset')}
+      </Button>
+    </div>
+  )
+}
+
+export default React.memo(Header)

web/app/components/datasets/documents/create-from-pipeline/process-documents/hooks.ts

@@ -0,0 +1,61 @@
+import { useMemo } from 'react'
+import { BaseFieldType } from '@/app/components/base/form/form-scenarios/base/types'
+import { useStore } from '@/app/components/workflow/store'
+import { usePublishedPipelineProcessingParams } from '@/service/use-pipeline'
+import { PipelineInputVarType } from '@/models/pipeline'
+
+type PartialInputVarType = PipelineInputVarType.textInput | PipelineInputVarType.number | PipelineInputVarType.select | PipelineInputVarType.checkbox
+
+const VAR_TYPE_MAP: Record<PartialInputVarType, BaseFieldType> = {
+  [PipelineInputVarType.textInput]: BaseFieldType.textInput,
+  [PipelineInputVarType.number]: BaseFieldType.numberInput,
+  [PipelineInputVarType.select]: BaseFieldType.select,
+  [PipelineInputVarType.checkbox]: BaseFieldType.checkbox,
+}
+
+export const useConfigurations = (datasourceNodeId: string) => {
+  const pipelineId = useStore(state => state.pipelineId)
+  const { data: paramsConfig } = usePublishedPipelineProcessingParams({
+    pipeline_id: pipelineId!,
+    node_id: datasourceNodeId,
+  })
+
+  const initialData = useMemo(() => {
+    const variables = paramsConfig?.variables || []
+    return variables.reduce((acc, item) => {
+      const type = VAR_TYPE_MAP[item.type as PartialInputVarType]
+      if (type === BaseFieldType.textInput)
+        acc[item.variable] = ''
+      if (type === BaseFieldType.numberInput)
+        acc[item.variable] = 0
+      if (type === BaseFieldType.select)
+        acc[item.variable] = item.options?.[0] || ''
+      if (type === BaseFieldType.checkbox)
+        acc[item.variable] = true
+      return acc
+    }, {} as Record<string, any>)
+  }, [paramsConfig])
+
+  const configurations = useMemo(() => {
+    const variables = paramsConfig?.variables || []
+    const configs = variables.map(item => ({
+      type: VAR_TYPE_MAP[item.type as PartialInputVarType],
+      variable: item.variable,
+      label: item.label,
+      required: item.required,
+      maxLength: item.max_length,
+      options: item.options?.map(option => ({
+        label: option,
+        value: option,
+      })),
+      showConditions: [],
+      default: item.default_value,
+    }))
+    return configs
+  }, [paramsConfig])
+
+  return {
+    initialData,
+    configurations,
+  }
+}

web/app/components/datasets/documents/create-from-pipeline/process-documents/index.tsx

@@ -0,0 +1,56 @@
+import { generateZodSchema } from '@/app/components/base/form/form-scenarios/base/utils'
+import { useConfigurations } from './hooks'
+import Options from './options'
+import Actions from './actions'
+import { useCallback, useRef } from 'react'
+import Header from './header'
+
+type ProcessDocumentsProps = {
+  dataSourceNodeId: string
+  onProcess: (data: Record<string, any>) => void
+  onBack: () => void
+}
+
+const ProcessDocuments = ({
+  dataSourceNodeId,
+  onProcess,
+  onBack,
+}: ProcessDocumentsProps) => {
+  const formRef = useRef<any>(null)
+  const { initialData, configurations } = useConfigurations(dataSourceNodeId)
+  const schema = generateZodSchema(configurations)
+
+  const handleProcess = useCallback(() => {
+    formRef.current?.submit()
+  }, [])
+
+  const handlePreview = useCallback(() => {
+    formRef.current?.submit()
+  }, [])
+
+  const handleReset = useCallback(() => {
+    formRef.current?.reset()
+  }, [])
+
+  return (
+    <div className='flex flex-col gap-y-4 pt-4'>
+      <div className='flex flex-col rounded-lg border-components-panel-border bg-components-panel-bg'>
+        <Header
+          onReset={handleReset}
+          disableReset={formRef.current.isDirty()}
+          onPreview={handlePreview}
+        />
+        <Options
+          ref={formRef}
+          initialData={initialData}
+          configurations={configurations}
+          schema={schema}
+          onSubmit={onProcess}
+        />
+      </div>
+      <Actions onBack={onBack} onProcess={handleProcess} />
+    </div>
+  )
+}
+
+export default ProcessDocuments

web/app/components/datasets/documents/create-from-pipeline/process-documents/options.tsx

@@ -0,0 +1,82 @@
+import { useAppForm } from '@/app/components/base/form'
+import BaseField from '@/app/components/base/form/form-scenarios/base/field'
+import type { BaseConfiguration } from '@/app/components/base/form/form-scenarios/base/types'
+import Toast from '@/app/components/base/toast'
+import { useImperativeHandle } from 'react'
+import type { ZodSchema } from 'zod'
+
+type OptionsProps = {
+  initialData: Record<string, any>
+  configurations: BaseConfiguration[]
+  schema: ZodSchema
+  onSubmit: (data: Record<string, any>) => void
+  ref: React.RefObject<any>
+}
+
+const Options = ({
+  initialData,
+  configurations,
+  schema,
+  onSubmit,
+  ref,
+}: OptionsProps) => {
+  const form = useAppForm({
+    defaultValues: initialData,
+    validators: {
+      onSubmit: ({ value }) => {
+        const result = schema.safeParse(value)
+        if (!result.success) {
+          const issues = result.error.issues
+          const firstIssue = issues[0]
+          const errorMessage = `"${firstIssue.path.join('.')}" ${firstIssue.message}`
+          Toast.notify({
+            type: 'error',
+            message: errorMessage,
+          })
+          return errorMessage
+        }
+        return undefined
+      },
+    },
+    onSubmit: ({ value }) => {
+      onSubmit(value)
+    },
+  })
+
+  useImperativeHandle(ref, () => {
+    return {
+      submit: () => {
+        form.handleSubmit()
+      },
+      reset: () => {
+        form.reset()
+      },
+      isDirty: () => {
+        return form.state.isDirty
+      },
+    }
+  }, [form])
+
+  return (
+    <form
+      className='w-full'
+      onSubmit={(e) => {
+        e.preventDefault()
+        e.stopPropagation()
+        form.handleSubmit()
+      }}
+    >
+      <div className='flex flex-col gap-3 px-4 py-3'>
+        {configurations.map((config, index) => {
+          const FieldComponent = BaseField({
+            initialData,
+            config,
+          })
+          return <FieldComponent key={index} form={form} />
+        })}
+      </div>
+    </form>
+  )
+}
+
+export default Options

web/app/components/rag-pipeline/components/panel/test-run/document-processing/hooks.ts

@@ -1,7 +1,7 @@
 import { useMemo } from 'react'
 import { BaseFieldType } from '@/app/components/base/form/form-scenarios/base/types'
 import { useStore } from '@/app/components/workflow/store'
-import { usePipelineProcessingParams } from '@/service/use-pipeline'
+import { useDraftPipelineProcessingParams } from '@/service/use-pipeline'
 import { PipelineInputVarType } from '@/models/pipeline'
 
 type PartialInputVarType = PipelineInputVarType.textInput | PipelineInputVarType.number | PipelineInputVarType.select | PipelineInputVarType.checkbox
@@ -15,7 +15,7 @@ const VAR_TYPE_MAP: Record<PartialInputVarType, BaseFieldType> = {
 
 export const useConfigurations = (datasourceNodeId: string) => {
   const pipelineId = useStore(state => state.pipelineId)
-  const { data: paramsConfig } = usePipelineProcessingParams({
+  const { data: paramsConfig } = useDraftPipelineProcessingParams({
     pipeline_id: pipelineId!,
     node_id: datasourceNodeId,
   })

web/app/components/rag-pipeline/components/panel/test-run/index.tsx

@@ -39,7 +39,6 @@ const TestRunPanel = () => {
   const allFileLoaded = (fileList.length > 0 && fileList.every(file => file.file.id))
   const isVectorSpaceFull = plan.usage.vectorSpace >= plan.total.vectorSpace
   const isShowVectorSpaceFull = allFileLoaded && isVectorSpaceFull && enableBilling
-  const notSupportBatchUpload = enableBilling && plan.type === 'sandbox'
   const nextDisabled = useMemo(() => {
     if (!fileList.length)
       return true
@@ -156,7 +155,7 @@ const TestRunPanel = () => {
                     files={fileList}
                     updateFile={updateFile}
                     updateFileList={updateFileList}
-                    notSupportBatchUpload={notSupportBatchUpload}
+                    notSupportBatchUpload={false} // only support single file upload in test run
                   />
                 )}
                 {datasource?.type === DataSourceType.NOTION && (

web/i18n/en-US/dataset-pipeline.ts

@@ -25,6 +25,8 @@ const translation = {
     useTemplate: 'Use this Knowledge Pipeline',
     backToDataSource: 'Back to Data Source',
     process: 'Process',
+    dataSource: 'Data Source',
+    saveAndProcess: 'Save & Process',
   },
   knowledgeNameAndIcon: 'Knowledge name & icon',
   knowledgeNameAndIconPlaceholder: 'Please enter the name of the Knowledge Base',
@@ -81,6 +83,9 @@ const translation = {
     stepOne: {
       preview: 'Preview',
     },
+    stepTwo: {
+      chunkSettings: 'Chunk Settings',
+    },
     characters: 'characters',
   },
 }

web/i18n/zh-Hans/dataset-pipeline.ts

@@ -25,6 +25,8 @@ const translation = {
     useTemplate: '使用此知识库流水线',
     backToDataSource: '返回数据源',
     process: '处理',
+    dataSource: '数据源',
+    saveAndProcess: '保存并处理',
   },
   knowledgeNameAndIcon: '知识库名称和图标',
   knowledgeNameAndIconPlaceholder: '请输入知识库名称',
@@ -81,6 +83,9 @@ const translation = {
     stepOne: {
       preview: '预览',
     },
+    stepTwo: {
+      chunkSettings: '分段设置',
+    },
     characters: '字符',
   },
 }

web/service/use-pipeline.ts

@@ -132,13 +132,26 @@ export const useDatasourceNodeRun = (
   })
 }
 
-// Get the config of shared input fields
-export const usePipelineProcessingParams = (params: PipelineProcessingParamsRequest) => {
+export const useDraftPipelineProcessingParams = (params: PipelineProcessingParamsRequest) => {
   const { pipeline_id, node_id } = params
   return useQuery<PipelineProcessingParamsResponse>({
     queryKey: [NAME_SPACE, 'pipeline-processing-params', pipeline_id],
     queryFn: () => {
-      return get<PipelineProcessingParamsResponse>(`/rag/pipelines/${pipeline_id}/workflows/processing/parameters`, {
+      return get<PipelineProcessingParamsResponse>(`/rag/pipelines/${pipeline_id}/workflows/draft/processing/parameters`, {
+        params: {
+          node_id,
+        },
+      })
+    },
+  })
+}
+
+export const usePublishedPipelineProcessingParams = (params: PipelineProcessingParamsRequest) => {
+  const { pipeline_id, node_id } = params
+  return useQuery<PipelineProcessingParamsResponse>({
+    queryKey: [NAME_SPACE, 'pipeline-processing-params', pipeline_id],
+    queryFn: () => {
+      return get<PipelineProcessingParamsResponse>(`/rag/pipelines/${pipeline_id}/workflows/published/processing/parameters`, {
         params: {
           node_id,
         },