diff --git a/web/app/components/datasets/documents/create-from-pipeline/index.tsx b/web/app/components/datasets/documents/create-from-pipeline/index.tsx index 19edb8d586..93288b6f50 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/index.tsx +++ b/web/app/components/datasets/documents/create-from-pipeline/index.tsx @@ -13,7 +13,6 @@ import FireCrawl from '@/app/components/rag-pipeline/components/panel/test-run/d import JinaReader from '@/app/components/rag-pipeline/components/panel/test-run/data-source/website/jina-reader' import WaterCrawl from '@/app/components/rag-pipeline/components/panel/test-run/data-source/website/water-crawl' import Actions from './data-source/actions' -import DocumentProcessing from '@/app/components/rag-pipeline/components/panel/test-run/document-processing' import { useTranslation } from 'react-i18next' import type { Datasource } from '@/app/components/rag-pipeline/components/panel/test-run/types' import LeftHeader from './left-header' @@ -25,6 +24,7 @@ import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-so import FilePreview from './preview/file-preview' import NotionPagePreview from './preview/notion-page-preview' import WebsitePreview from './preview/web-preview' +import ProcessDocuments from './process-documents' const TestRunPanel = () => { const { t } = useTranslation() @@ -229,7 +229,7 @@ const TestRunPanel = () => { } { currentStep === 2 && ( - void + onProcess: () => void +} + +const Actions = ({ + onBack, + onProcess, +}: ActionsProps) => { + const { t } = useTranslation() + + return ( +
+ + +
+ ) +} + +export default React.memo(Actions) diff --git a/web/app/components/datasets/documents/create-from-pipeline/process-documents/header.tsx b/web/app/components/datasets/documents/create-from-pipeline/process-documents/header.tsx new file mode 100644 index 0000000000..d9912fb57f --- /dev/null +++ b/web/app/components/datasets/documents/create-from-pipeline/process-documents/header.tsx @@ -0,0 +1,33 @@ +import React from 'react' +import Button from '@/app/components/base/button' +import { useTranslation } from 'react-i18next' + +type HeaderProps = { + onReset: () => void + disableReset?: boolean + onPreview?: () => void +} + +const Header = ({ + onReset, + disableReset = true, + onPreview, +}: HeaderProps) => { + const { t } = useTranslation() + + return ( +
+
+ {t('datasetPipeline.addDocuments.stepTwo.chunkSettings')} +
+ + +
+ ) +} + +export default React.memo(Header) diff --git a/web/app/components/datasets/documents/create-from-pipeline/process-documents/hooks.ts b/web/app/components/datasets/documents/create-from-pipeline/process-documents/hooks.ts new file mode 100644 index 0000000000..186d477d4c --- /dev/null +++ b/web/app/components/datasets/documents/create-from-pipeline/process-documents/hooks.ts @@ -0,0 +1,61 @@ +import { useMemo } from 'react' +import { BaseFieldType } from '@/app/components/base/form/form-scenarios/base/types' +import { useStore } from '@/app/components/workflow/store' +import { usePublishedPipelineProcessingParams } from '@/service/use-pipeline' +import { PipelineInputVarType } from '@/models/pipeline' + +type PartialInputVarType = PipelineInputVarType.textInput | PipelineInputVarType.number | PipelineInputVarType.select | PipelineInputVarType.checkbox + +const VAR_TYPE_MAP: Record = { + [PipelineInputVarType.textInput]: BaseFieldType.textInput, + [PipelineInputVarType.number]: BaseFieldType.numberInput, + [PipelineInputVarType.select]: BaseFieldType.select, + [PipelineInputVarType.checkbox]: BaseFieldType.checkbox, +} + +export const useConfigurations = (datasourceNodeId: string) => { + const pipelineId = useStore(state => state.pipelineId) + const { data: paramsConfig } = usePublishedPipelineProcessingParams({ + pipeline_id: pipelineId!, + node_id: datasourceNodeId, + }) + + const initialData = useMemo(() => { + const variables = paramsConfig?.variables || [] + return variables.reduce((acc, item) => { + const type = VAR_TYPE_MAP[item.type as PartialInputVarType] + if (type === BaseFieldType.textInput) + acc[item.variable] = '' + if (type === BaseFieldType.numberInput) + acc[item.variable] = 0 + if (type === BaseFieldType.select) + acc[item.variable] = item.options?.[0] || '' + if (type === BaseFieldType.checkbox) + acc[item.variable] = true + return acc + }, {} as Record) + }, [paramsConfig]) + + const configurations = useMemo(() => { + const variables = paramsConfig?.variables || [] + const configs = variables.map(item => ({ + type: VAR_TYPE_MAP[item.type as PartialInputVarType], + variable: item.variable, + label: item.label, + required: item.required, + maxLength: item.max_length, + options: item.options?.map(option => ({ + label: option, + value: option, + })), + showConditions: [], + default: item.default_value, + })) + return configs + }, [paramsConfig]) + + return { + initialData, + configurations, + } +} diff --git a/web/app/components/datasets/documents/create-from-pipeline/process-documents/index.tsx b/web/app/components/datasets/documents/create-from-pipeline/process-documents/index.tsx new file mode 100644 index 0000000000..02fa29f395 --- /dev/null +++ b/web/app/components/datasets/documents/create-from-pipeline/process-documents/index.tsx @@ -0,0 +1,56 @@ +import { generateZodSchema } from '@/app/components/base/form/form-scenarios/base/utils' +import { useConfigurations } from './hooks' +import Options from './options' +import Actions from './actions' +import { useCallback, useRef } from 'react' +import Header from './header' + +type ProcessDocumentsProps = { + dataSourceNodeId: string + onProcess: (data: Record) => void + onBack: () => void +} + +const ProcessDocuments = ({ + dataSourceNodeId, + onProcess, + onBack, +}: ProcessDocumentsProps) => { + const formRef = useRef(null) + const { initialData, configurations } = useConfigurations(dataSourceNodeId) + const schema = generateZodSchema(configurations) + + const handleProcess = useCallback(() => { + formRef.current?.submit() + }, []) + + const handlePreview = useCallback(() => { + formRef.current?.submit() + }, []) + + const handleReset = useCallback(() => { + formRef.current?.reset() + }, []) + + return ( +
+
+
+ +
+ +
+ ) +} + +export default ProcessDocuments diff --git a/web/app/components/datasets/documents/create-from-pipeline/process-documents/options.tsx b/web/app/components/datasets/documents/create-from-pipeline/process-documents/options.tsx new file mode 100644 index 0000000000..c8b4d5eb41 --- /dev/null +++ b/web/app/components/datasets/documents/create-from-pipeline/process-documents/options.tsx @@ -0,0 +1,82 @@ +import { useAppForm } from '@/app/components/base/form' +import BaseField from '@/app/components/base/form/form-scenarios/base/field' +import type { BaseConfiguration } from '@/app/components/base/form/form-scenarios/base/types' +import Toast from '@/app/components/base/toast' +import { useImperativeHandle } from 'react' +import type { ZodSchema } from 'zod' + +type OptionsProps = { + initialData: Record + configurations: BaseConfiguration[] + schema: ZodSchema + onSubmit: (data: Record) => void + ref: React.RefObject +} + +const Options = ({ + initialData, + configurations, + schema, + onSubmit, + ref, +}: OptionsProps) => { + const form = useAppForm({ + defaultValues: initialData, + validators: { + onSubmit: ({ value }) => { + const result = schema.safeParse(value) + if (!result.success) { + const issues = result.error.issues + const firstIssue = issues[0] + const errorMessage = `"${firstIssue.path.join('.')}" ${firstIssue.message}` + Toast.notify({ + type: 'error', + message: errorMessage, + }) + return errorMessage + } + return undefined + }, + }, + onSubmit: ({ value }) => { + onSubmit(value) + }, + }) + + useImperativeHandle(ref, () => { + return { + submit: () => { + form.handleSubmit() + }, + reset: () => { + form.reset() + }, + isDirty: () => { + return form.state.isDirty + }, + } + }, [form]) + + return ( +
{ + e.preventDefault() + e.stopPropagation() + form.handleSubmit() + }} + > +
+ {configurations.map((config, index) => { + const FieldComponent = BaseField({ + initialData, + config, + }) + return + })} +
+
+ ) +} + +export default Options diff --git a/web/app/components/rag-pipeline/components/panel/test-run/document-processing/hooks.ts b/web/app/components/rag-pipeline/components/panel/test-run/document-processing/hooks.ts index 30b2fc588a..921a7d57b2 100644 --- a/web/app/components/rag-pipeline/components/panel/test-run/document-processing/hooks.ts +++ b/web/app/components/rag-pipeline/components/panel/test-run/document-processing/hooks.ts @@ -1,7 +1,7 @@ import { useMemo } from 'react' import { BaseFieldType } from '@/app/components/base/form/form-scenarios/base/types' import { useStore } from '@/app/components/workflow/store' -import { usePipelineProcessingParams } from '@/service/use-pipeline' +import { useDraftPipelineProcessingParams } from '@/service/use-pipeline' import { PipelineInputVarType } from '@/models/pipeline' type PartialInputVarType = PipelineInputVarType.textInput | PipelineInputVarType.number | PipelineInputVarType.select | PipelineInputVarType.checkbox @@ -15,7 +15,7 @@ const VAR_TYPE_MAP: Record = { export const useConfigurations = (datasourceNodeId: string) => { const pipelineId = useStore(state => state.pipelineId) - const { data: paramsConfig } = usePipelineProcessingParams({ + const { data: paramsConfig } = useDraftPipelineProcessingParams({ pipeline_id: pipelineId!, node_id: datasourceNodeId, }) diff --git a/web/app/components/rag-pipeline/components/panel/test-run/index.tsx b/web/app/components/rag-pipeline/components/panel/test-run/index.tsx index ac053853a4..4604d8d404 100644 --- a/web/app/components/rag-pipeline/components/panel/test-run/index.tsx +++ b/web/app/components/rag-pipeline/components/panel/test-run/index.tsx @@ -39,7 +39,6 @@ const TestRunPanel = () => { const allFileLoaded = (fileList.length > 0 && fileList.every(file => file.file.id)) const isVectorSpaceFull = plan.usage.vectorSpace >= plan.total.vectorSpace const isShowVectorSpaceFull = allFileLoaded && isVectorSpaceFull && enableBilling - const notSupportBatchUpload = enableBilling && plan.type === 'sandbox' const nextDisabled = useMemo(() => { if (!fileList.length) return true @@ -156,7 +155,7 @@ const TestRunPanel = () => { files={fileList} updateFile={updateFile} updateFileList={updateFileList} - notSupportBatchUpload={notSupportBatchUpload} + notSupportBatchUpload={false} // only support single file upload in test run /> )} {datasource?.type === DataSourceType.NOTION && ( diff --git a/web/i18n/en-US/dataset-pipeline.ts b/web/i18n/en-US/dataset-pipeline.ts index 645eeeeb63..f1bf5592e4 100644 --- a/web/i18n/en-US/dataset-pipeline.ts +++ b/web/i18n/en-US/dataset-pipeline.ts @@ -25,6 +25,8 @@ const translation = { useTemplate: 'Use this Knowledge Pipeline', backToDataSource: 'Back to Data Source', process: 'Process', + dataSource: 'Data Source', + saveAndProcess: 'Save & Process', }, knowledgeNameAndIcon: 'Knowledge name & icon', knowledgeNameAndIconPlaceholder: 'Please enter the name of the Knowledge Base', @@ -81,6 +83,9 @@ const translation = { stepOne: { preview: 'Preview', }, + stepTwo: { + chunkSettings: 'Chunk Settings', + }, characters: 'characters', }, } diff --git a/web/i18n/zh-Hans/dataset-pipeline.ts b/web/i18n/zh-Hans/dataset-pipeline.ts index d3e9ab420e..777acee90b 100644 --- a/web/i18n/zh-Hans/dataset-pipeline.ts +++ b/web/i18n/zh-Hans/dataset-pipeline.ts @@ -25,6 +25,8 @@ const translation = { useTemplate: '使用此知识库流水线', backToDataSource: '返回数据源', process: '处理', + dataSource: '数据源', + saveAndProcess: '保存并处理', }, knowledgeNameAndIcon: '知识库名称和图标', knowledgeNameAndIconPlaceholder: '请输入知识库名称', @@ -81,6 +83,9 @@ const translation = { stepOne: { preview: '预览', }, + stepTwo: { + chunkSettings: '分段设置', + }, characters: '字符', }, } diff --git a/web/service/use-pipeline.ts b/web/service/use-pipeline.ts index d9b7afc2b0..58b2a32756 100644 --- a/web/service/use-pipeline.ts +++ b/web/service/use-pipeline.ts @@ -132,13 +132,26 @@ export const useDatasourceNodeRun = ( }) } -// Get the config of shared input fields -export const usePipelineProcessingParams = (params: PipelineProcessingParamsRequest) => { +export const useDraftPipelineProcessingParams = (params: PipelineProcessingParamsRequest) => { const { pipeline_id, node_id } = params return useQuery({ queryKey: [NAME_SPACE, 'pipeline-processing-params', pipeline_id], queryFn: () => { - return get(`/rag/pipelines/${pipeline_id}/workflows/processing/parameters`, { + return get(`/rag/pipelines/${pipeline_id}/workflows/draft/processing/parameters`, { + params: { + node_id, + }, + }) + }, + }) +} + +export const usePublishedPipelineProcessingParams = (params: PipelineProcessingParamsRequest) => { + const { pipeline_id, node_id } = params + return useQuery({ + queryKey: [NAME_SPACE, 'pipeline-processing-params', pipeline_id], + queryFn: () => { + return get(`/rag/pipelines/${pipeline_id}/workflows/published/processing/parameters`, { params: { node_id, },