yujunjun/dify
1
0
Fork 0
mirror of https://github.com/langgenius/dify.git synced 2025-11-02 03:43:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: refactor data source handling and add form for document processing

Browse Source
This commit is contained in:
twwu 2025-06-04 16:50:27 +08:00
parent c155afac29
commit 9934eac15c
10 changed files with 109 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
web/app/components/datasets/documents/create-from-pipeline/hooks.ts
View File

@ -51,7 +51,6 @@ export const useDatasourceOptions = (pipelineNodes: Node<DataSourceNodeType>[])
return {
nodeId: node.id,
type: node.data.provider_type as DatasourceType,
variables: node.data.variables || [],
description: node.data.desc || '',
docTitle: '', // todo: Add docTitle and docLink if needed, or remove these properties if not used
docLink: '',

2
web/app/components/datasets/documents/create-from-pipeline/index.tsx
View File

@ -274,7 +274,7 @@ const CreateFormPipeline = () => {
{datasource?.type === DatasourceType.websiteCrawl && (
<WebsiteCrawl
nodeId={datasource?.nodeId || ''}
variables={datasource?.variables}
variables={[]} // todo: replace with actual variables if needed
headerInfo={{
title: datasource.description,
docTitle: datasource.docTitle || '',

20
web/app/components/rag-pipeline/components/input-field/preview/data-source.tsx
View File

@ -1,7 +1,18 @@
import React from 'react'
import { useTranslation } from 'react-i18next'
import DataSourceOptions from '../../panel/test-run/data-source-options'
import Form from './form'
import type { Datasource } from '../../panel/test-run/types'
const DataSource = () => {
type DatasourceProps = {
onSelect: (dataSource: Datasource) => void
datasourceNodeId: string
}
const DataSource = ({
onSelect: setDatasource,
datasourceNodeId,
}: DatasourceProps) => {
const { t } = useTranslation()
return (
@ -9,6 +20,13 @@ const DataSource = () => {
<div className='system-sm-semibold-uppercase px-4 pt-2 text-text-secondary'>
{t('datasetPipeline.inputFieldPanel.preview.stepOneTitle')}
</div>
<div className='px-4 py-2'>
<DataSourceOptions
onSelect={setDatasource}
datasourceNodeId={datasourceNodeId}
/>
</div>
<Form variables={[]} />
</div>
)
}

41
web/app/components/rag-pipeline/components/input-field/preview/form.tsx Normal file
View File

@ -0,0 +1,41 @@
import { useAppForm } from '@/app/components/base/form'
import BaseField from '@/app/components/base/form/form-scenarios/base/field'
import type { RAGPipelineVariables } from '@/models/pipeline'
import { useConfigurations, useInitialData } from '../../panel/test-run/data-source/website-crawl/base/options/hooks'
type FormProps = {
variables: RAGPipelineVariables
}
const Form = ({
variables,
}: FormProps) => {
const initialData = useInitialData(variables)
const configurations = useConfigurations(variables)
const form = useAppForm({
defaultValues: initialData,
})
return (
<form
className='w-full'
onSubmit={(e) => {
e.preventDefault()
e.stopPropagation()
}}
>
<div className='flex flex-col gap-y-3 px-4 py-3'>
{configurations.map((config, index) => {
const FieldComponent = BaseField({
initialData,
config,
})
return <FieldComponent key={index} form={form} />
})}
</div>
</form>
)
}
export default Form

14
web/app/components/rag-pipeline/components/input-field/preview/index.tsx
View File

@ -1,7 +1,12 @@
import { useState } from 'react'
import { RiCloseLine } from '@remixicon/react'
import DialogWrapper from './dialog-wrapper'
import { useTranslation } from 'react-i18next'
import Badge from '@/app/components/base/badge'
import DataSource from './data-source'
import Divider from '@/app/components/base/divider'
import ProcessDocuments from './process-documents'
import type { Datasource } from '../../panel/test-run/types'
type PreviewPanelProps = {
show: boolean
@ -13,6 +18,7 @@ const PreviewPanel = ({
onClose,
}: PreviewPanelProps) => {
const { t } = useTranslation()
const [datasource, setDatasource] = useState<Datasource>()
return (
<DialogWrapper
@ -34,6 +40,14 @@ const PreviewPanel = ({
<RiCloseLine className='size-4 text-text-tertiary' />
</button>
</div>
<DataSource
onSelect={setDatasource}
datasourceNodeId={datasource?.nodeId || ''}
/>
<div className='px-4 py-2'>
<Divider type='horizontal' className='bg-divider-subtle' />
</div>
<ProcessDocuments dataSourceNodeId={datasource?.nodeId || ''} />
</DialogWrapper>
)
}

31
web/app/components/rag-pipeline/components/input-field/preview/process-documents.tsx Normal file
View File

@ -0,0 +1,31 @@
import React from 'react'
import { useTranslation } from 'react-i18next'
import { useStore } from '@/app/components/workflow/store'
import { useDraftPipelineProcessingParams } from '@/service/use-pipeline'
import Form from './form'
type ProcessDocumentsProps = {
dataSourceNodeId: string
}
const ProcessDocuments = ({
dataSourceNodeId,
}: ProcessDocumentsProps) => {
const { t } = useTranslation()
const pipelineId = useStore(state => state.pipelineId)
const { data: paramsConfig } = useDraftPipelineProcessingParams({
pipeline_id: pipelineId!,
node_id: dataSourceNodeId,
})
return (
<div className='flex flex-col'>
<div className='system-sm-semibold-uppercase px-4 pt-2 text-text-secondary'>
{t('datasetPipeline.inputFieldPanel.preview.stepTwoTitle')}
</div>
<Form variables={paramsConfig?.variables || []} />
</div>
)
}
export default React.memo(ProcessDocuments)

1
web/app/components/rag-pipeline/components/panel/test-run/hooks.ts
View File

@ -49,7 +49,6 @@ export const useDatasourceOptions = () => {
return {
nodeId: node.id,
type: node.data.provider_type as DatasourceType,
variables: node.data.variables || [],
description: '', // todo: Add description
docTitle: '', // todo: Add docTitle and docLink
docLink: '',

2
web/app/components/rag-pipeline/components/panel/test-run/index.tsx
View File

@ -139,7 +139,7 @@ const TestRunPanel = () => {
{datasource?.type === DatasourceType.websiteCrawl && (
<WebsiteCrawl
nodeId={datasource?.nodeId || ''}
variables={datasource?.variables}
variables={[]} // todo: replace with actual variables if needed
checkedCrawlResult={websitePages}
headerInfo={{
title: datasource.description,

3
web/app/components/rag-pipeline/components/panel/test-run/types.ts
View File

@ -1,5 +1,5 @@
import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
import type { DatasourceType, RAGPipelineVariables } from '@/models/pipeline'
import type { DatasourceType } from '@/models/pipeline'
export enum TestRunStep {
dataSource = 'dataSource',
@ -15,7 +15,6 @@ export type DataSourceOption = {
export type Datasource = {
nodeId: string
type: DatasourceType
variables: RAGPipelineVariables
description: string
docTitle?: string
docLink?: string

1
web/service/use-pipeline.ts
View File

@ -148,6 +148,7 @@ export const useDraftPipelineProcessingParams = (params: PipelineProcessingParam
})
},
staleTime: 0,
enabled: !!pipeline_id && !!node_id,
})
}