2025-05-21 16:37:02 +08:00
|
|
|
'use client'
|
|
|
|
import { useCallback, useMemo, useState } from 'react'
|
2025-05-22 14:49:40 +08:00
|
|
|
import DataSourceOptions from './data-source-options'
|
|
|
|
import type { CrawlResultItem, CustomFile as File, FileItem } from '@/models/datasets'
|
2025-05-21 16:37:02 +08:00
|
|
|
import { DataSourceType } from '@/models/datasets'
|
2025-05-22 14:49:40 +08:00
|
|
|
import LocalFile from '@/app/components/rag-pipeline/components/panel/test-run/data-source/local-file'
|
2025-05-21 16:37:02 +08:00
|
|
|
import produce from 'immer'
|
|
|
|
import { useProviderContextSelector } from '@/context/provider-context'
|
|
|
|
import { DataSourceProvider, type NotionPage } from '@/models/common'
|
|
|
|
import Notion from '@/app/components/rag-pipeline/components/panel/test-run/data-source/notion'
|
2025-05-22 14:49:40 +08:00
|
|
|
import VectorSpaceFull from '@/app/components/billing/vector-space-full'
|
2025-05-21 16:37:02 +08:00
|
|
|
import FireCrawl from '@/app/components/rag-pipeline/components/panel/test-run/data-source/website/firecrawl'
|
|
|
|
import JinaReader from '@/app/components/rag-pipeline/components/panel/test-run/data-source/website/jina-reader'
|
|
|
|
import WaterCrawl from '@/app/components/rag-pipeline/components/panel/test-run/data-source/website/water-crawl'
|
2025-05-22 14:49:40 +08:00
|
|
|
import Actions from './data-source/actions'
|
2025-05-21 16:37:02 +08:00
|
|
|
import DocumentProcessing from '@/app/components/rag-pipeline/components/panel/test-run/document-processing'
|
2025-05-22 14:49:40 +08:00
|
|
|
import { useTranslation } from 'react-i18next'
|
|
|
|
import type { Datasource } from '@/app/components/rag-pipeline/components/panel/test-run/types'
|
2025-05-21 16:37:02 +08:00
|
|
|
import LeftHeader from './left-header'
|
2025-05-22 14:49:40 +08:00
|
|
|
import { usePublishedPipelineInfo } from '@/service/use-pipeline'
|
|
|
|
import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
|
|
|
|
import Loading from '@/app/components/base/loading'
|
|
|
|
import type { Node } from '@/app/components/workflow/types'
|
|
|
|
import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
|
|
|
|
import FilePreview from './preview/file-preview'
|
|
|
|
import NotionPagePreview from './preview/notion-page-preview'
|
|
|
|
import WebsitePreview from './preview/web-preview'
|
2025-05-21 16:37:02 +08:00
|
|
|
|
|
|
|
const TestRunPanel = () => {
|
|
|
|
const { t } = useTranslation()
|
|
|
|
const [currentStep, setCurrentStep] = useState(1)
|
|
|
|
const [datasource, setDatasource] = useState<Datasource>()
|
|
|
|
const [fileList, setFiles] = useState<FileItem[]>([])
|
|
|
|
const [notionPages, setNotionPages] = useState<NotionPage[]>([])
|
|
|
|
const [websitePages, setWebsitePages] = useState<CrawlResultItem[]>([])
|
|
|
|
const [websiteCrawlJobId, setWebsiteCrawlJobId] = useState('')
|
2025-05-22 14:49:40 +08:00
|
|
|
const [currentFile, setCurrentFile] = useState<File | undefined>()
|
|
|
|
const [currentNotionPage, setCurrentNotionPage] = useState<NotionPage | undefined>()
|
|
|
|
const [currentWebsite, setCurrentWebsite] = useState<CrawlResultItem | undefined>()
|
2025-05-21 16:37:02 +08:00
|
|
|
|
|
|
|
const plan = useProviderContextSelector(state => state.plan)
|
|
|
|
const enableBilling = useProviderContextSelector(state => state.enableBilling)
|
2025-05-22 14:49:40 +08:00
|
|
|
const pipelineId = useDatasetDetailContextWithSelector(s => s.dataset?.pipeline_id)
|
2025-05-21 16:37:02 +08:00
|
|
|
|
2025-05-22 14:49:40 +08:00
|
|
|
const { data: pipelineInfo, isFetching: isFetchingPipelineInfo } = usePublishedPipelineInfo(pipelineId || '')
|
2025-05-21 16:37:02 +08:00
|
|
|
|
|
|
|
const allFileLoaded = (fileList.length > 0 && fileList.every(file => file.file.id))
|
|
|
|
const isVectorSpaceFull = plan.usage.vectorSpace >= plan.total.vectorSpace
|
|
|
|
const isShowVectorSpaceFull = allFileLoaded && isVectorSpaceFull && enableBilling
|
|
|
|
const notSupportBatchUpload = enableBilling && plan.type === 'sandbox'
|
|
|
|
const nextDisabled = useMemo(() => {
|
|
|
|
if (!fileList.length)
|
|
|
|
return true
|
|
|
|
if (fileList.some(file => !file.file.id))
|
|
|
|
return true
|
|
|
|
return isShowVectorSpaceFull
|
|
|
|
}, [fileList, isShowVectorSpaceFull])
|
|
|
|
|
|
|
|
const nextBtnDisabled = useMemo(() => {
|
|
|
|
if (!datasource) return true
|
|
|
|
if (datasource.type === DataSourceType.FILE)
|
|
|
|
return nextDisabled
|
|
|
|
if (datasource.type === DataSourceType.NOTION)
|
|
|
|
return isShowVectorSpaceFull || !notionPages.length
|
|
|
|
if (datasource.type === DataSourceProvider.fireCrawl
|
|
|
|
|| datasource.type === DataSourceProvider.jinaReader
|
|
|
|
|| datasource.type === DataSourceProvider.waterCrawl)
|
|
|
|
return isShowVectorSpaceFull || !websitePages.length
|
|
|
|
return false
|
|
|
|
}, [datasource, nextDisabled, isShowVectorSpaceFull, notionPages.length, websitePages.length])
|
|
|
|
|
|
|
|
const updateFile = (fileItem: FileItem, progress: number, list: FileItem[]) => {
|
|
|
|
const newList = produce(list, (draft) => {
|
|
|
|
const targetIndex = draft.findIndex(file => file.fileID === fileItem.fileID)
|
|
|
|
draft[targetIndex] = {
|
|
|
|
...draft[targetIndex],
|
|
|
|
progress,
|
|
|
|
}
|
|
|
|
})
|
|
|
|
setFiles(newList)
|
|
|
|
}
|
|
|
|
|
2025-05-22 14:49:40 +08:00
|
|
|
const updateFileList = useCallback((preparedFiles: FileItem[]) => {
|
2025-05-21 16:37:02 +08:00
|
|
|
setFiles(preparedFiles)
|
2025-05-22 14:49:40 +08:00
|
|
|
}, [])
|
2025-05-21 16:37:02 +08:00
|
|
|
|
2025-05-22 14:49:40 +08:00
|
|
|
const updateNotionPages = useCallback((value: NotionPage[]) => {
|
2025-05-21 16:37:02 +08:00
|
|
|
setNotionPages(value)
|
2025-05-22 14:49:40 +08:00
|
|
|
}, [])
|
|
|
|
|
|
|
|
const updateCurrentFile = useCallback((file: File) => {
|
|
|
|
setCurrentFile(file)
|
|
|
|
}, [])
|
|
|
|
|
|
|
|
const hideFilePreview = useCallback(() => {
|
|
|
|
setCurrentFile(undefined)
|
|
|
|
}, [])
|
|
|
|
|
|
|
|
const updateCurrentPage = useCallback((page: NotionPage) => {
|
|
|
|
setCurrentNotionPage(page)
|
|
|
|
}, [])
|
|
|
|
|
|
|
|
const hideNotionPagePreview = useCallback(() => {
|
|
|
|
setCurrentNotionPage(undefined)
|
|
|
|
}, [])
|
|
|
|
|
|
|
|
const updateCurrentWebsite = useCallback((website: CrawlResultItem) => {
|
|
|
|
setCurrentWebsite(website)
|
|
|
|
}, [])
|
|
|
|
|
|
|
|
const hideWebsitePreview = useCallback(() => {
|
|
|
|
setCurrentWebsite(undefined)
|
|
|
|
}, [])
|
2025-05-21 16:37:02 +08:00
|
|
|
|
|
|
|
const handleNextStep = useCallback(() => {
|
|
|
|
setCurrentStep(preStep => preStep + 1)
|
|
|
|
}, [])
|
|
|
|
|
|
|
|
const handleBackStep = useCallback(() => {
|
|
|
|
setCurrentStep(preStep => preStep - 1)
|
|
|
|
}, [])
|
|
|
|
|
|
|
|
const handleProcess = useCallback((data: Record<string, any>) => {
|
|
|
|
if (!datasource)
|
|
|
|
return
|
|
|
|
const datasourceInfo: Record<string, any> = {}
|
|
|
|
let datasource_type = ''
|
|
|
|
if (datasource.type === DataSourceType.FILE) {
|
|
|
|
datasource_type = 'local_file'
|
|
|
|
datasourceInfo.fileId = fileList.map(file => file.fileID)
|
|
|
|
}
|
|
|
|
if (datasource.type === DataSourceType.NOTION) {
|
|
|
|
datasource_type = 'online_document'
|
|
|
|
datasourceInfo.workspaceId = notionPages[0].workspace_id
|
|
|
|
datasourceInfo.page = notionPages.map((page) => {
|
|
|
|
const { workspace_id, ...rest } = page
|
|
|
|
return rest
|
|
|
|
})
|
|
|
|
}
|
|
|
|
if (datasource.type === DataSourceProvider.fireCrawl
|
|
|
|
|| datasource.type === DataSourceProvider.jinaReader
|
|
|
|
|| datasource.type === DataSourceProvider.waterCrawl) {
|
|
|
|
datasource_type = 'website_crawl'
|
|
|
|
datasourceInfo.jobId = websiteCrawlJobId
|
|
|
|
datasourceInfo.result = websitePages
|
|
|
|
}
|
2025-05-22 14:49:40 +08:00
|
|
|
// todo: Run Pipeline
|
|
|
|
console.log('datasource_type', datasource_type)
|
2025-05-21 16:37:02 +08:00
|
|
|
}, [datasource, fileList, notionPages, websiteCrawlJobId, websitePages])
|
|
|
|
|
2025-05-22 14:49:40 +08:00
|
|
|
if (isFetchingPipelineInfo) {
|
|
|
|
return (
|
|
|
|
<Loading type='app' />
|
|
|
|
)
|
|
|
|
}
|
|
|
|
|
2025-05-21 16:37:02 +08:00
|
|
|
return (
|
|
|
|
<div
|
|
|
|
className='relative flex h-[calc(100vh-56px)] min-w-[1512px] rounded-t-2xl border-t border-effects-highlight bg-background-default-subtle'
|
|
|
|
>
|
|
|
|
<div className='flex flex-1 flex-col px-14'>
|
|
|
|
<LeftHeader
|
|
|
|
title={t('datasetPipeline.addDocuments.title')}
|
|
|
|
currentStep={currentStep}
|
|
|
|
/>
|
|
|
|
<div className='grow overflow-y-auto'>
|
|
|
|
{
|
|
|
|
currentStep === 1 && (
|
2025-05-22 14:49:40 +08:00
|
|
|
<div className='flex flex-col gap-y-5 pt-4'>
|
|
|
|
<DataSourceOptions
|
2025-05-21 16:37:02 +08:00
|
|
|
datasourceNodeId={datasource?.nodeId || ''}
|
|
|
|
onSelect={setDatasource}
|
2025-05-22 14:49:40 +08:00
|
|
|
pipelineNodes={(pipelineInfo?.graph.nodes || []) as Node<DataSourceNodeType>[]}
|
|
|
|
/>
|
|
|
|
{datasource?.type === DataSourceType.FILE && (
|
|
|
|
<LocalFile
|
|
|
|
files={fileList}
|
|
|
|
updateFile={updateFile}
|
|
|
|
updateFileList={updateFileList}
|
|
|
|
onPreview={updateCurrentFile}
|
|
|
|
notSupportBatchUpload={notSupportBatchUpload}
|
|
|
|
/>
|
|
|
|
)}
|
|
|
|
{datasource?.type === DataSourceType.NOTION && (
|
|
|
|
<Notion
|
|
|
|
nodeId={datasource?.nodeId || ''}
|
|
|
|
notionPages={notionPages}
|
|
|
|
updateNotionPages={updateNotionPages}
|
|
|
|
canPreview
|
|
|
|
onPreview={updateCurrentPage}
|
|
|
|
/>
|
|
|
|
)}
|
|
|
|
{datasource?.type === DataSourceProvider.fireCrawl && (
|
|
|
|
<FireCrawl
|
|
|
|
nodeId={datasource?.nodeId || ''}
|
|
|
|
variables={datasource?.variables}
|
|
|
|
checkedCrawlResult={websitePages}
|
|
|
|
onCheckedCrawlResultChange={setWebsitePages}
|
|
|
|
onJobIdChange={setWebsiteCrawlJobId}
|
|
|
|
onPreview={updateCurrentWebsite}
|
|
|
|
/>
|
|
|
|
)}
|
|
|
|
{datasource?.type === DataSourceProvider.jinaReader && (
|
|
|
|
<JinaReader
|
|
|
|
nodeId={datasource?.nodeId || ''}
|
|
|
|
variables={datasource?.variables}
|
|
|
|
checkedCrawlResult={websitePages}
|
|
|
|
onCheckedCrawlResultChange={setWebsitePages}
|
|
|
|
onJobIdChange={setWebsiteCrawlJobId}
|
|
|
|
onPreview={updateCurrentWebsite}
|
|
|
|
/>
|
|
|
|
)}
|
|
|
|
{datasource?.type === DataSourceProvider.waterCrawl && (
|
|
|
|
<WaterCrawl
|
|
|
|
nodeId={datasource?.nodeId || ''}
|
|
|
|
variables={datasource?.variables}
|
|
|
|
checkedCrawlResult={websitePages}
|
|
|
|
onCheckedCrawlResultChange={setWebsitePages}
|
|
|
|
onJobIdChange={setWebsiteCrawlJobId}
|
|
|
|
onPreview={updateCurrentWebsite}
|
|
|
|
/>
|
|
|
|
)}
|
|
|
|
{isShowVectorSpaceFull && (
|
|
|
|
<VectorSpaceFull />
|
|
|
|
)}
|
2025-05-21 16:37:02 +08:00
|
|
|
<Actions disabled={nextBtnDisabled} handleNextStep={handleNextStep} />
|
2025-05-22 14:49:40 +08:00
|
|
|
</div>
|
2025-05-21 16:37:02 +08:00
|
|
|
)
|
|
|
|
}
|
|
|
|
{
|
|
|
|
currentStep === 2 && (
|
|
|
|
<DocumentProcessing
|
|
|
|
dataSourceNodeId={datasource?.nodeId || ''}
|
|
|
|
onProcess={handleProcess}
|
|
|
|
onBack={handleBackStep}
|
|
|
|
/>
|
|
|
|
)
|
|
|
|
}
|
|
|
|
</div>
|
|
|
|
</div>
|
|
|
|
{/* Preview */}
|
|
|
|
<div className='flex h-full flex-1 shrink-0 flex-col pl-2 pt-2'>
|
2025-05-22 14:49:40 +08:00
|
|
|
{
|
|
|
|
currentStep === 1 && (
|
|
|
|
<>
|
|
|
|
{currentFile && <FilePreview file={currentFile} hidePreview={hideFilePreview} />}
|
|
|
|
{currentNotionPage && <NotionPagePreview currentPage={currentNotionPage} hidePreview={hideNotionPagePreview} />}
|
|
|
|
{currentWebsite && <WebsitePreview payload={currentWebsite} hidePreview={hideWebsitePreview} />}
|
|
|
|
</>
|
|
|
|
)
|
|
|
|
}
|
2025-05-21 16:37:02 +08:00
|
|
|
</div>
|
|
|
|
</div>
|
|
|
|
)
|
|
|
|
}
|
|
|
|
|
|
|
|
export default TestRunPanel
|