2025-05-21 16:37:02 +08:00
|
|
|
import { useTranslation } from 'react-i18next'
|
|
|
|
import { AddDocumentsStep } from './types'
|
2025-06-26 13:46:12 +08:00
|
|
|
import type { DataSourceOption } from '@/app/components/rag-pipeline/components/panel/test-run/types'
|
2025-05-28 18:34:26 +08:00
|
|
|
import { useCallback, useMemo, useRef, useState } from 'react'
|
2025-05-22 14:49:40 +08:00
|
|
|
import { BlockEnum, type Node } from '@/app/components/workflow/types'
|
|
|
|
import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
|
2025-07-03 18:34:54 +08:00
|
|
|
import type { CrawlResult, CrawlResultItem } from '@/models/datasets'
|
2025-06-23 15:38:24 +08:00
|
|
|
import { CrawlStep } from '@/models/datasets'
|
2025-07-04 14:03:04 +08:00
|
|
|
import { useDataSourceStoreWithSelector } from './data-source/store'
|
2025-05-21 16:37:02 +08:00
|
|
|
|
|
|
|
export const useAddDocumentsSteps = () => {
|
|
|
|
const { t } = useTranslation()
|
2025-05-28 18:34:26 +08:00
|
|
|
const [currentStep, setCurrentStep] = useState(1)
|
|
|
|
|
|
|
|
const handleNextStep = useCallback(() => {
|
|
|
|
setCurrentStep(preStep => preStep + 1)
|
|
|
|
}, [])
|
|
|
|
|
|
|
|
const handleBackStep = useCallback(() => {
|
|
|
|
setCurrentStep(preStep => preStep - 1)
|
|
|
|
}, [])
|
|
|
|
|
2025-05-21 16:37:02 +08:00
|
|
|
const steps = [
|
|
|
|
{
|
|
|
|
label: t('datasetPipeline.addDocuments.steps.chooseDatasource'),
|
|
|
|
value: AddDocumentsStep.dataSource,
|
|
|
|
},
|
|
|
|
{
|
2025-05-22 23:05:58 +08:00
|
|
|
label: t('datasetPipeline.addDocuments.steps.processDocuments'),
|
2025-05-21 16:37:02 +08:00
|
|
|
value: AddDocumentsStep.processDocuments,
|
|
|
|
},
|
|
|
|
{
|
2025-05-22 23:05:58 +08:00
|
|
|
label: t('datasetPipeline.addDocuments.steps.processingDocuments'),
|
2025-05-21 16:37:02 +08:00
|
|
|
value: AddDocumentsStep.processingDocuments,
|
|
|
|
},
|
|
|
|
]
|
2025-05-28 18:34:26 +08:00
|
|
|
|
|
|
|
return {
|
|
|
|
steps,
|
|
|
|
currentStep,
|
|
|
|
handleNextStep,
|
|
|
|
handleBackStep,
|
|
|
|
}
|
2025-05-21 16:37:02 +08:00
|
|
|
}
|
2025-05-22 14:49:40 +08:00
|
|
|
|
|
|
|
export const useDatasourceOptions = (pipelineNodes: Node<DataSourceNodeType>[]) => {
|
2025-05-27 11:01:38 +08:00
|
|
|
const datasourceNodes = pipelineNodes.filter(node => node.data.type === BlockEnum.DataSource)
|
2025-05-22 14:49:40 +08:00
|
|
|
|
|
|
|
const options = useMemo(() => {
|
|
|
|
const options: DataSourceOption[] = []
|
2025-05-27 11:01:38 +08:00
|
|
|
datasourceNodes.forEach((node) => {
|
|
|
|
const label = node.data.title
|
|
|
|
options.push({
|
|
|
|
label,
|
|
|
|
value: node.id,
|
2025-05-27 14:17:55 +08:00
|
|
|
data: node.data,
|
2025-05-27 11:01:38 +08:00
|
|
|
})
|
2025-05-22 14:49:40 +08:00
|
|
|
})
|
|
|
|
return options
|
2025-05-27 11:01:38 +08:00
|
|
|
}, [datasourceNodes])
|
2025-05-22 14:49:40 +08:00
|
|
|
|
2025-06-26 13:46:12 +08:00
|
|
|
return options
|
2025-05-22 14:49:40 +08:00
|
|
|
}
|
2025-05-28 18:34:26 +08:00
|
|
|
|
|
|
|
export const useLocalFile = () => {
|
2025-07-04 14:03:04 +08:00
|
|
|
const fileList = useDataSourceStoreWithSelector(state => state.localFileList)
|
|
|
|
const previewFileRef = useDataSourceStoreWithSelector(state => state.previewLocalFileRef)
|
|
|
|
const currentLocalFile = useDataSourceStoreWithSelector(state => state.currentLocalFile)
|
|
|
|
const setCurrentFile = useDataSourceStoreWithSelector(state => state.setCurrentLocalFile)
|
2025-05-28 18:34:26 +08:00
|
|
|
|
|
|
|
const allFileLoaded = useMemo(() => (fileList.length > 0 && fileList.every(file => file.file.id)), [fileList])
|
|
|
|
|
2025-07-03 18:34:54 +08:00
|
|
|
const hidePreviewLocalFile = useCallback(() => {
|
2025-05-28 18:34:26 +08:00
|
|
|
setCurrentFile(undefined)
|
2025-07-03 18:34:54 +08:00
|
|
|
}, [setCurrentFile])
|
2025-05-28 18:34:26 +08:00
|
|
|
|
|
|
|
return {
|
|
|
|
fileList,
|
2025-07-03 18:34:54 +08:00
|
|
|
previewFileRef,
|
2025-05-28 18:34:26 +08:00
|
|
|
allFileLoaded,
|
2025-07-03 18:34:54 +08:00
|
|
|
currentLocalFile,
|
|
|
|
hidePreviewLocalFile,
|
2025-05-28 18:34:26 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2025-06-06 10:08:19 +08:00
|
|
|
export const useOnlineDocuments = () => {
|
2025-07-04 14:03:04 +08:00
|
|
|
const onlineDocuments = useDataSourceStoreWithSelector(state => state.onlineDocuments)
|
|
|
|
const previewOnlineDocumentRef = useDataSourceStoreWithSelector(state => state.previewOnlineDocumentRef)
|
|
|
|
const setCurrentDocument = useDataSourceStoreWithSelector(state => state.setCurrentDocument)
|
|
|
|
const currentDocument = useDataSourceStoreWithSelector(state => state.currentDocument)
|
2025-07-01 16:32:21 +08:00
|
|
|
|
2025-07-03 18:34:54 +08:00
|
|
|
const hidePreviewOnlineDocument = useCallback(() => {
|
2025-06-23 16:31:09 +08:00
|
|
|
setCurrentDocument(undefined)
|
2025-07-03 18:34:54 +08:00
|
|
|
}, [setCurrentDocument])
|
2025-05-28 18:34:26 +08:00
|
|
|
|
|
|
|
return {
|
2025-06-06 10:08:19 +08:00
|
|
|
onlineDocuments,
|
2025-06-23 16:31:09 +08:00
|
|
|
currentDocument,
|
2025-07-03 18:34:54 +08:00
|
|
|
previewOnlineDocumentRef,
|
|
|
|
hidePreviewOnlineDocument,
|
2025-05-28 18:34:26 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
export const useWebsiteCrawl = () => {
|
|
|
|
const [websitePages, setWebsitePages] = useState<CrawlResultItem[]>([])
|
|
|
|
const [currentWebsite, setCurrentWebsite] = useState<CrawlResultItem | undefined>()
|
2025-06-23 15:38:24 +08:00
|
|
|
const [crawlResult, setCrawlResult] = useState<CrawlResult | undefined>()
|
|
|
|
const [step, setStep] = useState<CrawlStep>(CrawlStep.init)
|
|
|
|
const [previewIndex, setPreviewIndex] = useState<number>(-1)
|
2025-05-28 18:34:26 +08:00
|
|
|
|
|
|
|
const previewWebsitePage = useRef<CrawlResultItem>(websitePages[0])
|
|
|
|
|
2025-06-23 15:38:24 +08:00
|
|
|
const updateCurrentWebsite = useCallback((website: CrawlResultItem, index: number) => {
|
2025-05-28 18:34:26 +08:00
|
|
|
setCurrentWebsite(website)
|
2025-06-23 15:38:24 +08:00
|
|
|
setPreviewIndex(index)
|
2025-05-28 18:34:26 +08:00
|
|
|
}, [])
|
|
|
|
|
|
|
|
const hideWebsitePreview = useCallback(() => {
|
|
|
|
setCurrentWebsite(undefined)
|
2025-06-23 15:38:24 +08:00
|
|
|
setPreviewIndex(-1)
|
2025-05-28 18:34:26 +08:00
|
|
|
}, [])
|
|
|
|
|
2025-06-06 17:00:34 +08:00
|
|
|
const updataCheckedCrawlResultChange = useCallback((checkedCrawlResult: CrawlResultItem[]) => {
|
|
|
|
setWebsitePages(checkedCrawlResult)
|
|
|
|
previewWebsitePage.current = checkedCrawlResult[0]
|
|
|
|
}, [])
|
|
|
|
|
2025-05-28 18:34:26 +08:00
|
|
|
return {
|
|
|
|
websitePages,
|
2025-06-23 15:38:24 +08:00
|
|
|
crawlResult,
|
|
|
|
setCrawlResult,
|
|
|
|
step,
|
|
|
|
setStep,
|
2025-05-28 18:34:26 +08:00
|
|
|
previewWebsitePage,
|
2025-06-06 17:00:34 +08:00
|
|
|
updataCheckedCrawlResultChange,
|
2025-05-28 18:34:26 +08:00
|
|
|
currentWebsite,
|
|
|
|
updateCurrentWebsite,
|
2025-06-23 15:38:24 +08:00
|
|
|
previewIndex,
|
2025-05-28 18:34:26 +08:00
|
|
|
hideWebsitePreview,
|
|
|
|
}
|
|
|
|
}
|
2025-07-01 16:54:44 +08:00
|
|
|
|
|
|
|
export const useOnlineDrive = () => {
|
2025-07-03 18:34:54 +08:00
|
|
|
return {}
|
2025-07-01 16:54:44 +08:00
|
|
|
}
|