diff --git a/web/app/components/datasets/documents/create-from-pipeline/data-source-options/index.tsx b/web/app/components/datasets/documents/create-from-pipeline/data-source-options/index.tsx index d0a410f5e0..cb88aec1da 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/data-source-options/index.tsx +++ b/web/app/components/datasets/documents/create-from-pipeline/data-source-options/index.tsx @@ -16,14 +16,18 @@ const DataSourceOptions = ({ datasourceNodeId, onSelect, }: DataSourceOptionsProps) => { - const { datasources, options } = useDatasourceOptions(pipelineNodes) + const options = useDatasourceOptions(pipelineNodes) const handelSelect = useCallback((value: string) => { - const selectedOption = datasources.find(option => option.nodeId === value) + const selectedOption = options.find(option => option.value === value) if (!selectedOption) return - onSelect(selectedOption) - }, [datasources, onSelect]) + const datasource = { + nodeId: selectedOption.value, + nodeData: selectedOption.data, + } + onSelect(datasource) + }, [onSelect, options]) useEffect(() => { if (options.length > 0 && !datasourceNodeId) diff --git a/web/app/components/datasets/documents/create-from-pipeline/hooks.ts b/web/app/components/datasets/documents/create-from-pipeline/hooks.ts index 01dfd040df..07351930d0 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/hooks.ts +++ b/web/app/components/datasets/documents/create-from-pipeline/hooks.ts @@ -1,10 +1,9 @@ import { useTranslation } from 'react-i18next' import { AddDocumentsStep } from './types' -import type { DataSourceOption, Datasource } from '@/app/components/rag-pipeline/components/panel/test-run/types' +import type { DataSourceOption } from '@/app/components/rag-pipeline/components/panel/test-run/types' import { useCallback, useMemo, useRef, useState } from 'react' import { BlockEnum, type Node } from '@/app/components/workflow/types' import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types' -import type { DatasourceType } from '@/models/pipeline' import type { CrawlResult, CrawlResultItem, DocumentItem, FileItem } from '@/models/datasets' import { CrawlStep } from '@/models/datasets' import produce from 'immer' @@ -47,18 +46,6 @@ export const useAddDocumentsSteps = () => { export const useDatasourceOptions = (pipelineNodes: Node[]) => { const datasourceNodes = pipelineNodes.filter(node => node.data.type === BlockEnum.DataSource) - const datasources: Datasource[] = useMemo(() => { - return datasourceNodes.map((node) => { - return { - nodeId: node.id, - type: node.data.provider_type as DatasourceType, - description: node.data.datasource_label, - docTitle: 'How to use?', - docLink: '', - fileExtensions: node.data.fileExtensions || [], - } - }) - }, [datasourceNodes]) const options = useMemo(() => { const options: DataSourceOption[] = [] @@ -70,10 +57,29 @@ export const useDatasourceOptions = (pipelineNodes: Node[]) data: node.data, }) }) + // todo: delete mock data + options.push({ + label: 'Google Drive', + value: '123456', + // @ts-expect-error mock data + data: { + datasource_parameters: {}, + datasource_configurations: {}, + type: BlockEnum.DataSource, + title: 'Google Drive', + plugin_id: 'langgenius/google-drive', + provider_type: 'online_drive', + provider_name: 'google_drive', + datasource_name: 'google-drive', + datasource_label: 'Google Drive', + selected: false, + }, + }) + return options }, [datasourceNodes]) - return { datasources, options } + return options } export const useLocalFile = () => { diff --git a/web/app/components/datasets/documents/create-from-pipeline/index.tsx b/web/app/components/datasets/documents/create-from-pipeline/index.tsx index 0c90371850..de763cb937 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/index.tsx +++ b/web/app/components/datasets/documents/create-from-pipeline/index.tsx @@ -84,17 +84,18 @@ const CreateFormPipeline = () => { const isVectorSpaceFull = plan.usage.vectorSpace >= plan.total.vectorSpace const isShowVectorSpaceFull = allFileLoaded && isVectorSpaceFull && enableBilling const notSupportBatchUpload = enableBilling && plan.type === 'sandbox' + const datasourceType = datasource?.nodeData.provider_type const nextBtnDisabled = useMemo(() => { if (!datasource) return true - if (datasource.type === DatasourceType.localFile) + if (datasourceType === DatasourceType.localFile) return isShowVectorSpaceFull || !fileList.length || fileList.some(file => !file.file.id) - if (datasource.type === DatasourceType.onlineDocument) + if (datasourceType === DatasourceType.onlineDocument) return isShowVectorSpaceFull || !onlineDocuments.length - if (datasource.type === DatasourceType.websiteCrawl) + if (datasourceType === DatasourceType.websiteCrawl) return isShowVectorSpaceFull || !websitePages.length return false - }, [datasource, isShowVectorSpaceFull, fileList, onlineDocuments.length, websitePages.length]) + }, [datasource, datasourceType, isShowVectorSpaceFull, fileList, onlineDocuments.length, websitePages.length]) const { mutateAsync: runPublishedPipeline, isIdle, isPending } = useRunPublishedPipeline() @@ -102,7 +103,7 @@ const CreateFormPipeline = () => { if (!datasource) return const datasourceInfoList: Record[] = [] - if (datasource.type === DatasourceType.localFile) { + if (datasourceType === DatasourceType.localFile) { const { id, name, type, size, extension, mime_type } = previewFile.current as File const documentInfo = { related_id: id, @@ -116,7 +117,7 @@ const CreateFormPipeline = () => { } datasourceInfoList.push(documentInfo) } - if (datasource.type === DatasourceType.onlineDocument) { + if (datasourceType === DatasourceType.onlineDocument) { const { workspace_id, ...rest } = previewOnlineDocument.current const documentInfo = { workspace_id, @@ -124,13 +125,13 @@ const CreateFormPipeline = () => { } datasourceInfoList.push(documentInfo) } - if (datasource.type === DatasourceType.websiteCrawl) + if (datasourceType === DatasourceType.websiteCrawl) datasourceInfoList.push(previewWebsitePage.current) await runPublishedPipeline({ pipeline_id: pipelineId!, inputs: data, start_node_id: datasource.nodeId, - datasource_type: datasource.type, + datasource_type: datasourceType as DatasourceType, datasource_info_list: datasourceInfoList, is_preview: true, }, { @@ -138,13 +139,13 @@ const CreateFormPipeline = () => { setEstimateData((res as PublishedPipelineRunPreviewResponse).data.outputs) }, }) - }, [datasource, pipelineId, previewFile, previewOnlineDocument, previewWebsitePage, runPublishedPipeline]) + }, [datasource, datasourceType, pipelineId, previewFile, previewOnlineDocument, previewWebsitePage, runPublishedPipeline]) const handleProcess = useCallback(async (data: Record) => { if (!datasource) return const datasourceInfoList: Record[] = [] - if (datasource.type === DatasourceType.localFile) { + if (datasourceType === DatasourceType.localFile) { fileList.forEach((file) => { const { id, name, type, size, extension, mime_type } = file.file const documentInfo = { @@ -160,7 +161,7 @@ const CreateFormPipeline = () => { datasourceInfoList.push(documentInfo) }) } - if (datasource.type === DatasourceType.onlineDocument) { + if (datasourceType === DatasourceType.onlineDocument) { onlineDocuments.forEach((page) => { const { workspace_id, ...rest } = page const documentInfo = { @@ -170,7 +171,7 @@ const CreateFormPipeline = () => { datasourceInfoList.push(documentInfo) }) } - if (datasource.type === DatasourceType.websiteCrawl) { + if (datasourceType === DatasourceType.websiteCrawl) { websitePages.forEach((websitePage) => { datasourceInfoList.push(websitePage) }) @@ -179,7 +180,7 @@ const CreateFormPipeline = () => { pipeline_id: pipelineId!, inputs: data, start_node_id: datasource.nodeId, - datasource_type: datasource.type, + datasource_type: datasourceType as DatasourceType, datasource_info_list: datasourceInfoList, is_preview: false, }, { @@ -189,7 +190,7 @@ const CreateFormPipeline = () => { handleNextStep() }, }) - }, [datasource, fileList, handleNextStep, onlineDocuments, pipelineId, runPublishedPipeline, websitePages]) + }, [datasource, datasourceType, fileList, handleNextStep, onlineDocuments, pipelineId, runPublishedPipeline, websitePages]) const onClickProcess = useCallback(() => { isPreview.current = false @@ -246,38 +247,30 @@ const CreateFormPipeline = () => { onSelect={setDatasource} pipelineNodes={(pipelineInfo?.graph.nodes || []) as Node[]} /> - {datasource?.type === DatasourceType.localFile && ( + {datasourceType === DatasourceType.localFile && ( )} - {datasource?.type === DatasourceType.onlineDocument && ( + {datasourceType === DatasourceType.onlineDocument && ( )} - {datasource?.type === DatasourceType.websiteCrawl && ( + {datasourceType === DatasourceType.websiteCrawl && ( { currentStep === 2 && ( {
file.file)} onlineDocuments={onlineDocuments} websitePages={websitePages} diff --git a/web/app/components/rag-pipeline/components/panel/test-run/data-source-options/index.tsx b/web/app/components/rag-pipeline/components/panel/test-run/data-source-options/index.tsx index f9a0b4c9b8..717138dc2f 100644 --- a/web/app/components/rag-pipeline/components/panel/test-run/data-source-options/index.tsx +++ b/web/app/components/rag-pipeline/components/panel/test-run/data-source-options/index.tsx @@ -12,14 +12,18 @@ const DataSourceOptions = ({ dataSourceNodeId, onSelect, }: DataSourceOptionsProps) => { - const { datasources, options } = useDatasourceOptions() + const options = useDatasourceOptions() const handelSelect = useCallback((value: string) => { - const selectedOption = datasources.find(option => option.nodeId === value) + const selectedOption = options.find(option => option.value === value) if (!selectedOption) return - onSelect(selectedOption) - }, [datasources, onSelect]) + const datasource = { + nodeId: selectedOption.value, + nodeData: selectedOption.data, + } + onSelect(datasource) + }, [onSelect, options]) useEffect(() => { if (options.length > 0 && !dataSourceNodeId) @@ -33,9 +37,10 @@ const DataSourceOptions = ({ ))}
diff --git a/web/app/components/rag-pipeline/components/panel/test-run/data-source-options/option-card.tsx b/web/app/components/rag-pipeline/components/panel/test-run/data-source-options/option-card.tsx index 0e1dc0b3ff..27813a4bfd 100644 --- a/web/app/components/rag-pipeline/components/panel/test-run/data-source-options/option-card.tsx +++ b/web/app/components/rag-pipeline/components/panel/test-run/data-source-options/option-card.tsx @@ -1,4 +1,4 @@ -import React from 'react' +import React, { useCallback } from 'react' import cn from '@/utils/classnames' import BlockIcon from '@/app/components/workflow/block-icon' import { BlockEnum } from '@/app/components/workflow/types' @@ -7,19 +7,25 @@ import { useToolIcon } from '@/app/components/workflow/hooks' type OptionCardProps = { label: string + value: string selected: boolean nodeData: DataSourceNodeType - onClick?: () => void + onClick?: (value: string) => void } const OptionCard = ({ label, + value, selected, nodeData, onClick, }: OptionCardProps) => { const toolIcon = useToolIcon(nodeData) + const handleClickCard = useCallback(() => { + onClick?.(value) + }, [value, onClick]) + return (
void canPreview?: boolean @@ -17,7 +14,7 @@ type OnlineDocumentsProps = { const OnlineDocuments = ({ nodeId, - headerInfo, + nodeData, onlineDocuments, updateOnlineDocuments, canPreview = false, @@ -27,7 +24,7 @@ const OnlineDocuments = ({ return ( page.page_id)} onSelect={updateOnlineDocuments} canPreview={canPreview} diff --git a/web/app/components/rag-pipeline/components/panel/test-run/data-source/online-documents/online-document-selector.tsx b/web/app/components/rag-pipeline/components/panel/test-run/data-source/online-documents/online-document-selector.tsx index d89d214fb9..bafe71eb8a 100644 --- a/web/app/components/rag-pipeline/components/panel/test-run/data-source/online-documents/online-document-selector.tsx +++ b/web/app/components/rag-pipeline/components/panel/test-run/data-source/online-documents/online-document-selector.tsx @@ -9,6 +9,7 @@ import { DatasourceType } from '@/models/pipeline' import { ssePost } from '@/service/base' import Toast from '@/app/components/base/toast' import type { DataSourceNodeCompletedResponse } from '@/types/pipeline' +import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types' type OnlineDocumentSelectorProps = { value?: string[] @@ -18,11 +19,7 @@ type OnlineDocumentSelectorProps = { onPreview?: (selectedPage: NotionPage) => void isInPipeline?: boolean nodeId: string - headerInfo: { - title: string - docTitle: string - docLink: string - } + nodeData: DataSourceNodeType } const OnlineDocumentSelector = ({ @@ -33,7 +30,7 @@ const OnlineDocumentSelector = ({ onPreview, isInPipeline = false, nodeId, - headerInfo, + nodeData, }: OnlineDocumentSelectorProps) => { const pipelineId = useDatasetDetailContextWithSelector(s => s.dataset?.pipeline_id) const [documentsData, setDocumentsData] = useState([]) @@ -118,6 +115,14 @@ const OnlineDocumentSelector = ({ setCurrentWorkspaceId(firstWorkspaceId) }, [firstWorkspaceId]) + const headerInfo = useMemo(() => { + return { + title: nodeData.title, + docTitle: 'How to use?', + docLink: 'https://docs.dify.ai', + } + }, [nodeData]) + if (!documentsData?.length) return null diff --git a/web/app/components/rag-pipeline/components/panel/test-run/data-source/online-drive/connect/index.tsx b/web/app/components/rag-pipeline/components/panel/test-run/data-source/online-drive/connect/index.tsx new file mode 100644 index 0000000000..9287bb24bf --- /dev/null +++ b/web/app/components/rag-pipeline/components/panel/test-run/data-source/online-drive/connect/index.tsx @@ -0,0 +1,34 @@ +import BlockIcon from '@/app/components/workflow/block-icon' +import { useToolIcon } from '@/app/components/workflow/hooks' +import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types' +import { BlockEnum } from '@/app/components/workflow/types' + +type ConnectProps = { + nodeData: DataSourceNodeType +} + +const Connect = ({ + nodeData, +}: ConnectProps) => { + const toolIcon = useToolIcon(nodeData) + + return ( +
+
+ +
+

+ To connect your online drive, please follow the instructions provided by your service provider. +

+ +
+ ) +} + +export default Connect diff --git a/web/app/components/rag-pipeline/components/panel/test-run/data-source/online-drive/index.tsx b/web/app/components/rag-pipeline/components/panel/test-run/data-source/online-drive/index.tsx new file mode 100644 index 0000000000..2d2e10707a --- /dev/null +++ b/web/app/components/rag-pipeline/components/panel/test-run/data-source/online-drive/index.tsx @@ -0,0 +1,16 @@ +import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types' +import Connect from './connect' + +type OnlineDriveProps = { + nodeData: DataSourceNodeType +} + +const OnlineDrive = ({ + nodeData, +}: OnlineDriveProps) => { + return ( + + ) +} + +export default OnlineDrive diff --git a/web/app/components/rag-pipeline/components/panel/test-run/data-source/website-crawl/base/crawler.tsx b/web/app/components/rag-pipeline/components/panel/test-run/data-source/website-crawl/base/crawler.tsx index 187557dd2b..278c22918a 100644 --- a/web/app/components/rag-pipeline/components/panel/test-run/data-source/website-crawl/base/crawler.tsx +++ b/web/app/components/rag-pipeline/components/panel/test-run/data-source/website-crawl/base/crawler.tsx @@ -1,5 +1,5 @@ 'use client' -import React, { useCallback, useEffect, useRef, useState } from 'react' +import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react' import { useTranslation } from 'react-i18next' import type { CrawlResult, CrawlResultItem } from '@/models/datasets' import { CrawlStep } from '@/models/datasets' @@ -19,22 +19,19 @@ import type { DataSourceNodeCompletedResponse, DataSourceNodeProcessingResponse, } from '@/types/pipeline' +import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types' const I18N_PREFIX = 'datasetCreation.stepOne.website' export type CrawlerProps = { nodeId: string + nodeData: DataSourceNodeType crawlResult: CrawlResult | undefined setCrawlResult: (payload: CrawlResult) => void step: CrawlStep setStep: (step: CrawlStep) => void checkedCrawlResult: CrawlResultItem[] onCheckedCrawlResultChange: (payload: CrawlResultItem[]) => void - headerInfo: { - title: string - docTitle: string - docLink: string - } previewIndex?: number onPreview?: (payload: CrawlResultItem, index: number) => void isInPipeline?: boolean @@ -42,12 +39,12 @@ export type CrawlerProps = { const Crawler = ({ nodeId, + nodeData, crawlResult, setCrawlResult, step, setStep, checkedCrawlResult, - headerInfo, onCheckedCrawlResultChange, previewIndex, onPreview, @@ -125,6 +122,14 @@ const Crawler = ({ handleRun(value) }, [handleRun]) + const headerInfo = useMemo(() => { + return { + title: nodeData.title, + docTitle: 'How to use?', + docLink: 'https://docs.dify.ai', + } + }, [nodeData]) + return (
{ export const useDatasourceOptions = () => { const nodes = useNodes() const datasourceNodes = nodes.filter(node => node.data.type === BlockEnum.DataSource) - const datasources: Datasource[] = useMemo(() => { - return datasourceNodes.map((node) => { - return { - nodeId: node.id, - type: node.data.provider_type as DatasourceType, - description: node.data.datasource_label, - docTitle: 'How to use?', - docLink: '', - fileExtensions: node.data.fileExtensions || [], - } - }) - }, [datasourceNodes]) const options = useMemo(() => { const options: DataSourceOption[] = [] @@ -68,10 +55,28 @@ export const useDatasourceOptions = () => { data: node.data, }) }) + // todo: delete mock data + options.push({ + label: 'Google Drive', + value: '123456', + // @ts-expect-error mock data + data: { + datasource_parameters: {}, + datasource_configurations: {}, + type: BlockEnum.DataSource, + title: 'Google Drive', + plugin_id: 'langgenius/google-drive', + provider_type: 'online_drive', + provider_name: 'google_drive', + datasource_name: 'google-drive', + datasource_label: 'Google Drive', + selected: false, + }, + }) return options }, [datasourceNodes]) - return { datasources, options } + return options } export const useLocalFile = () => { diff --git a/web/app/components/rag-pipeline/components/panel/test-run/index.tsx b/web/app/components/rag-pipeline/components/panel/test-run/index.tsx index cc04b40559..e4976293ca 100644 --- a/web/app/components/rag-pipeline/components/panel/test-run/index.tsx +++ b/web/app/components/rag-pipeline/components/panel/test-run/index.tsx @@ -16,6 +16,7 @@ import { TransferMethod } from '@/types/app' import CloseButton from './close-button' import Header from './header' import FooterTips from './footer-tips' +import OnlineDrive from './data-source/online-drive' const TestRunPanel = () => { const setShowDebugAndPreviewPanel = useWorkflowStoreWithSelector(state => state.setShowDebugAndPreviewPanel) @@ -51,17 +52,18 @@ const TestRunPanel = () => { const isVectorSpaceFull = plan.usage.vectorSpace >= plan.total.vectorSpace const isShowVectorSpaceFull = allFileLoaded && isVectorSpaceFull && enableBilling + const datasourceType = datasource?.nodeData.provider_type const nextBtnDisabled = useMemo(() => { if (!datasource) return true - if (datasource.type === DatasourceType.localFile) + if (datasourceType === DatasourceType.localFile) return isShowVectorSpaceFull || !fileList.length || fileList.some(file => !file.file.id) - if (datasource.type === DatasourceType.onlineDocument) + if (datasourceType === DatasourceType.onlineDocument) return isShowVectorSpaceFull || !onlineDocuments.length - if (datasource.type === DatasourceType.websiteCrawl) + if (datasourceType === DatasourceType.websiteCrawl) return isShowVectorSpaceFull || !websitePages.length return false - }, [datasource, isShowVectorSpaceFull, fileList, onlineDocuments.length, websitePages.length]) + }, [datasource, datasourceType, isShowVectorSpaceFull, fileList, onlineDocuments.length, websitePages.length]) const handleClose = () => { setShowDebugAndPreviewPanel(false) @@ -71,7 +73,7 @@ const TestRunPanel = () => { if (!datasource) return const datasourceInfoList: Record[] = [] - if (datasource.type === DatasourceType.localFile) { + if (datasourceType === DatasourceType.localFile) { const { id, name, type, size, extension, mime_type } = fileList[0].file const documentInfo = { related_id: id, @@ -85,7 +87,7 @@ const TestRunPanel = () => { } datasourceInfoList.push(documentInfo) } - if (datasource.type === DatasourceType.onlineDocument) { + if (datasourceType === DatasourceType.onlineDocument) { const { workspace_id, ...rest } = onlineDocuments[0] const documentInfo = { workspace_id, @@ -93,15 +95,15 @@ const TestRunPanel = () => { } datasourceInfoList.push(documentInfo) } - if (datasource.type === DatasourceType.websiteCrawl) + if (datasourceType === DatasourceType.websiteCrawl) datasourceInfoList.push(websitePages[0]) handleRun({ inputs: data, start_node_id: datasource.nodeId, - datasource_type: datasource.type, + datasource_type: datasourceType, datasource_info_list: datasourceInfoList, }) - }, [datasource, fileList, handleRun, onlineDocuments, websitePages]) + }, [datasource, datasourceType, fileList, handleRun, onlineDocuments, websitePages]) return (
{ dataSourceNodeId={datasource?.nodeId || ''} onSelect={setDatasource} /> - {datasource?.type === DatasourceType.localFile && ( + {datasourceType === DatasourceType.localFile && ( )} - {datasource?.type === DatasourceType.onlineDocument && ( + {datasourceType === DatasourceType.onlineDocument && ( )} - {datasource?.type === DatasourceType.websiteCrawl && ( + {datasourceType === DatasourceType.websiteCrawl && ( { isInPipeline /> )} + {datasourceType === DatasourceType.onlineDrive && ( + + ) + } {isShowVectorSpaceFull && ( )} @@ -169,7 +169,7 @@ const TestRunPanel = () => { { currentStep === 2 && ( diff --git a/web/app/components/rag-pipeline/components/panel/test-run/types.ts b/web/app/components/rag-pipeline/components/panel/test-run/types.ts index 18b88414d9..ac5ca3ced2 100644 --- a/web/app/components/rag-pipeline/components/panel/test-run/types.ts +++ b/web/app/components/rag-pipeline/components/panel/test-run/types.ts @@ -1,5 +1,4 @@ import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types' -import type { DatasourceType } from '@/models/pipeline' export enum TestRunStep { dataSource = 'dataSource', @@ -14,9 +13,5 @@ export type DataSourceOption = { export type Datasource = { nodeId: string - type: DatasourceType - description: string - docTitle?: string - docLink?: string - fileExtensions?: string[] + nodeData: DataSourceNodeType }