From 8e4165defefb7c0231a6b3dba089c2ffaade96d4 Mon Sep 17 00:00:00 2001 From: zxhlyh Date: Fri, 6 Jun 2025 10:54:44 +0800 Subject: [PATCH] datasource --- .../workflow/hooks/use-checklist.ts | 28 +++++++++++--- .../workflow/nodes/data-source/constants.ts | 1 - .../workflow/nodes/data-source/default.ts | 35 ++++++++++++++++-- .../components/workflow/utils/data-source.ts | 37 +++++++++++++++++++ web/app/components/workflow/utils/index.ts | 1 + 5 files changed, 92 insertions(+), 10 deletions(-) create mode 100644 web/app/components/workflow/utils/data-source.ts diff --git a/web/app/components/workflow/hooks/use-checklist.ts b/web/app/components/workflow/hooks/use-checklist.ts index be5c2ec0db..912fe24531 100644 --- a/web/app/components/workflow/hooks/use-checklist.ts +++ b/web/app/components/workflow/hooks/use-checklist.ts @@ -11,8 +11,12 @@ import type { Node, } from '../types' import { BlockEnum } from '../types' -import { useStore } from '../store' import { + useStore, + useWorkflowStore, +} from '../store' +import { + getDataSourceCheckParams, getToolCheckParams, getValidTreeNodes, } from '../utils' @@ -25,6 +29,7 @@ import { useWorkflow, } from '../hooks' import type { ToolNodeType } from '../nodes/tool/types' +import type { DataSourceNodeType } from '../nodes/data-source/types' import { useNodesMetaData } from './use-nodes-meta-data' import { useToastContext } from '@/app/components/base/toast' import { useGetLanguage } from '@/context/i18n' @@ -42,6 +47,7 @@ export const useChecklist = (nodes: Node[], edges: Edge[]) => { const buildInTools = useStore(s => s.buildInTools) const customTools = useStore(s => s.customTools) const workflowTools = useStore(s => s.workflowTools) + const dataSourceList = useStore(s => s.dataSourceList) const { data: strategyProviders } = useStrategyProviders() const datasetsDetail = useDatasetsDetailStore(s => s.datasetsDetail) const { getStartNodes } = useWorkflow() @@ -82,6 +88,9 @@ export const useChecklist = (nodes: Node[], edges: Edge[]) => { if (node.data.type === BlockEnum.Tool) moreDataForCheckValid = getToolCheckParams(node.data as ToolNodeType, buildInTools, customTools, workflowTools, language) + if (node.data.type === BlockEnum.DataSource) + moreDataForCheckValid = getDataSourceCheckParams(node.data as DataSourceNodeType, dataSourceList || [], language) + const toolIcon = getToolIcon(node.data) if (node.data.type === BlockEnum.Agent) { const data = node.data as AgentNodeType @@ -126,7 +135,7 @@ export const useChecklist = (nodes: Node[], edges: Edge[]) => { }) return list - }, [nodes, edges, buildInTools, customTools, workflowTools, language, nodesExtraData, t, strategyProviders, getCheckData, getStartNodes, getToolIcon]) + }, [nodes, edges, buildInTools, customTools, workflowTools, language, nodesExtraData, t, strategyProviders, getCheckData, getStartNodes, getToolIcon, dataSourceList]) return needWarningNodes } @@ -134,9 +143,6 @@ export const useChecklist = (nodes: Node[], edges: Edge[]) => { export const useChecklistBeforePublish = () => { const { t } = useTranslation() const language = useGetLanguage() - const buildInTools = useStore(s => s.buildInTools) - const customTools = useStore(s => s.customTools) - const workflowTools = useStore(s => s.workflowTools) const { notify } = useToastContext() const store = useStoreApi() const { nodesMap: nodesExtraData } = useNodesMetaData() @@ -144,6 +150,7 @@ export const useChecklistBeforePublish = () => { const updateDatasetsDetail = useDatasetsDetailStore(s => s.updateDatasetsDetail) const updateTime = useRef(0) const { getStartNodes } = useWorkflow() + const workflowStore = useWorkflowStore() const getCheckData = useCallback((data: CommonNodeType<{}>, datasets: DataSet[]) => { let checkData = data @@ -171,6 +178,12 @@ export const useChecklistBeforePublish = () => { getNodes, edges, } = store.getState() + const { + dataSourceList, + buildInTools, + customTools, + workflowTools, + } = workflowStore.getState() const nodes = getNodes() const filteredNodes = nodes.filter(node => node.type === CUSTOM_NODE) const startNodes = getStartNodes(filteredNodes) @@ -213,6 +226,9 @@ export const useChecklistBeforePublish = () => { if (node.data.type === BlockEnum.Tool) moreDataForCheckValid = getToolCheckParams(node.data as ToolNodeType, buildInTools, customTools, workflowTools, language) + if (node.data.type === BlockEnum.DataSource) + moreDataForCheckValid = getDataSourceCheckParams(node.data as DataSourceNodeType, dataSourceList || [], language) + if (node.data.type === BlockEnum.Agent) { const data = node.data as AgentNodeType const isReadyForCheckValid = !!strategyProviders @@ -251,7 +267,7 @@ export const useChecklistBeforePublish = () => { } return true - }, [store, notify, t, buildInTools, customTools, workflowTools, language, nodesExtraData, strategyProviders, updateDatasetsDetail, getCheckData, getStartNodes]) + }, [store, notify, t, language, nodesExtraData, strategyProviders, updateDatasetsDetail, getCheckData, getStartNodes, workflowStore]) return { handleCheckBeforePublish, diff --git a/web/app/components/workflow/nodes/data-source/constants.ts b/web/app/components/workflow/nodes/data-source/constants.ts index 5b6de604f1..c80b7d4f62 100644 --- a/web/app/components/workflow/nodes/data-source/constants.ts +++ b/web/app/components/workflow/nodes/data-source/constants.ts @@ -20,7 +20,6 @@ export const DEFAULT_FILE_EXTENSIONS_IN_LOCAL_FILE_DATA_SOURCE = [ 'epub', 'ppt', 'md', - 'html', ] export const COMMON_OUTPUT = [ diff --git a/web/app/components/workflow/nodes/data-source/default.ts b/web/app/components/workflow/nodes/data-source/default.ts index 5f46bed514..bb5c9375b1 100644 --- a/web/app/components/workflow/nodes/data-source/default.ts +++ b/web/app/components/workflow/nodes/data-source/default.ts @@ -8,6 +8,9 @@ import { FILE_OUTPUT, WEBSITE_OUTPUT, } from './constants' +import { VarType as VarKindType } from '@/app/components/workflow/nodes/tool/types' + +const i18nPrefix = 'workflow.errorMsg' const metaData = genNodeMetaData({ sort: -1, @@ -19,10 +22,36 @@ const nodeDefault: NodeDefault = { datasource_parameters: {}, datasource_configurations: {}, }, - checkValid() { + checkValid(payload, t, moreDataForCheckValid) { + const { dataSourceInputsSchema, notAuthed } = moreDataForCheckValid + let errorMessage = '' + if (notAuthed) + errorMessage = t(`${i18nPrefix}.authRequired`) + + if (!errorMessage) { + dataSourceInputsSchema.filter((field: any) => { + return field.required + }).forEach((field: any) => { + const targetVar = payload.datasource_parameters[field.variable] + if (!targetVar) { + errorMessage = t(`${i18nPrefix}.fieldRequired`, { field: field.label }) + return + } + const { type: variable_type, value } = targetVar + if (variable_type === VarKindType.variable) { + if (!errorMessage && (!value || value.length === 0)) + errorMessage = t(`${i18nPrefix}.fieldRequired`, { field: field.label }) + } + else { + if (!errorMessage && (value === undefined || value === null || value === '')) + errorMessage = t(`${i18nPrefix}.fieldRequired`, { field: field.label }) + } + }) + } + return { - isValid: true, - errorMessage: '', + isValid: !errorMessage, + errorMessage, } }, getOutputVars(payload, ragVars = []) { diff --git a/web/app/components/workflow/utils/data-source.ts b/web/app/components/workflow/utils/data-source.ts new file mode 100644 index 0000000000..5b2db5437d --- /dev/null +++ b/web/app/components/workflow/utils/data-source.ts @@ -0,0 +1,37 @@ +import type { + InputVar, + ToolWithProvider, +} from '../types' +import type { DataSourceNodeType } from '../nodes/data-source/types' +import { CollectionType } from '@/app/components/tools/types' +import { toolParametersToFormSchemas } from '@/app/components/tools/utils/to-form-schema' + +export const getDataSourceCheckParams = ( + toolData: DataSourceNodeType, + dataSourceList: ToolWithProvider[], + language: string, +) => { + const { plugin_id, provider_type, datasource_name } = toolData + const isBuiltIn = provider_type === CollectionType.builtIn + const currentDataSource = dataSourceList.find(item => item.plugin_id === plugin_id) + const currentDataSourceItem = currentDataSource?.tools.find(tool => tool.name === datasource_name) + const formSchemas = currentDataSourceItem ? toolParametersToFormSchemas(currentDataSourceItem.parameters) : [] + + return { + dataSourceInputsSchema: (() => { + const formInputs: InputVar[] = [] + formSchemas.forEach((item: any) => { + formInputs.push({ + label: item.label[language] || item.label.en_US, + variable: item.variable, + type: item.type, + required: item.required, + hide: item.hide, + }) + }) + return formInputs + })(), + notAuthed: isBuiltIn && !!currentDataSource?.allow_delete && !currentDataSource?.is_authorized, + language, + } +} diff --git a/web/app/components/workflow/utils/index.ts b/web/app/components/workflow/utils/index.ts index 62f609e4b5..e9ae2d1ef0 100644 --- a/web/app/components/workflow/utils/index.ts +++ b/web/app/components/workflow/utils/index.ts @@ -7,3 +7,4 @@ export * from './tool' export * from './workflow' export * from './variable' export * from './gen-node-meta-data' +export * from './data-source'