datasource

web/app/components/workflow/hooks/use-checklist.ts

@@ -11,8 +11,12 @@ import type {
   Node,
 } from '../types'
 import { BlockEnum } from '../types'
-import { useStore } from '../store'
 import {
+  useStore,
+  useWorkflowStore,
+} from '../store'
+import {
+  getDataSourceCheckParams,
   getToolCheckParams,
   getValidTreeNodes,
 } from '../utils'
@@ -25,6 +29,7 @@ import {
   useWorkflow,
 } from '../hooks'
 import type { ToolNodeType } from '../nodes/tool/types'
+import type { DataSourceNodeType } from '../nodes/data-source/types'
 import { useNodesMetaData } from './use-nodes-meta-data'
 import { useToastContext } from '@/app/components/base/toast'
 import { useGetLanguage } from '@/context/i18n'
@@ -42,6 +47,7 @@ export const useChecklist = (nodes: Node[], edges: Edge[]) => {
   const buildInTools = useStore(s => s.buildInTools)
   const customTools = useStore(s => s.customTools)
   const workflowTools = useStore(s => s.workflowTools)
+  const dataSourceList = useStore(s => s.dataSourceList)
   const { data: strategyProviders } = useStrategyProviders()
   const datasetsDetail = useDatasetsDetailStore(s => s.datasetsDetail)
   const { getStartNodes } = useWorkflow()
@@ -82,6 +88,9 @@ export const useChecklist = (nodes: Node[], edges: Edge[]) => {
       if (node.data.type === BlockEnum.Tool)
         moreDataForCheckValid = getToolCheckParams(node.data as ToolNodeType, buildInTools, customTools, workflowTools, language)
 
+      if (node.data.type === BlockEnum.DataSource)
+        moreDataForCheckValid = getDataSourceCheckParams(node.data as DataSourceNodeType, dataSourceList || [], language)
+
       const toolIcon = getToolIcon(node.data)
       if (node.data.type === BlockEnum.Agent) {
         const data = node.data as AgentNodeType
@@ -126,7 +135,7 @@ export const useChecklist = (nodes: Node[], edges: Edge[]) => {
     })
 
     return list
-  }, [nodes, edges, buildInTools, customTools, workflowTools, language, nodesExtraData, t, strategyProviders, getCheckData, getStartNodes, getToolIcon])
+  }, [nodes, edges, buildInTools, customTools, workflowTools, language, nodesExtraData, t, strategyProviders, getCheckData, getStartNodes, getToolIcon, dataSourceList])
 
   return needWarningNodes
 }
@@ -134,9 +143,6 @@ export const useChecklist = (nodes: Node[], edges: Edge[]) => {
 export const useChecklistBeforePublish = () => {
   const { t } = useTranslation()
   const language = useGetLanguage()
-  const buildInTools = useStore(s => s.buildInTools)
-  const customTools = useStore(s => s.customTools)
-  const workflowTools = useStore(s => s.workflowTools)
   const { notify } = useToastContext()
   const store = useStoreApi()
   const { nodesMap: nodesExtraData } = useNodesMetaData()
@@ -144,6 +150,7 @@ export const useChecklistBeforePublish = () => {
   const updateDatasetsDetail = useDatasetsDetailStore(s => s.updateDatasetsDetail)
   const updateTime = useRef(0)
   const { getStartNodes } = useWorkflow()
+  const workflowStore = useWorkflowStore()
 
   const getCheckData = useCallback((data: CommonNodeType<{}>, datasets: DataSet[]) => {
     let checkData = data
@@ -171,6 +178,12 @@ export const useChecklistBeforePublish = () => {
       getNodes,
       edges,
     } = store.getState()
+    const {
+      dataSourceList,
+      buildInTools,
+      customTools,
+      workflowTools,
+    } = workflowStore.getState()
     const nodes = getNodes()
     const filteredNodes = nodes.filter(node => node.type === CUSTOM_NODE)
     const startNodes = getStartNodes(filteredNodes)
@@ -213,6 +226,9 @@ export const useChecklistBeforePublish = () => {
       if (node.data.type === BlockEnum.Tool)
         moreDataForCheckValid = getToolCheckParams(node.data as ToolNodeType, buildInTools, customTools, workflowTools, language)
 
+      if (node.data.type === BlockEnum.DataSource)
+        moreDataForCheckValid = getDataSourceCheckParams(node.data as DataSourceNodeType, dataSourceList || [], language)
+
       if (node.data.type === BlockEnum.Agent) {
         const data = node.data as AgentNodeType
         const isReadyForCheckValid = !!strategyProviders
@@ -251,7 +267,7 @@ export const useChecklistBeforePublish = () => {
     }
 
     return true
-  }, [store, notify, t, buildInTools, customTools, workflowTools, language, nodesExtraData, strategyProviders, updateDatasetsDetail, getCheckData, getStartNodes])
+  }, [store, notify, t, language, nodesExtraData, strategyProviders, updateDatasetsDetail, getCheckData, getStartNodes, workflowStore])
 
   return {
     handleCheckBeforePublish,

web/app/components/workflow/nodes/data-source/constants.ts

@@ -20,7 +20,6 @@ export const DEFAULT_FILE_EXTENSIONS_IN_LOCAL_FILE_DATA_SOURCE = [
   'epub',
   'ppt',
   'md',
-  'html',
 ]
 
 export const COMMON_OUTPUT = [

web/app/components/workflow/nodes/data-source/default.ts

@@ -8,6 +8,9 @@ import {
   FILE_OUTPUT,
   WEBSITE_OUTPUT,
 } from './constants'
+import { VarType as VarKindType } from '@/app/components/workflow/nodes/tool/types'
+
+const i18nPrefix = 'workflow.errorMsg'
 
 const metaData = genNodeMetaData({
   sort: -1,
@@ -19,10 +22,36 @@ const nodeDefault: NodeDefault<DataSourceNodeType> = {
     datasource_parameters: {},
     datasource_configurations: {},
   },
-  checkValid() {
+  checkValid(payload, t, moreDataForCheckValid) {
+    const { dataSourceInputsSchema, notAuthed } = moreDataForCheckValid
+    let errorMessage = ''
+    if (notAuthed)
+      errorMessage = t(`${i18nPrefix}.authRequired`)
+
+    if (!errorMessage) {
+      dataSourceInputsSchema.filter((field: any) => {
+        return field.required
+      }).forEach((field: any) => {
+        const targetVar = payload.datasource_parameters[field.variable]
+        if (!targetVar) {
+          errorMessage = t(`${i18nPrefix}.fieldRequired`, { field: field.label })
+          return
+        }
+        const { type: variable_type, value } = targetVar
+        if (variable_type === VarKindType.variable) {
+          if (!errorMessage && (!value || value.length === 0))
+            errorMessage = t(`${i18nPrefix}.fieldRequired`, { field: field.label })
+        }
+        else {
+          if (!errorMessage && (value === undefined || value === null || value === ''))
+            errorMessage = t(`${i18nPrefix}.fieldRequired`, { field: field.label })
+        }
+      })
+    }
+
     return {
-      isValid: true,
+      isValid: !errorMessage,
-      errorMessage: '',
+      errorMessage,
     }
   },
   getOutputVars(payload, ragVars = []) {

web/app/components/workflow/utils/data-source.ts

@@ -0,0 +1,37 @@
+import type {
+  InputVar,
+  ToolWithProvider,
+} from '../types'
+import type { DataSourceNodeType } from '../nodes/data-source/types'
+import { CollectionType } from '@/app/components/tools/types'
+import { toolParametersToFormSchemas } from '@/app/components/tools/utils/to-form-schema'
+
+export const getDataSourceCheckParams = (
+  toolData: DataSourceNodeType,
+  dataSourceList: ToolWithProvider[],
+  language: string,
+) => {
+  const { plugin_id, provider_type, datasource_name } = toolData
+  const isBuiltIn = provider_type === CollectionType.builtIn
+  const currentDataSource = dataSourceList.find(item => item.plugin_id === plugin_id)
+  const currentDataSourceItem = currentDataSource?.tools.find(tool => tool.name === datasource_name)
+  const formSchemas = currentDataSourceItem ? toolParametersToFormSchemas(currentDataSourceItem.parameters) : []
+
+  return {
+    dataSourceInputsSchema: (() => {
+      const formInputs: InputVar[] = []
+      formSchemas.forEach((item: any) => {
+        formInputs.push({
+          label: item.label[language] || item.label.en_US,
+          variable: item.variable,
+          type: item.type,
+          required: item.required,
+          hide: item.hide,
+        })
+      })
+      return formInputs
+    })(),
+    notAuthed: isBuiltIn && !!currentDataSource?.allow_delete && !currentDataSource?.is_authorized,
+    language,
+  }
+}

web/app/components/workflow/utils/index.ts

@@ -7,3 +7,4 @@ export * from './tool'
 export * from './workflow'
 export * from './variable'
 export * from './gen-node-meta-data'
+export * from './data-source'