Merge branch 'feat/rag-pipeline' of https://github.com/langgenius/dify into feat/rag-pipeline

web/app/components/workflow/block-selector/data-sources.tsx

@@ -7,6 +7,7 @@ import type {
   OnSelectBlock,
   ToolWithProvider,
 } from '../types'
+import type { ToolDefaultValue } from './types'
 import Tools from './tools'
 import { ViewType } from './view-type-select'
 import cn from '@/utils/classnames'
@@ -30,8 +31,14 @@ const DataSources = ({
   const pluginRef = useRef<ListRef>(null)
   const wrapElemRef = useRef<HTMLDivElement>(null)
   const formatedDataSources = dataSources.map(item => ({ ...item, tools: item.datasources || [] }))
-  const handleSelect = useCallback<OnSelectBlock>((_, toolDefaultValue) => {
-    onSelect(BlockEnum.DataSource, toolDefaultValue)
+  const handleSelect = useCallback((_: any, toolDefaultValue: ToolDefaultValue) => {
+    onSelect(BlockEnum.DataSource, toolDefaultValue && {
+      provider_id: toolDefaultValue?.provider_id,
+      provider_type: toolDefaultValue?.provider_type,
+      provider_name: toolDefaultValue?.provider_name,
+      datasource_name: toolDefaultValue?.tool_name,
+      datasource_label: toolDefaultValue?.tool_label,
+    })
   }, [onSelect])
 
   return (
@@ -45,7 +52,7 @@ const DataSources = ({
           className={toolContentClassName}
           showWorkflowEmpty={false}
           tools={formatedDataSources}
-          onSelect={handleSelect}
+          onSelect={handleSelect as OnSelectBlock}
           viewType={ViewType.flat}
           hasSearchText={!!searchText}
         />

web/app/components/workflow/block-selector/tabs.tsx

@@ -4,9 +4,9 @@ import { useAllBuiltInTools, useAllCustomTools, useAllWorkflowTools } from '@/se
 import type {
   BlockEnum,
   NodeDefault,
+  OnSelectBlock,
   ToolWithProvider,
 } from '../types'
-import type { ToolDefaultValue } from './types'
 import { TabsEnum } from './types'
 import Blocks from './blocks'
 import AllTools from './all-tools'
@@ -16,7 +16,7 @@ export type TabsProps = {
   activeTab: TabsEnum
   searchText: string
   tags: string[]
-  onSelect: (type: BlockEnum, tool?: ToolDefaultValue) => void
+  onSelect: OnSelectBlock
   availableBlocksTypes?: BlockEnum[]
   blocks: NodeDefault[]
   dataSources?: ToolWithProvider[]

web/app/components/workflow/block-selector/types.ts

@@ -33,6 +33,14 @@ export type ToolDefaultValue = {
   output_schema: Record<string, any>
 }
 
+export type DataSourceDefaultValue = {
+  provider_id: string
+  provider_type: string
+  provider_name: string
+  datasource_name: string
+  datasource_label: string
+}
+
 export type ToolValue = {
   provider_name: string
   tool_name: string

web/app/components/workflow/nodes/_base/components/input-field/add.tsx

@@ -0,0 +1,12 @@
+import { RiAddLine } from '@remixicon/react'
+import ActionButton from '@/app/components/base/action-button'
+
+const Add = () => {
+  return (
+    <ActionButton>
+      <RiAddLine className='h-4 w-4' />
+    </ActionButton>
+  )
+}
+
+export default Add

web/app/components/workflow/nodes/_base/components/input-field/index.tsx

@@ -0,0 +1,24 @@
+import { BoxGroupField } from '@/app/components/workflow/nodes/_base/components/layout'
+import Add from './add'
+
+const InputField = () => {
+  return (
+    <BoxGroupField
+      fieldProps={{
+        supportCollapse: true,
+        fieldTitleProps: {
+          title: 'input field',
+          operation: <Add />,
+        },
+      }}
+      boxGroupProps={{
+        boxProps: {
+          withBorderBottom: true,
+        },
+      }}
+    >
+      input field
+    </BoxGroupField>
+  )
+}
+export default InputField

web/app/components/workflow/nodes/_base/components/layout/box-group-field.tsx

@@ -0,0 +1,29 @@
+import type { ReactNode } from 'react'
+import { memo } from 'react'
+import type {
+  BoxGroupProps,
+  FieldProps,
+} from '.'
+import {
+  BoxGroup,
+  Field,
+} from '.'
+
+type BoxGroupFieldProps = {
+  children?: ReactNode
+  boxGroupProps?: Omit<BoxGroupProps, 'children'>
+  fieldProps?: Omit<FieldProps, 'children'>
+}
+export const BoxGroupField = memo(({
+  children,
+  fieldProps,
+  boxGroupProps,
+}: BoxGroupFieldProps) => {
+  return (
+    <BoxGroup {...boxGroupProps}>
+      <Field {...fieldProps}>
+        {children}
+      </Field>
+    </BoxGroup>
+  )
+})

web/app/components/workflow/nodes/_base/components/layout/box-group.tsx

@@ -9,16 +9,16 @@ import type {
   GroupProps,
 } from '.'
 
-type GroupWithBoxProps = {
+export type BoxGroupProps = {
   children?: ReactNode
   boxProps?: Omit<BoxProps, 'children'>
   groupProps?: Omit<GroupProps, 'children'>
 }
-export const GroupWithBox = memo(({
+export const BoxGroup = memo(({
   children,
   boxProps,
   groupProps,
-}: GroupWithBoxProps) => {
+}: BoxGroupProps) => {
   return (
     <Box {...boxProps}>
       <Group {...groupProps}>

web/app/components/workflow/nodes/_base/components/layout/field-title.tsx

@@ -1,25 +1,58 @@
 import type { ReactNode } from 'react'
-import { memo } from 'react'
+import {
+  memo,
+  useState,
+} from 'react'
+import { ArrowDownRoundFill } from '@/app/components/base/icons/src/vender/solid/general'
 import Tooltip from '@/app/components/base/tooltip'
 import cn from '@/utils/classnames'
 
 export type FieldTitleProps = {
-  title: string
+  title?: string
   operation?: ReactNode
   subTitle?: string | ReactNode
   tooltip?: string
+  showArrow?: boolean
+  disabled?: boolean
+  collapsed?: boolean
+  onCollapse?: (collapsed: boolean) => void
 }
 export const FieldTitle = memo(({
   title,
   operation,
   subTitle,
   tooltip,
+  showArrow,
+  disabled,
+  collapsed,
+  onCollapse,
 }: FieldTitleProps) => {
+  const [collapsedLocal, setCollapsedLocal] = useState(true)
+  const collapsedMerged = collapsed !== undefined ? collapsed : collapsedLocal
+
   return (
     <div className={cn('mb-0.5', !!subTitle && 'mb-1')}>
-      <div className='flex items-center justify-between py-1'>
+      <div
+        className='group/collapse flex items-center justify-between py-1'
+        onClick={() => {
+          if (!disabled) {
+            setCollapsedLocal(!collapsedMerged)
+            onCollapse?.(!collapsedMerged)
+          }
+        }}
+      >
         <div className='system-sm-semibold-uppercase flex items-center text-text-secondary'>
           {title}
+          {
+            showArrow && (
+              <ArrowDownRoundFill
+                className={cn(
+                  'h-4 w-4 cursor-pointer text-text-quaternary group-hover/collapse:text-text-secondary',
+                  collapsedMerged && 'rotate-[270deg]',
+                )}
+              />
+            )
+          }
           {
             tooltip && (
               <Tooltip

web/app/components/workflow/nodes/_base/components/layout/field.tsx

@@ -1,20 +1,36 @@
 import type { ReactNode } from 'react'
-import { memo } from 'react'
+import {
+  memo,
+  useState,
+} from 'react'
 import type { FieldTitleProps } from '.'
 import { FieldTitle } from '.'
 
 export type FieldProps = {
-  fieldTitleProps: FieldTitleProps
-  children: ReactNode
+  fieldTitleProps?: FieldTitleProps
+  children?: ReactNode
+  disabled?: boolean
+  supportCollapse?: boolean
 }
 export const Field = memo(({
   fieldTitleProps,
   children,
+  supportCollapse,
+  disabled,
 }: FieldProps) => {
+  const [collapsed, setCollapsed] = useState(false)
+
   return (
     <div>
-      <FieldTitle {...fieldTitleProps} />
-      {children}
+      <FieldTitle
+        {...fieldTitleProps}
+        collapsed={collapsed}
+        onCollapse={setCollapsed}
+        showArrow={supportCollapse}
+        disabled={disabled}
+      />
+      {supportCollapse && !collapsed && children}
+      {!supportCollapse && children}
     </div>
   )
 })

web/app/components/workflow/nodes/_base/components/layout/group-field.tsx

@@ -0,0 +1,29 @@
+import type { ReactNode } from 'react'
+import { memo } from 'react'
+import type {
+  FieldProps,
+  GroupProps,
+} from '.'
+import {
+  Field,
+  Group,
+} from '.'
+
+type GroupFieldProps = {
+  children?: ReactNode
+  groupProps?: Omit<GroupProps, 'children'>
+  fieldProps?: Omit<FieldProps, 'children'>
+}
+export const GroupField = memo(({
+  children,
+  fieldProps,
+  groupProps,
+}: GroupFieldProps) => {
+  return (
+    <Group {...groupProps}>
+      <Field {...fieldProps}>
+        {children}
+      </Field>
+    </Group>
+  )
+})

web/app/components/workflow/nodes/_base/components/layout/index.tsx

@@ -1,5 +1,7 @@
 export * from './box'
 export * from './group'
-export * from './group-with-box'
+export * from './box-group'
 export * from './field-title'
 export * from './field'
+export * from './group-field'
+export * from './box-group-field'

web/app/components/workflow/nodes/data-source/panel.tsx

@@ -1,128 +1,101 @@
 import type { FC } from 'react'
-import {
-  useMemo,
-} from 'react'
 import { useTranslation } from 'react-i18next'
 import { memo } from 'react'
 import type { DataSourceNodeType } from './types'
 import { CollectionType } from '@/app/components/tools/types'
 import type { NodePanelProps } from '@/app/components/workflow/types'
-import {
-  Field,
-  GroupWithBox,
-} from '@/app/components/workflow/nodes/_base/components/layout'
+import { BoxGroupField } from '@/app/components/workflow/nodes/_base/components/layout'
 import OutputVars, { VarItem } from '@/app/components/workflow/nodes/_base/components/output-vars'
-import StructureOutputItem from '@/app/components/workflow/nodes/_base/components/variable/object-child-tree-panel/show'
 import TagInput from '@/app/components/base/tag-input'
-import { Type } from '../llm/types'
 import { useConfig } from './hooks/use-config'
 
 const Panel: FC<NodePanelProps<DataSourceNodeType>> = ({ id, data }) => {
   const { t } = useTranslation()
   const {
-    output_schema = {},
     provider_id,
     provider_type,
     fileExtensions = [],
   } = data
   const { handleFileExtensionsChange } = useConfig(id)
-  const outputSchema = useMemo(() => {
-      const res: any[] = []
-      if (!output_schema)
-        return []
-      Object.keys(output_schema.properties).forEach((outputKey) => {
-        const output = output_schema.properties[outputKey]
-        const type = output.type
-        if (type === 'object') {
-          res.push({
-            name: outputKey,
-            value: output,
-          })
-        }
-        else {
-          res.push({
-            name: outputKey,
-            type: output.type === 'array'
-              ? `Array[${output.items?.type.slice(0, 1).toLocaleUpperCase()}${output.items?.type.slice(1)}]`
-              : `${output.type.slice(0, 1).toLocaleUpperCase()}${output.type.slice(1)}`,
-            description: output.description,
-          })
-        }
-      })
-      return res
-    }, [output_schema])
-  const hasObjectOutput = useMemo(() => {
-    if (!output_schema)
-      return false
-    const properties = output_schema.properties
-    return Object.keys(properties).some(key => properties[key].type === 'object')
-  }, [output_schema])
+  const isLocalFile = provider_id === 'langgenius/file/file' && provider_type === CollectionType.datasource
 
   return (
     <div >
       {
-        provider_id === 'langgenius/file/file' && provider_type === CollectionType.datasource && (
-          <GroupWithBox boxProps={{ withBorderBottom: true }}>
-            <Field
-              fieldTitleProps={{
+        isLocalFile && (
+          <BoxGroupField
+            boxGroupProps={{
+              boxProps: { withBorderBottom: true },
+            }}
+            fieldProps={{
+              fieldTitleProps: {
                 title: t('workflow.nodes.dataSource.supportedFileFormats'),
-              }}
-            >
-              <div className='rounded-lg bg-components-input-bg-normal p-1 pt-0'>
-                <TagInput
-                  items={fileExtensions}
-                  onChange={handleFileExtensionsChange}
-                  placeholder={t('workflow.nodes.dataSource.supportedFileFormatsPlaceholder')}
-                  inputClassName='bg-transparent'
-                />
-              </div>
-            </Field>
-          </GroupWithBox>
+              },
+            }}
+          >
+            <div className='rounded-lg bg-components-input-bg-normal p-1 pt-0'>
+              <TagInput
+                items={fileExtensions}
+                onChange={handleFileExtensionsChange}
+                placeholder={t('workflow.nodes.dataSource.supportedFileFormatsPlaceholder')}
+                inputClassName='bg-transparent'
+              />
+            </div>
+          </BoxGroupField>
         )
       }
       <OutputVars>
         <VarItem
-          name='text'
+          name='datasource_type'
           type='string'
-          description={t('workflow.nodes.tool.outputVars.text')}
-          isIndent={hasObjectOutput}
+          description={'local_file, online_document, website_crawl'}
         />
-        <VarItem
-          name='files'
-          type='array[file]'
-          description={t('workflow.nodes.tool.outputVars.files.title')}
-          isIndent={hasObjectOutput}
-        />
-        <VarItem
-          name='json'
-          type='array[object]'
-          description={t('workflow.nodes.tool.outputVars.json')}
-          isIndent={hasObjectOutput}
-        />
-        {outputSchema.map((outputItem: any) => (
-          <div key={outputItem.name}>
-            {outputItem.value?.type === 'object' ? (
-              <StructureOutputItem
-                rootClassName='code-sm-semibold text-text-secondary'
-                payload={{
-                  schema: {
-                    type: Type.object,
-                    properties: {
-                      [outputItem.name]: outputItem.value,
-                    },
-                    additionalProperties: false,
-                  },
-                }} />
-            ) : (
-              <VarItem
-                name={outputItem.name}
-                type={outputItem.type.toLocaleLowerCase()}
-                description={outputItem.description}
-                isIndent={hasObjectOutput}
-              />
-            )}
-          </div>
-        ))}
+        {
+          isLocalFile && (
+            <VarItem
+              name='file'
+              type='Object'
+              description={'file'}
+              subItems={[
+                {
+                  name: 'type',
+                  type: 'string',
+                  description: '',
+                },
+                {
+                  name: 'upload_file_id',
+                  type: 'string',
+                  description: '',
+                },
+                {
+                  name: 'name',
+                  type: 'string',
+                  description: '',
+                },
+                {
+                  name: 'size',
+                  type: 'number',
+                  description: '',
+                },
+                {
+                  name: 'extension',
+                  type: 'string',
+                  description: '',
+                },
+                {
+                  name: 'mime_type',
+                  type: 'string',
+                  description: '',
+                },
+                {
+                  name: 'upload_file_url',
+                  type: 'string',
+                  description: '',
+                },
+              ]}
+            />
+          )
+        }
       </OutputVars>
     </div>
   )

web/app/components/workflow/nodes/data-source/types.ts

@@ -1,11 +1,26 @@
-import type { CommonNodeType } from '@/app/components/workflow/types'
+import type { CommonNodeType, ValueSelector } from '@/app/components/workflow/types'
 import type { RAGPipelineVariables } from '@/models/pipeline'
 import type { CollectionType } from '@/app/components/tools/types'
 
+export enum VarType {
+  variable = 'variable',
+  constant = 'constant',
+  mixed = 'mixed',
+}
+
+export type ToolVarInputs = Record<string, {
+  type: VarType
+  value?: string | ValueSelector | any
+}>
+
 export type DataSourceNodeType = CommonNodeType & {
   variables: RAGPipelineVariables
-  output_schema: Record<string, any>
+  fileExtensions?: string[]
   provider_id: string
   provider_type: CollectionType
-  fileExtensions?: string[]
+  provider_name: string
+  datasource_name: string
+  datasource_label: string
+  datasource_parameters: ToolVarInputs
+  datasource_configurations: Record<string, any>
 }

web/app/components/workflow/nodes/knowledge-base/components/input-variable.tsx

@@ -1,37 +0,0 @@
-import { memo } from 'react'
-import { useTranslation } from 'react-i18next'
-import VarReferencePicker from '@/app/components/workflow/nodes/_base/components/variable/var-reference-picker'
-import { Field } from '@/app/components/workflow/nodes/_base/components/layout'
-import type { ValueSelector } from '@/app/components/workflow/types'
-
-type InputVariableProps = {
-  nodeId: string
-  inputVariable?: string[]
-  onInputVariableChange: (inputVariable: string | ValueSelector) => void
-  readonly?: boolean
-}
-const InputVariable = ({
-  nodeId,
-  inputVariable = [],
-  onInputVariableChange,
-  readonly = false,
-}: InputVariableProps) => {
-  const { t } = useTranslation()
-
-  return (
-    <Field
-      fieldTitleProps={{
-        title: t('workflow.nodes.common.inputVars'),
-      }}
-    >
-      <VarReferencePicker
-        nodeId={nodeId}
-        isShowNodeName
-        value={inputVariable}
-        onChange={onInputVariableChange}
-        readonly={readonly}
-      />
-    </Field>
-  )
-}
-export default memo(InputVariable)

web/app/components/workflow/nodes/knowledge-base/panel.tsx

@@ -2,11 +2,11 @@ import type { FC } from 'react'
 import {
   memo,
 } from 'react'
+import { useTranslation } from 'react-i18next'
 import type { KnowledgeBaseNodeType } from './types'
 import {
   IndexMethodEnum,
 } from './types'
-import InputVariable from './components/input-variable'
 import ChunkStructure from './components/chunk-structure'
 import IndexMethod from './components/index-method'
 import RetrievalSetting from './components/retrieval-setting'
@@ -14,16 +14,19 @@ import EmbeddingModel from './components/embedding-model'
 import { useConfig } from './hooks/use-config'
 import type { NodePanelProps } from '@/app/components/workflow/types'
 import {
+  BoxGroup,
+  BoxGroupField,
   Group,
-  GroupWithBox,
 } from '@/app/components/workflow/nodes/_base/components/layout'
 import Split from '../_base/components/split'
 import { useNodesReadOnly } from '@/app/components/workflow/hooks'
+import VarReferencePicker from '@/app/components/workflow/nodes/_base/components/variable/var-reference-picker'
 
 const Panel: FC<NodePanelProps<KnowledgeBaseNodeType>> = ({
   id,
   data,
 }) => {
+  const { t } = useTranslation()
   const { nodesReadOnly } = useNodesReadOnly()
   const {
     handleChunkStructureChange,
@@ -42,14 +45,24 @@ const Panel: FC<NodePanelProps<KnowledgeBaseNodeType>> = ({
 
   return (
     <div>
-      <GroupWithBox boxProps={{ withBorderBottom: true }}>
-        <InputVariable
+      <BoxGroupField
+        boxGroupProps={{
+          boxProps: { withBorderBottom: true },
+        }}
+        fieldProps={{
+          fieldTitleProps: {
+            title: t('workflow.nodes.common.inputVars'),
+          },
+        }}
+      >
+        <VarReferencePicker
           nodeId={id}
-          inputVariable={data.index_chunk_variable_selector}
-          onInputVariableChange={handleInputVariableChange}
+          isShowNodeName
+          value={data.index_chunk_variable_selector}
+          onChange={handleInputVariableChange}
           readonly={nodesReadOnly}
         />
-      </GroupWithBox>
+      </BoxGroupField>
       <Group
         className='py-3'
         withBorderBottom
@@ -60,7 +73,7 @@ const Panel: FC<NodePanelProps<KnowledgeBaseNodeType>> = ({
           readonly={nodesReadOnly}
         />
       </Group>
-      <GroupWithBox>
+      <BoxGroup>
         <div className='space-y-3'>
           <IndexMethod
             chunkStructure={data.chunk_structure}
@@ -102,7 +115,7 @@ const Panel: FC<NodePanelProps<KnowledgeBaseNodeType>> = ({
             readonly={nodesReadOnly}
           />
         </div>
-      </GroupWithBox>
+      </BoxGroup>
     </div>
   )
 }

web/app/components/workflow/types.ts

@@ -5,7 +5,10 @@ import type {
   XYPosition,
 } from 'reactflow'
 import type { Resolution, TransferMethod } from '@/types/app'
-import type { ToolDefaultValue } from '@/app/components/workflow/block-selector/types'
+import type {
+  DataSourceDefaultValue,
+  ToolDefaultValue,
+} from '@/app/components/workflow/block-selector/types'
 import type { VarType as VarKindType } from '@/app/components/workflow/nodes/tool/types'
 import type { FileResponse, NodeTracing } from '@/types/workflow'
 import type { Collection, Tool } from '@/app/components/tools/types'
@@ -96,6 +99,7 @@ export type CommonNodeType<T = {}> = {
   retry_config?: WorkflowRetryConfig
   default_value?: DefaultValueForm[]
 } & T & Partial<Pick<ToolDefaultValue, 'provider_id' | 'provider_type' | 'provider_name' | 'tool_name'>>
+& Partial<Pick<DataSourceDefaultValue, 'provider_id' | 'provider_type' | 'provider_name' | 'datasource_name'>>
 
 export type CommonEdgeType = {
   _hovering?: boolean
@@ -311,7 +315,7 @@ export type NodeDefault<T = {}> = {
   checkValid: (payload: T, t: any, moreDataForCheckValid?: any) => { isValid: boolean; errorMessage?: string }
 }
 
-export type OnSelectBlock = (type: BlockEnum, toolDefaultValue?: ToolDefaultValue) => void
+export type OnSelectBlock = (type: BlockEnum, toolDefaultValue?: ToolDefaultValue | DataSourceDefaultValue) => void
 
 export enum WorkflowRunningStatus {
   Waiting = 'waiting',
@@ -341,7 +345,7 @@ export type OnNodeAdd = (
     nodeType: BlockEnum
     sourceHandle?: string
     targetHandle?: string
-    toolDefaultValue?: ToolDefaultValue
+    toolDefaultValue?: ToolDefaultValue | DataSourceDefaultValue
   },
   oldNodesPayload: {
     prevNodeId?: string