fix: pipeline init

web/app/components/base/form/components/field/mixed-variable-text-input/index.tsx

@@ -0,0 +1,39 @@
+import {
+  memo,
+} from 'react'
+import PromptEditor from '@/app/components/base/prompt-editor'
+import cn from '@/utils/classnames'
+import Placeholder from './placeholder'
+
+type MixedVariableTextInputProps = {
+  editable?: boolean
+  value?: string
+  onChange?: (text: string) => void
+}
+const MixedVariableTextInput = ({
+  editable = true,
+  value = '',
+  onChange,
+}: MixedVariableTextInputProps) => {
+  return (
+    <PromptEditor
+      wrapperClassName={cn(
+        'rounded-lg border border-transparent bg-components-input-bg-normal px-2 py-1',
+        'hover:border-components-input-border-hover hover:bg-components-input-bg-hover',
+        'focus-within:border-components-input-border-active focus-within:bg-components-input-bg-active focus-within:shadow-xs',
+      )}
+      className='caret:text-text-accent'
+      editable={editable}
+      value={value}
+      workflowVariableBlock={{
+        show: true,
+        variables: [],
+        workflowNodesMap: {},
+      }}
+      placeholder={<Placeholder />}
+      onChange={onChange}
+    />
+  )
+}
+
+export default memo(MixedVariableTextInput)

web/app/components/base/form/components/field/mixed-variable-text-input/placeholder.tsx

@@ -0,0 +1,49 @@
+import { useCallback } from 'react'
+import { useLexicalComposerContext } from '@lexical/react/LexicalComposerContext'
+import { FOCUS_COMMAND } from 'lexical'
+import { $insertNodes } from 'lexical'
+import { CustomTextNode } from '@/app/components/base/prompt-editor/plugins/custom-text/node'
+import Badge from '@/app/components/base/badge'
+
+const Placeholder = () => {
+  const [editor] = useLexicalComposerContext()
+
+  const handleInsert = useCallback((text: string) => {
+    editor.update(() => {
+      const textNode = new CustomTextNode(text)
+      $insertNodes([textNode])
+    })
+    editor.dispatchCommand(FOCUS_COMMAND, undefined as any)
+  }, [editor])
+
+  return (
+    <div
+      className='pointer-events-auto flex h-full w-full cursor-text items-center px-2'
+      onClick={(e) => {
+        e.stopPropagation()
+        handleInsert('')
+      }}
+    >
+      <div className='flex grow items-center'>
+        Type or press
+        <div className='system-kbd mx-0.5 flex h-4 w-4 items-center justify-center rounded bg-components-kbd-bg-gray text-text-placeholder'>/</div>
+        <div
+          className='system-sm-regular cursor-pointer text-components-input-text-placeholder underline decoration-dotted decoration-auto underline-offset-auto hover:text-text-tertiary'
+          onClick={((e) => {
+            e.stopPropagation()
+            handleInsert('/')
+          })}
+        >
+          insert variable
+        </div>
+      </div>
+      <Badge
+        className='shrink-0'
+        text='String'
+        uppercase={false}
+      />
+    </div>
+  )
+}
+
+export default Placeholder

web/app/components/base/prompt-editor/index.tsx

@@ -64,8 +64,9 @@ import cn from '@/utils/classnames'
 export type PromptEditorProps = {
   instanceId?: string
   compact?: boolean
+  wrapperClassName?: string
   className?: string
-  placeholder?: string
+  placeholder?: string | JSX.Element
   placeholderClassName?: string
   style?: React.CSSProperties
   value?: string
@@ -85,6 +86,7 @@ export type PromptEditorProps = {
 const PromptEditor: FC<PromptEditorProps> = ({
   instanceId,
   compact,
+  wrapperClassName,
   className,
   placeholder,
   placeholderClassName,
@@ -147,10 +149,25 @@ const PromptEditor: FC<PromptEditorProps> = ({
 
   return (
     <LexicalComposer initialConfig={{ ...initialConfig, editable }}>
-      <div className='relative min-h-5'>
+      <div className={cn('relative', wrapperClassName)}>
         <RichTextPlugin
-          contentEditable={<ContentEditable className={`${className} outline-none ${compact ? 'text-[13px] leading-5' : 'text-sm leading-6'} text-text-secondary`} style={style || {}} />}
-          placeholder={<Placeholder value={placeholder} className={cn('truncate', placeholderClassName)} compact={compact} />}
+          contentEditable={
+            <ContentEditable
+              className={cn(
+                'text-text-secondary outline-none',
+                compact ? 'text-[13px] leading-5' : 'text-sm leading-6',
+                className,
+              )}
+              style={style || {}}
+            />
+          }
+          placeholder={
+            <Placeholder
+              value={placeholder}
+              className={cn('truncate', placeholderClassName)}
+              compact={compact}
+            />
+          }
           ErrorBoundary={LexicalErrorBoundary}
         />
         <ComponentPickerBlock

web/app/components/base/prompt-editor/plugins/custom-text/node.tsx

@@ -16,7 +16,6 @@ export class CustomTextNode extends TextNode {
 
   createDOM(config: EditorConfig) {
     const dom = super.createDOM(config)
-    dom.classList.add('align-middle')
     return dom
   }
 

web/app/components/base/prompt-editor/plugins/placeholder.tsx

@@ -8,16 +8,16 @@ const Placeholder = ({
   className,
 }: {
   compact?: boolean
-  value?: string
+  value?: string | JSX.Element
   className?: string
 }) => {
   const { t } = useTranslation()
 
   return (
     <div className={cn(
-      className,
       'pointer-events-none absolute left-0 top-0 h-full w-full select-none text-sm text-components-input-text-placeholder',
       compact ? 'text-[13px] leading-5' : 'text-sm leading-6',
+      className,
     )}>
       {value || t('common.promptEditor.placeholder')}
     </div>

web/app/components/rag-pipeline/hooks/use-pipeline-config.ts

@@ -0,0 +1,53 @@
+import { useCallback } from 'react'
+import {
+  useStore,
+  useWorkflowStore,
+} from '@/app/components/workflow/store'
+import { useWorkflowConfig } from '@/service/use-workflow'
+import type { ToolWithProvider } from '@/app/components/workflow/types'
+import type { FetchWorkflowDraftResponse } from '@/types/workflow'
+
+export const usePipelineConfig = () => {
+  const pipelineId = useStore(s => s.pipelineId)
+  const workflowStore = useWorkflowStore()
+
+  const handleUpdateWorkflowConfig = useCallback((config: Record<string, any>) => {
+    const { setWorkflowConfig } = workflowStore.getState()
+
+    setWorkflowConfig(config)
+  }, [workflowStore])
+  useWorkflowConfig(
+    pipelineId ? `/rag/pipeline/${pipelineId}/workflows/draft/config` : '',
+    handleUpdateWorkflowConfig,
+  )
+
+  const handleUpdateDataSourceList = useCallback((dataSourceList: ToolWithProvider[]) => {
+    const { setDataSourceList } = workflowStore.getState()
+
+    setDataSourceList!(dataSourceList)
+  }, [workflowStore])
+  useWorkflowConfig<ToolWithProvider[]>(
+    '/rag/pipelines/datasource-plugins',
+    handleUpdateDataSourceList,
+  )
+
+  const handleUpdateNodesDefaultConfigs = useCallback((nodesDefaultConfigs: Record<string, any>) => {
+    const { setNodesDefaultConfigs } = workflowStore.getState()
+
+    setNodesDefaultConfigs!(nodesDefaultConfigs)
+  }, [workflowStore])
+  useWorkflowConfig(
+    pipelineId ? `/rag/pipeline/${pipelineId}/workflows/default-workflow-block-configs` : '',
+    handleUpdateNodesDefaultConfigs,
+  )
+
+  const handleUpdatePublishedAt = useCallback((publishedWorkflow: FetchWorkflowDraftResponse) => {
+    const { setPublishedAt } = workflowStore.getState()
+
+    setPublishedAt(publishedWorkflow?.created_at)
+  }, [workflowStore])
+  useWorkflowConfig(
+    pipelineId ? `/rag/pipeline/${pipelineId}/workflows/publish` : '',
+    handleUpdatePublishedAt,
+  )
+}

web/app/components/rag-pipeline/hooks/use-pipeline-init.ts

@@ -5,18 +5,15 @@ import {
 } from 'react'
 import { useParams } from 'next/navigation'
 import {
-  useStore,
   useWorkflowStore,
 } from '@/app/components/workflow/store'
 import { usePipelineTemplate } from './use-pipeline-template'
 import {
-  fetchNodesDefaultConfigs,
-  fetchPublishedWorkflow,
   fetchWorkflowDraft,
   syncWorkflowDraft,
 } from '@/service/workflow'
 import type { FetchWorkflowDraftResponse } from '@/types/workflow'
-// import { useWorkflowConfig } from '@/service/use-workflow'
+import { usePipelineConfig } from './use-pipeline-config'
 
 export const usePipelineInit = () => {
   const { datasetId } = useParams()
@@ -25,31 +22,33 @@ export const usePipelineInit = () => {
     nodes: nodesTemplate,
     edges: edgesTemplate,
   } = usePipelineTemplate()
-  const setSyncWorkflowDraftHash = useStore(s => s.setSyncWorkflowDraftHash)
   const [data, setData] = useState<FetchWorkflowDraftResponse>()
   const [isLoading, setIsLoading] = useState(true)
+
   useEffect(() => {
-    // workflowStore.setState({ pipelineId: datasetId as string })
+    workflowStore.setState({ pipelineId: datasetId as string })
   }, [datasetId, workflowStore])
 
-  const handleUpdateWorkflowConfig = useCallback((config: Record<string, any>) => {
-    const { setWorkflowConfig } = workflowStore.getState()
-
-    setWorkflowConfig(config)
-  }, [workflowStore])
-  // useWorkflowConfig(`/rag/pipeline/${datasetId}/workflows/draft/config`, handleUpdateWorkflowConfig)
+  usePipelineConfig()
 
   const handleGetInitialWorkflowData = useCallback(async () => {
+    const {
+      setEnvSecrets,
+      setEnvironmentVariables,
+      setSyncWorkflowDraftHash,
+      setDraftUpdatedAt,
+      setToolPublished,
+    } = workflowStore.getState()
     try {
       const res = await fetchWorkflowDraft(`/rag/pipeline/${datasetId}/workflows/draft`)
       setData(res)
-      workflowStore.setState({
-        envSecrets: (res.environment_variables || []).filter(env => env.value_type === 'secret').reduce((acc, env) => {
+      setDraftUpdatedAt(res.updated_at)
+      setToolPublished(res.tool_published)
+      setEnvSecrets((res.environment_variables || []).filter(env => env.value_type === 'secret').reduce((acc, env) => {
           acc[env.id] = env.value
           return acc
-        }, {} as Record<string, string>),
-        environmentVariables: res.environment_variables?.map(env => env.value_type === 'secret' ? { ...env, value: '[__HIDDEN__]' } : env) || [],
-      })
+        }, {} as Record<string, string>))
+      setEnvironmentVariables(res.environment_variables?.map(env => env.value_type === 'secret' ? { ...env, value: '[__HIDDEN__]' } : env) || [])
       setSyncWorkflowDraftHash(res.hash)
       setIsLoading(false)
     }
@@ -68,49 +67,21 @@ export const usePipelineInit = () => {
                 environment_variables: [],
               },
             }).then((res) => {
-              workflowStore.getState().setDraftUpdatedAt(res.updated_at)
+              const { setDraftUpdatedAt } = workflowStore.getState()
+              setDraftUpdatedAt(res.updated_at)
               handleGetInitialWorkflowData()
             })
           }
         })
       }
     }
-  }, [nodesTemplate, edgesTemplate, workflowStore, setSyncWorkflowDraftHash, datasetId])
+  }, [nodesTemplate, edgesTemplate, workflowStore, datasetId])
 
   useEffect(() => {
-    // handleGetInitialWorkflowData()
-
+    handleGetInitialWorkflowData()
+  // eslint-disable-next-line react-hooks/exhaustive-deps
   }, [])
 
-  const handleFetchPreloadData = useCallback(async () => {
-    try {
-      const nodesDefaultConfigsData = await fetchNodesDefaultConfigs(`/rag/pipeline/${datasetId}/workflows/default-workflow-block-configs`)
-      const publishedWorkflow = await fetchPublishedWorkflow(`/rag/pipeline/${datasetId}/workflows/publish`)
-      workflowStore.setState({
-        nodesDefaultConfigs: nodesDefaultConfigsData.reduce((acc, block) => {
-          if (!acc[block.type])
-            acc[block.type] = { ...block.config }
-          return acc
-        }, {} as Record<string, any>),
-      })
-      workflowStore.getState().setPublishedAt(publishedWorkflow?.created_at)
-    }
-    catch (e) {
-      console.error(e)
-    }
-  }, [workflowStore, datasetId])
-
-  useEffect(() => {
-    // handleFetchPreloadData()
-  }, [handleFetchPreloadData])
-
-  useEffect(() => {
-    if (data) {
-      workflowStore.getState().setDraftUpdatedAt(data.updated_at)
-      workflowStore.getState().setToolPublished(data.tool_published)
-    }
-  }, [data, workflowStore])
-
   return {
     data,
     isLoading,

web/app/components/rag-pipeline/store/index.ts

@@ -1,5 +1,6 @@
 import type { RAGPipelineVariables } from '@/models/pipeline'
 import type { StateCreator } from 'zustand'
+import type { ToolWithProvider } from '../../workflow/types'
 import { InputVarType } from '../../workflow/types'
 
 export type RagPipelineSliceShape = {
@@ -10,6 +11,8 @@ export type RagPipelineSliceShape = {
   setNodesDefaultConfigs: (nodesDefaultConfigs: Record<string, any>) => void
   ragPipelineVariables: RAGPipelineVariables
   setRagPipelineVariables: (ragPipelineVariables: RAGPipelineVariables) => void
+  dataSourceList: ToolWithProvider[]
+  setDataSourceList: (dataSourceList: ToolWithProvider[]) => void
 }
 
 export type CreateRagPipelineSliceSlice = StateCreator<RagPipelineSliceShape>
@@ -50,4 +53,6 @@ export const createRagPipelineSliceSlice: StateCreator<RagPipelineSliceShape> =
       }],
     }],
   setRagPipelineVariables: (ragPipelineVariables: RAGPipelineVariables) => set(() => ({ ragPipelineVariables })),
+  dataSourceList: [],
+  setDataSourceList: (dataSourceList: ToolWithProvider[]) => set(() => ({ dataSourceList })),
 })

web/app/components/tools/types.ts

@@ -29,6 +29,7 @@ export enum CollectionType {
   custom = 'api',
   model = 'model',
   workflow = 'workflow',
+  datasource = 'datasource',
 }
 
 export type Emoji = {

web/app/components/workflow/block-selector/data-sources.tsx

@@ -0,0 +1,51 @@
+import {
+  useRef,
+} from 'react'
+import type {
+  OnSelectBlock,
+  ToolWithProvider,
+} from '../types'
+import Tools from './tools'
+import { ViewType } from './view-type-select'
+import cn from '@/utils/classnames'
+import type { ListRef } from '@/app/components/workflow/block-selector/market-place-plugin/list'
+
+type AllToolsProps = {
+  className?: string
+  toolContentClassName?: string
+  searchText: string
+  onSelect: OnSelectBlock
+  dataSources: ToolWithProvider[]
+}
+
+const DataSources = ({
+  className,
+  toolContentClassName,
+  searchText,
+  onSelect,
+  dataSources,
+}: AllToolsProps) => {
+  const pluginRef = useRef<ListRef>(null)
+  const wrapElemRef = useRef<HTMLDivElement>(null)
+
+  return (
+    <div className={cn(className)}>
+      <div
+        ref={wrapElemRef}
+        className='max-h-[464px] overflow-y-auto'
+        onScroll={pluginRef.current?.handleScroll}
+      >
+        <Tools
+          className={toolContentClassName}
+          showWorkflowEmpty={false}
+          tools={dataSources}
+          onSelect={onSelect}
+          viewType={ViewType.flat}
+          hasSearchText={!!searchText}
+        />
+      </div>
+    </div>
+  )
+}
+
+export default DataSources

web/app/components/workflow/block-selector/hooks.ts

@@ -8,7 +8,7 @@ import {
   ToolTypeEnum,
 } from './types'
 
-export const useTabs = (noBlocks?: boolean) => {
+export const useTabs = (noBlocks?: boolean, noSources?: boolean) => {
   const { t } = useTranslation()
   const tabs = useMemo(() => {
     return [
@@ -22,16 +22,22 @@ export const useTabs = (noBlocks?: boolean) => {
               },
             ]
       ),
-      {
-        key: TabsEnum.Sources,
-        name: t('workflow.tabs.sources'),
-      },
+      ...(
+        noSources
+          ? []
+          : [
+            {
+              key: TabsEnum.Sources,
+              name: t('workflow.tabs.sources'),
+            },
+          ]
+      ),
       {
         key: TabsEnum.Tools,
         name: t('workflow.tabs.tools'),
       },
     ]
-  }, [t, noBlocks])
+  }, [t, noBlocks, noSources])
   const initialTab = useMemo(() => {
     if (noBlocks)
       return TabsEnum.Sources

web/app/components/workflow/block-selector/index.tsx

@@ -3,6 +3,7 @@ import type { NodeSelectorProps } from './main'
 import NodeSelector from './main'
 import { useHooksStore } from '@/app/components/workflow/hooks-store/store'
 import { BlockEnum } from '@/app/components/workflow/types'
+import { useStore } from '@/app/components/workflow/store'
 
 const NodeSelectorWrapper = (props: NodeSelectorProps) => {
   const availableNodesMetaData = useHooksStore(s => s.availableNodesMetaData)
@@ -27,10 +28,13 @@ const NodeSelectorWrapper = (props: NodeSelectorProps) => {
     })
   }, [availableNodesMetaData?.nodes])
 
+  const dataSourceList = useStore(s => s.dataSourceList)
+
   return (
     <NodeSelector
       {...props}
       blocks={blocks}
+      dataSources={dataSourceList || []}
     />
   )
 }

web/app/components/workflow/block-selector/main.tsx

@@ -17,6 +17,7 @@ import type {
   BlockEnum,
   NodeDefault,
   OnSelectBlock,
+  ToolWithProvider,
 } from '../types'
 import Tabs from './tabs'
 import { TabsEnum } from './types'
@@ -49,6 +50,7 @@ export type NodeSelectorProps = {
   availableBlocksTypes?: BlockEnum[]
   disabled?: boolean
   blocks?: NodeDefault[]
+  dataSources?: ToolWithProvider[]
 }
 const NodeSelector: FC<NodeSelectorProps> = ({
   open: openFromProps,
@@ -65,6 +67,7 @@ const NodeSelector: FC<NodeSelectorProps> = ({
   availableBlocksTypes,
   disabled,
   blocks = [],
+  dataSources = [],
 }) => {
   const { t } = useTranslation()
   const [searchText, setSearchText] = useState('')
@@ -95,7 +98,7 @@ const NodeSelector: FC<NodeSelectorProps> = ({
     activeTab,
     setActiveTab,
     tabs,
-  } = useTabs(!blocks.length)
+  } = useTabs(!blocks.length, !dataSources.length)
 
   const searchPlaceholder = useMemo(() => {
     if (activeTab === TabsEnum.Blocks)
@@ -193,6 +196,7 @@ const NodeSelector: FC<NodeSelectorProps> = ({
             tags={tags}
             availableBlocksTypes={availableBlocksTypes}
             blocks={blocks}
+            dataSources={dataSources}
           />
         </div>
       </PortalToFollowElemContent>

web/app/components/workflow/block-selector/tabs.tsx

@@ -4,11 +4,13 @@ import { useAllBuiltInTools, useAllCustomTools, useAllWorkflowTools } from '@/se
 import type {
   BlockEnum,
   NodeDefault,
+  ToolWithProvider,
 } from '../types'
 import type { ToolDefaultValue } from './types'
 import { TabsEnum } from './types'
 import Blocks from './blocks'
 import AllTools from './all-tools'
+import DataSources from './data-sources'
 
 export type TabsProps = {
   activeTab: TabsEnum
@@ -17,6 +19,7 @@ export type TabsProps = {
   onSelect: (type: BlockEnum, tool?: ToolDefaultValue) => void
   availableBlocksTypes?: BlockEnum[]
   blocks: NodeDefault[]
+  dataSources?: ToolWithProvider[]
 }
 const Tabs: FC<TabsProps> = ({
   activeTab,
@@ -25,6 +28,7 @@ const Tabs: FC<TabsProps> = ({
   onSelect,
   availableBlocksTypes,
   blocks,
+  dataSources = [],
 }) => {
   const { data: buildInTools } = useAllBuiltInTools()
   const { data: customTools } = useAllCustomTools()
@@ -42,6 +46,15 @@ const Tabs: FC<TabsProps> = ({
           />
         )
       }
+      {
+        activeTab === TabsEnum.Sources && !!dataSources.length && (
+          <DataSources
+            searchText={searchText}
+            onSelect={onSelect}
+            dataSources={dataSources}
+          />
+        )
+      }
       {
         activeTab === TabsEnum.Tools && (
           <AllTools

web/service/use-workflow.ts

@@ -21,11 +21,12 @@ export const useAppWorkflow = (appID: string) => {
   })
 }
 
-export const useWorkflowConfig = (url: string, onSuccess: (v: WorkflowConfigResponse) => void) => {
+export const useWorkflowConfig = <T = WorkflowConfigResponse>(url: string, onSuccess: (v: T) => void) => {
   return useQuery({
+    enabled: !!url,
     queryKey: [NAME_SPACE, 'config', url],
     queryFn: async () => {
-      const data = await get<WorkflowConfigResponse>(url)
+      const data = await get<T>(url)
       onSuccess(data)
       return data
     },