Merge branch 'feat/rag-pipeline' into deploy/rag-dev

web/app/components/datasets/documents/create-from-pipeline/data-source-options/hooks.tsx

@@ -24,7 +24,7 @@ export const useDatasourceIcon = (data: DataSourceNodeType) => {
   useDataSourceList(!!pipelineId, handleUpdateDataSourceList)
 
   const datasourceIcon = useMemo(() => {
-    return dataSourceList?.find(toolWithProvider => toolWithProvider.name === data.provider_id)?.icon
+    return dataSourceList?.find(toolWithProvider => toolWithProvider.plugin_id === data.plugin_id)?.icon
   }, [data, dataSourceList])
 
   return datasourceIcon

web/app/components/datasets/settings/chunk-structure/index.tsx

@@ -29,6 +29,7 @@ const ChunkStructure = ({
             effectColor={option.effectColor}
             showEffectColor
             className='gap-x-1.5 p-3 pr-4'
+            disabled
           />
         ))
       }

web/app/components/rag-pipeline/components/rag-pipeline-header/publisher/popup.tsx

@@ -31,7 +31,10 @@ import { useToastContext } from '@/app/components/base/toast'
 import { useParams, useRouter } from 'next/navigation'
 import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
 import { useInvalid } from '@/service/use-base'
-import { publishedPipelineInfoQueryKeyPrefix } from '@/service/use-pipeline'
+import {
+  publishedPipelineInfoQueryKeyPrefix,
+  usePublishAsCustomizedPipeline,
+} from '@/service/use-pipeline'
 import Confirm from '@/app/components/base/confirm'
 
 const PUBLISH_SHORTCUT = ['⌘', '⇧', 'P']
@@ -58,6 +61,10 @@ const Popup = () => {
     setFalse: hidePublishing,
     setTrue: showPublishing,
   }] = useBoolean(false)
+  const {
+    mutate: publishAsCustomizedPipeline,
+    isPending: isPublishingAsCustomizedPipeline,
+  } = usePublishAsCustomizedPipeline()
 
   const invalidPublishedPipelineInfo = useInvalid([...publishedPipelineInfoQueryKeyPrefix, pipelineId])
 
@@ -183,6 +190,8 @@ const Popup = () => {
         <Button
           className='w-full hover:bg-state-accent-hover hover:text-text-accent'
           variant='tertiary'
+          onClick={() => publishAsCustomizedPipeline({ pipelineId: pipelineId || '' })}
+          disabled={!publishedAt || isPublishingAsCustomizedPipeline}
         >
           <div className='flex grow items-center'>
             <RiHammerLine className='mr-2 h-4 w-4' />

web/app/components/workflow/hooks/use-nodes-meta-data.ts

@@ -30,7 +30,7 @@ export const useNodeMetaData = (node: Node) => {
   const nodeMetaData = availableNodesMetaData.nodesMap?.[data.type]
   const author = useMemo(() => {
     if (data.type === BlockEnum.DataSource)
-      return dataSourceList?.find(dataSource => dataSource.id === data.provider_id)?.author
+      return dataSourceList?.find(dataSource => dataSource.plugin_id === data.plugin_id)?.author
 
     if (data.type === BlockEnum.Tool) {
       if (data.provider_type === CollectionType.builtIn)
@@ -44,7 +44,7 @@ export const useNodeMetaData = (node: Node) => {
 
   const description = useMemo(() => {
     if (data.type === BlockEnum.DataSource)
-      return dataSourceList?.find(dataSource => dataSource.id === data.provider_id)?.description[language]
+      return dataSourceList?.find(dataSource => dataSource.plugin_id === data.plugin_id)?.description[language]
     if (data.type === BlockEnum.Tool) {
       if (data.provider_type === CollectionType.builtIn)
         return buildInTools.find(toolWithProvider => canFindTool(toolWithProvider.id, data.provider_id))?.description[language]

web/app/components/workflow/hooks/use-tool-icon.ts

@@ -33,7 +33,7 @@ export const useToolIcon = (data: Node['data']) => {
       return targetTools.find(toolWithProvider => canFindTool(toolWithProvider.id, data.provider_id))?.icon
     }
     if (data.type === BlockEnum.DataSource)
-      return dataSourceList?.find(toolWithProvider => toolWithProvider.name === data.provider_id)?.icon
+      return dataSourceList?.find(toolWithProvider => toolWithProvider.plugin_id === data.plugin_id)?.icon
   }, [data, buildInTools, customTools, workflowTools, dataSourceList])
 
   return toolIcon
@@ -61,7 +61,7 @@ export const useGetToolIcon = () => {
     }
 
     if (data.type === BlockEnum.DataSource)
-      return dataSourceList?.find(toolWithProvider => toolWithProvider.name === data.provider_id)?.icon
+      return dataSourceList?.find(toolWithProvider => toolWithProvider.plugin_id === data.plugin_id)?.icon
   }, [workflowStore])
 
   return getToolIcon

web/i18n/en-US/dataset-documents.ts

@@ -97,6 +97,9 @@ const translation = {
       upload_file: 'Upload File',
       notion: 'Sync form Notion',
       github: 'Sync form Github',
+      local_file: 'Local File',
+      website_crawl: 'Website Crawl',
+      online_document: 'Online Document',
     },
     type: {
       book: 'Book',

web/i18n/zh-Hans/dataset-documents.ts

@@ -96,6 +96,9 @@ const translation = {
       upload_file: '文件上传',
       notion: '从 Notion 同步的文档',
       github: '从 Github 同步的代码',
+      local_file: '本地文件',
+      website_crawl: '网站爬取',
+      online_document: '在线文档',
     },
     type: {
       book: '书籍',

web/service/use-pipeline.ts

@@ -342,3 +342,14 @@ export const useExportPipelineDSL = () => {
     },
   })
 }
+
+export const usePublishAsCustomizedPipeline = () => {
+  return useMutation({
+    mutationKey: [NAME_SPACE, 'publish-as-customized-pipeline'],
+    mutationFn: ({
+      pipelineId,
+    }: { pipelineId: string }) => {
+      return get(`/rag/customized/pipelines/${pipelineId}/publish`)
+    },
+  })
+}