mirror of
https://github.com/langgenius/dify.git
synced 2025-11-02 11:52:58 +00:00
feat: Implement Notion connector and related components for data source selection in the RAG pipeline
This commit is contained in:
twwu
parent
d768094376
commit
44b9ce0951
27
web/app/components/base/notion-connector/index.tsx
Normal file
27
web/app/components/base/notion-connector/index.tsx
Normal file
@ -0,0 +1,27 @@
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import { Notion } from '../icons/src/public/common'
|
||||
import { Icon3Dots } from '../icons/src/vender/line/others'
|
||||
import Button from '../button'
|
||||
|
||||
type NotionConnectorProps = {
|
||||
onSetting: () => void
|
||||
}
|
||||
export const NotionConnector = ({ onSetting }: NotionConnectorProps) => {
|
||||
const { t } = useTranslation()
|
||||
|
||||
return (
|
||||
<div className='flex flex-col items-start rounded-2xl bg-workflow-process-bg p-6'>
|
||||
<div className='mb-2 h-12 w-12 rounded-[10px] border-[0.5px] border-components-card-border p-3 shadow-lg shadow-shadow-shadow-5'>
|
||||
<Notion className='size-6' />
|
||||
</div>
|
||||
<div className='mb-1 flex flex-col gap-y-1 pb-3 pt-1'>
|
||||
<span className='system-md-semibold text-text-secondary'>
|
||||
{t('datasetCreation.stepOne.notionSyncTitle')}
|
||||
<Icon3Dots className='relative -left-1.5 -top-2.5 inline h-4 w-4 text-text-secondary' />
|
||||
</span>
|
||||
<div className='system-sm-regular text-text-tertiary'>{t('datasetCreation.stepOne.notionSyncTip')}</div>
|
||||
</div>
|
||||
<Button className='h-8' variant='primary' onClick={onSetting}>{t('datasetCreation.stepOne.connect')}</Button>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@ -5,9 +5,9 @@ import WorkspaceSelector from './workspace-selector'
|
||||
import SearchInput from './search-input'
|
||||
import PageSelector from './page-selector'
|
||||
import { preImportNotionPages } from '@/service/datasets'
|
||||
import { NotionConnector } from '@/app/components/datasets/create/step-one'
|
||||
import type { DataSourceNotionPageMap, DataSourceNotionWorkspace, NotionPage } from '@/models/common'
|
||||
import { useModalContext } from '@/context/modal-context'
|
||||
import { useModalContextSelector } from '@/context/modal-context'
|
||||
import { NotionConnector } from '../notion-connector'
|
||||
|
||||
type NotionPageSelectorProps = {
|
||||
value?: string[]
|
||||
@ -30,7 +30,7 @@ const NotionPageSelector = ({
|
||||
const [prevData, setPrevData] = useState(data)
|
||||
const [searchValue, setSearchValue] = useState('')
|
||||
const [currentWorkspaceId, setCurrentWorkspaceId] = useState('')
|
||||
const { setShowAccountSettingModal } = useModalContext()
|
||||
const setShowAccountSettingModal = useModalContextSelector(s => s.setShowAccountSettingModal)
|
||||
|
||||
const notionWorkspaces = useMemo(() => {
|
||||
return data?.notion_info || []
|
||||
@ -87,11 +87,11 @@ const NotionPageSelector = ({
|
||||
}, [firstWorkspaceId])
|
||||
|
||||
return (
|
||||
<div className='rounded-xl border border-components-panel-border bg-background-default-subtle'>
|
||||
<>
|
||||
{
|
||||
data?.notion_info?.length
|
||||
? (
|
||||
<>
|
||||
<div className='rounded-xl border border-components-panel-border bg-background-default-subtle'>
|
||||
<div className='flex h-12 items-center gap-x-2 rounded-t-xl border-b border-b-divider-regular bg-components-panel-bg p-2'>
|
||||
<div className='flex grow items-center gap-x-1'>
|
||||
<WorkspaceSelector
|
||||
@ -123,13 +123,13 @@ const NotionPageSelector = ({
|
||||
onPreview={handlePreviewPage}
|
||||
/>
|
||||
</div>
|
||||
</>
|
||||
</div>
|
||||
)
|
||||
: (
|
||||
<NotionConnector onSetting={() => setShowAccountSettingModal({ payload: 'data-source', onCancelCallback: mutate })} />
|
||||
)
|
||||
}
|
||||
</div>
|
||||
</>
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@ -19,8 +19,9 @@ import { useDatasetDetailContext } from '@/context/dataset-detail'
|
||||
import { useProviderContext } from '@/context/provider-context'
|
||||
import VectorSpaceFull from '@/app/components/billing/vector-space-full'
|
||||
import classNames from '@/utils/classnames'
|
||||
import { Icon3Dots } from '@/app/components/base/icons/src/vender/line/others'
|
||||
import { ENABLE_WEBSITE_FIRECRAWL, ENABLE_WEBSITE_JINAREADER, ENABLE_WEBSITE_WATERCRAWL } from '@/config'
|
||||
import { NotionConnector } from '@/app/components/base/notion-connector'
|
||||
|
||||
type IStepOneProps = {
|
||||
datasetId?: string
|
||||
dataSourceType?: DataSourceType
|
||||
@ -42,27 +43,6 @@ type IStepOneProps = {
|
||||
onCrawlOptionsChange: (payload: CrawlOptions) => void
|
||||
}
|
||||
|
||||
type NotionConnectorProps = {
|
||||
onSetting: () => void
|
||||
}
|
||||
export const NotionConnector = ({ onSetting }: NotionConnectorProps) => {
|
||||
const { t } = useTranslation()
|
||||
|
||||
return (
|
||||
<div className='flex w-[640px] flex-col items-start rounded-2xl bg-workflow-process-bg p-6'>
|
||||
<span className={cn(s.notionIcon, 'mb-2 h-12 w-12 rounded-[10px] border-[0.5px] border-components-card-border p-3 shadow-lg shadow-shadow-shadow-5')} />
|
||||
<div className='mb-1 flex flex-col gap-y-1 pb-3 pt-1'>
|
||||
<span className='system-md-semibold text-text-secondary'>
|
||||
{t('datasetCreation.stepOne.notionSyncTitle')}
|
||||
<Icon3Dots className='relative -left-1.5 -top-2.5 inline h-4 w-4 text-text-secondary' />
|
||||
</span>
|
||||
<div className='system-sm-regular text-text-tertiary'>{t('datasetCreation.stepOne.notionSyncTip')}</div>
|
||||
</div>
|
||||
<Button className='h-8' variant='primary' onClick={onSetting}>{t('datasetCreation.stepOne.connect')}</Button>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
const StepOne = ({
|
||||
datasetId,
|
||||
dataSourceType: inCreatePageDataSourceType,
|
||||
|
||||
@ -1,10 +1,14 @@
|
||||
import { useMemo } from 'react'
|
||||
import type { PanelProps } from '@/app/components/workflow/panel'
|
||||
import Panel from '@/app/components/workflow/panel'
|
||||
import { useStore } from '@/app/components/workflow/store'
|
||||
import TestRunPanel from './test-run'
|
||||
|
||||
const RagPipelinePanelOnRight = () => {
|
||||
const showTestRunPanel = useStore(s => s.showTestRunPanel)
|
||||
return (
|
||||
<>
|
||||
{showTestRunPanel && <TestRunPanel />}
|
||||
</>
|
||||
)
|
||||
}
|
||||
|
||||
@ -0,0 +1,50 @@
|
||||
import { useCallback } from 'react'
|
||||
import { useDataSourceOptions } from '../hooks'
|
||||
import OptionCard from './option-card'
|
||||
import { File, Watercrawl } from '@/app/components/base/icons/src/public/knowledge'
|
||||
import { Notion } from '@/app/components/base/icons/src/public/common'
|
||||
import { Jina } from '@/app/components/base/icons/src/public/llm'
|
||||
import { DataSourceType } from '@/models/datasets'
|
||||
import { DataSourceProvider } from '@/models/common'
|
||||
|
||||
type DataSourceOptionsProps = {
|
||||
dataSources: string[]
|
||||
dataSourceType: string
|
||||
onSelect: (option: string) => void
|
||||
}
|
||||
|
||||
const DATA_SOURCE_ICONS = {
|
||||
[DataSourceType.FILE]: File as React.FC<React.SVGProps<SVGSVGElement>>,
|
||||
[DataSourceType.NOTION]: Notion as React.FC<React.SVGProps<SVGSVGElement>>,
|
||||
[DataSourceProvider.fireCrawl]: '🔥',
|
||||
[DataSourceProvider.jinaReader]: Jina as React.FC<React.SVGProps<SVGSVGElement>>,
|
||||
[DataSourceProvider.waterCrawl]: Watercrawl as React.FC<React.SVGProps<SVGSVGElement>>,
|
||||
}
|
||||
|
||||
const DataSourceOptions = ({
|
||||
dataSources,
|
||||
dataSourceType,
|
||||
onSelect,
|
||||
}: DataSourceOptionsProps) => {
|
||||
const options = useDataSourceOptions(dataSources)
|
||||
|
||||
const handelSelect = useCallback((value: string) => {
|
||||
onSelect(value)
|
||||
}, [onSelect])
|
||||
|
||||
return (
|
||||
<div className='grid w-full grid-cols-4 gap-1'>
|
||||
{options.map(option => (
|
||||
<OptionCard
|
||||
key={option.value}
|
||||
label={option.label}
|
||||
selected={dataSourceType === option.value}
|
||||
Icon={DATA_SOURCE_ICONS[option.value as keyof typeof DATA_SOURCE_ICONS]}
|
||||
onClick={handelSelect.bind(null, option.value)}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default DataSourceOptions
|
||||
@ -0,0 +1,40 @@
|
||||
import cn from '@/utils/classnames'
|
||||
|
||||
type OptionCardProps = {
|
||||
label: string
|
||||
Icon: React.FC<React.SVGProps<SVGSVGElement>> | string
|
||||
selected: boolean
|
||||
onClick?: () => void
|
||||
}
|
||||
|
||||
const OptionCard = ({
|
||||
label,
|
||||
Icon,
|
||||
selected,
|
||||
onClick,
|
||||
}: OptionCardProps) => {
|
||||
return (
|
||||
<div
|
||||
className={cn(
|
||||
'flex flex-col gap-1 rounded-xl border border-components-option-card-option-border bg-components-option-card-option-bg p-2.5 shadow-shadow-shadow-3',
|
||||
selected
|
||||
? 'border-components-option-card-option-selected-border bg-components-option-card-option-selected-bg shadow-xs ring-[0.5px] ring-inset ring-components-option-card-option-selected-border'
|
||||
: 'hover:bg-components-option-card-bg-hover hover:border-components-option-card-option-border-hover hover:shadow-xs',
|
||||
)}
|
||||
onClick={onClick}
|
||||
>
|
||||
<div className='flex size-7 items-center justify-center rounded-lg border-[0.5px] border-components-panel-border bg-background-default-dodge p-1'>
|
||||
{
|
||||
typeof Icon === 'string'
|
||||
? <div className='text-[18px] leading-[18px]'>{Icon}</div>
|
||||
: <Icon className='size-5' />
|
||||
}
|
||||
</div>
|
||||
<div className={cn('system-sm-medium text-text-secondary', selected && 'text-primary')}>
|
||||
{label}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default OptionCard
|
||||
@ -0,0 +1,334 @@
|
||||
'use client'
|
||||
import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import { useContext } from 'use-context-selector'
|
||||
import { RiDeleteBinLine, RiErrorWarningFill, RiUploadCloud2Line } from '@remixicon/react'
|
||||
import DocumentFileIcon from '@/app/components/datasets/common/document-file-icon'
|
||||
import cn from '@/utils/classnames'
|
||||
import type { CustomFile as File, FileItem } from '@/models/datasets'
|
||||
import { ToastContext } from '@/app/components/base/toast'
|
||||
import SimplePieChart from '@/app/components/base/simple-pie-chart'
|
||||
import { upload } from '@/service/base'
|
||||
import I18n from '@/context/i18n'
|
||||
import { LanguagesSupported } from '@/i18n/language'
|
||||
import { IS_CE_EDITION } from '@/config'
|
||||
import { Theme } from '@/types/app'
|
||||
import useTheme from '@/hooks/use-theme'
|
||||
import { useFileSupportTypes, useFileUploadConfig } from '@/service/use-common'
|
||||
|
||||
const FILES_NUMBER_LIMIT = 20
|
||||
|
||||
type IFileUploaderProps = {
|
||||
fileList: FileItem[]
|
||||
prepareFileList: (files: FileItem[]) => void
|
||||
onFileUpdate: (fileItem: FileItem, progress: number, list: FileItem[]) => void
|
||||
onFileListUpdate?: (files: FileItem[]) => void
|
||||
notSupportBatchUpload?: boolean
|
||||
}
|
||||
|
||||
const FileUploader = ({
|
||||
fileList,
|
||||
prepareFileList,
|
||||
onFileUpdate,
|
||||
onFileListUpdate,
|
||||
notSupportBatchUpload,
|
||||
}: IFileUploaderProps) => {
|
||||
const { t } = useTranslation()
|
||||
const { notify } = useContext(ToastContext)
|
||||
const { locale } = useContext(I18n)
|
||||
const [dragging, setDragging] = useState(false)
|
||||
const dropRef = useRef<HTMLDivElement>(null)
|
||||
const dragRef = useRef<HTMLDivElement>(null)
|
||||
const fileUploader = useRef<HTMLInputElement>(null)
|
||||
const hideUpload = notSupportBatchUpload && fileList.length > 0
|
||||
|
||||
const { data: fileUploadConfigResponse } = useFileUploadConfig()
|
||||
const { data: supportFileTypesResponse } = useFileSupportTypes()
|
||||
const supportTypes = supportFileTypesResponse?.allowed_extensions || []
|
||||
const supportTypesShowNames = (() => {
|
||||
const extensionMap: { [key: string]: string } = {
|
||||
md: 'markdown',
|
||||
pptx: 'pptx',
|
||||
htm: 'html',
|
||||
xlsx: 'xlsx',
|
||||
docx: 'docx',
|
||||
}
|
||||
|
||||
return [...supportTypes]
|
||||
.map(item => extensionMap[item] || item) // map to standardized extension
|
||||
.map(item => item.toLowerCase()) // convert to lower case
|
||||
.filter((item, index, self) => self.indexOf(item) === index) // remove duplicates
|
||||
.map(item => item.toUpperCase()) // convert to upper case
|
||||
.join(locale !== LanguagesSupported[1] ? ', ' : '、 ')
|
||||
})()
|
||||
const ACCEPTS = supportTypes.map((ext: string) => `.${ext}`)
|
||||
const fileUploadConfig = useMemo(() => fileUploadConfigResponse ?? {
|
||||
file_size_limit: 15,
|
||||
batch_count_limit: 5,
|
||||
}, [fileUploadConfigResponse])
|
||||
|
||||
const fileListRef = useRef<FileItem[]>([])
|
||||
|
||||
// utils
|
||||
const getFileType = (currentFile: File) => {
|
||||
if (!currentFile)
|
||||
return ''
|
||||
|
||||
const arr = currentFile.name.split('.')
|
||||
return arr[arr.length - 1]
|
||||
}
|
||||
|
||||
const getFileSize = (size: number) => {
|
||||
if (size / 1024 < 10)
|
||||
return `${(size / 1024).toFixed(2)}KB`
|
||||
|
||||
return `${(size / 1024 / 1024).toFixed(2)}MB`
|
||||
}
|
||||
|
||||
const isValid = useCallback((file: File) => {
|
||||
const { size } = file
|
||||
const ext = `.${getFileType(file)}`
|
||||
const isValidType = ACCEPTS.includes(ext.toLowerCase())
|
||||
if (!isValidType)
|
||||
notify({ type: 'error', message: t('datasetCreation.stepOne.uploader.validation.typeError') })
|
||||
|
||||
const isValidSize = size <= fileUploadConfig.file_size_limit * 1024 * 1024
|
||||
if (!isValidSize)
|
||||
notify({ type: 'error', message: t('datasetCreation.stepOne.uploader.validation.size', { size: fileUploadConfig.file_size_limit }) })
|
||||
|
||||
return isValidType && isValidSize
|
||||
}, [fileUploadConfig, notify, t, ACCEPTS])
|
||||
|
||||
const fileUpload = useCallback(async (fileItem: FileItem): Promise<FileItem> => {
|
||||
const formData = new FormData()
|
||||
formData.append('file', fileItem.file)
|
||||
const onProgress = (e: ProgressEvent) => {
|
||||
if (e.lengthComputable) {
|
||||
const percent = Math.floor(e.loaded / e.total * 100)
|
||||
onFileUpdate(fileItem, percent, fileListRef.current)
|
||||
}
|
||||
}
|
||||
|
||||
return upload({
|
||||
xhr: new XMLHttpRequest(),
|
||||
data: formData,
|
||||
onprogress: onProgress,
|
||||
}, false, undefined, '?source=datasets')
|
||||
.then((res: File) => {
|
||||
const completeFile = {
|
||||
fileID: fileItem.fileID,
|
||||
file: res,
|
||||
progress: -1,
|
||||
}
|
||||
const index = fileListRef.current.findIndex(item => item.fileID === fileItem.fileID)
|
||||
fileListRef.current[index] = completeFile
|
||||
onFileUpdate(completeFile, 100, fileListRef.current)
|
||||
return Promise.resolve({ ...completeFile })
|
||||
})
|
||||
.catch((e) => {
|
||||
notify({ type: 'error', message: e?.response?.code === 'forbidden' ? e?.response?.message : t('datasetCreation.stepOne.uploader.failed') })
|
||||
onFileUpdate(fileItem, -2, fileListRef.current)
|
||||
return Promise.resolve({ ...fileItem })
|
||||
})
|
||||
.finally()
|
||||
}, [fileListRef, notify, onFileUpdate, t])
|
||||
|
||||
const uploadBatchFiles = useCallback((bFiles: FileItem[]) => {
|
||||
bFiles.forEach(bf => (bf.progress = 0))
|
||||
return Promise.all(bFiles.map(fileUpload))
|
||||
}, [fileUpload])
|
||||
|
||||
const uploadMultipleFiles = useCallback(async (files: FileItem[]) => {
|
||||
const batchCountLimit = fileUploadConfig.batch_count_limit
|
||||
const length = files.length
|
||||
let start = 0
|
||||
let end = 0
|
||||
|
||||
while (start < length) {
|
||||
if (start + batchCountLimit > length)
|
||||
end = length
|
||||
else
|
||||
end = start + batchCountLimit
|
||||
const bFiles = files.slice(start, end)
|
||||
await uploadBatchFiles(bFiles)
|
||||
start = end
|
||||
}
|
||||
}, [fileUploadConfig, uploadBatchFiles])
|
||||
|
||||
const initialUpload = useCallback((files: File[]) => {
|
||||
if (!files.length)
|
||||
return false
|
||||
|
||||
if (files.length + fileList.length > FILES_NUMBER_LIMIT && !IS_CE_EDITION) {
|
||||
notify({ type: 'error', message: t('datasetCreation.stepOne.uploader.validation.filesNumber', { filesNumber: FILES_NUMBER_LIMIT }) })
|
||||
return false
|
||||
}
|
||||
|
||||
const preparedFiles = files.map((file, index) => ({
|
||||
fileID: `file${index}-${Date.now()}`,
|
||||
file,
|
||||
progress: -1,
|
||||
}))
|
||||
const newFiles = [...fileListRef.current, ...preparedFiles]
|
||||
prepareFileList(newFiles)
|
||||
fileListRef.current = newFiles
|
||||
uploadMultipleFiles(preparedFiles)
|
||||
}, [prepareFileList, uploadMultipleFiles, notify, t, fileList])
|
||||
|
||||
const handleDragEnter = (e: DragEvent) => {
|
||||
e.preventDefault()
|
||||
e.stopPropagation()
|
||||
e.target !== dragRef.current && setDragging(true)
|
||||
}
|
||||
const handleDragOver = (e: DragEvent) => {
|
||||
e.preventDefault()
|
||||
e.stopPropagation()
|
||||
}
|
||||
const handleDragLeave = (e: DragEvent) => {
|
||||
e.preventDefault()
|
||||
e.stopPropagation()
|
||||
e.target === dragRef.current && setDragging(false)
|
||||
}
|
||||
|
||||
const handleDrop = useCallback((e: DragEvent) => {
|
||||
e.preventDefault()
|
||||
e.stopPropagation()
|
||||
setDragging(false)
|
||||
if (!e.dataTransfer)
|
||||
return
|
||||
|
||||
let files = [...e.dataTransfer.files] as File[]
|
||||
if (notSupportBatchUpload)
|
||||
files = files.slice(0, 1)
|
||||
|
||||
const validFiles = files.filter(isValid)
|
||||
initialUpload(validFiles)
|
||||
}, [initialUpload, isValid, notSupportBatchUpload])
|
||||
|
||||
const selectHandle = () => {
|
||||
if (fileUploader.current)
|
||||
fileUploader.current.click()
|
||||
}
|
||||
|
||||
const removeFile = (fileID: string) => {
|
||||
if (fileUploader.current)
|
||||
fileUploader.current.value = ''
|
||||
|
||||
fileListRef.current = fileListRef.current.filter(item => item.fileID !== fileID)
|
||||
onFileListUpdate?.([...fileListRef.current])
|
||||
}
|
||||
const fileChangeHandle = useCallback((e: React.ChangeEvent<HTMLInputElement>) => {
|
||||
const files = [...(e.target.files ?? [])] as File[]
|
||||
initialUpload(files.filter(isValid))
|
||||
}, [isValid, initialUpload])
|
||||
|
||||
const { theme } = useTheme()
|
||||
const chartColor = useMemo(() => theme === Theme.dark ? '#5289ff' : '#296dff', [theme])
|
||||
|
||||
useEffect(() => {
|
||||
const dropElement = dropRef.current
|
||||
dropElement?.addEventListener('dragenter', handleDragEnter)
|
||||
dropElement?.addEventListener('dragover', handleDragOver)
|
||||
dropElement?.addEventListener('dragleave', handleDragLeave)
|
||||
dropElement?.addEventListener('drop', handleDrop)
|
||||
return () => {
|
||||
dropElement?.removeEventListener('dragenter', handleDragEnter)
|
||||
dropElement?.removeEventListener('dragover', handleDragOver)
|
||||
dropElement?.removeEventListener('dragleave', handleDragLeave)
|
||||
dropElement?.removeEventListener('drop', handleDrop)
|
||||
}
|
||||
}, [handleDrop])
|
||||
|
||||
return (
|
||||
<div>
|
||||
{!hideUpload && (
|
||||
<input
|
||||
ref={fileUploader}
|
||||
id='fileUploader'
|
||||
className='hidden'
|
||||
type='file'
|
||||
multiple={!notSupportBatchUpload}
|
||||
accept={ACCEPTS.join(',')}
|
||||
onChange={fileChangeHandle}
|
||||
/>
|
||||
)}
|
||||
{!hideUpload && (
|
||||
<div
|
||||
ref={dropRef}
|
||||
className={cn(
|
||||
'relative box-border flex min-h-20 flex-col items-center justify-center gap-1 rounded-xl border border-dashed border-components-dropzone-border bg-components-dropzone-bg px-4 py-3 text-xs leading-4 text-text-tertiary',
|
||||
dragging && 'border-components-dropzone-border-accent bg-components-dropzone-bg-accent',
|
||||
)}>
|
||||
<div className='flex min-h-5 items-center justify-center text-sm leading-4 text-text-secondary'>
|
||||
<RiUploadCloud2Line className='mr-2 size-5' />
|
||||
|
||||
<span>
|
||||
{t('datasetCreation.stepOne.uploader.button')}
|
||||
{supportTypes.length > 0 && (
|
||||
<label className='ml-1 cursor-pointer text-text-accent' onClick={selectHandle}>{t('datasetCreation.stepOne.uploader.browse')}</label>
|
||||
)}
|
||||
</span>
|
||||
</div>
|
||||
<div>{t('datasetCreation.stepOne.uploader.tip', {
|
||||
size: fileUploadConfig.file_size_limit,
|
||||
supportTypes: supportTypesShowNames,
|
||||
})}</div>
|
||||
{dragging && <div ref={dragRef} className='absolute left-0 top-0 h-full w-full' />}
|
||||
</div>
|
||||
)}
|
||||
{fileList.length > 0 && (
|
||||
<div className='mt-1 flex flex-col gap-y-1'>
|
||||
{fileList.map((fileItem, index) => {
|
||||
const isUploading = fileItem.progress >= 0 && fileItem.progress < 100
|
||||
const isError = fileItem.progress === -2
|
||||
return (
|
||||
<div
|
||||
key={`${fileItem.fileID}-${index}`}
|
||||
className={cn(
|
||||
'flex h-12 items-center rounded-lg border border-components-panel-border bg-components-panel-on-panel-item-bg shadow-xs shadow-shadow-shadow-4',
|
||||
isError && 'border-state-destructive-border bg-state-destructive-hover',
|
||||
)}
|
||||
>
|
||||
<div className='flex w-12 shrink-0 items-center justify-center'>
|
||||
<DocumentFileIcon
|
||||
className='size-6 shrink-0'
|
||||
name={fileItem.file.name}
|
||||
extension={getFileType(fileItem.file)}
|
||||
/>
|
||||
</div>
|
||||
<div className='flex shrink grow flex-col gap-0.5'>
|
||||
<div className='flex w-full'>
|
||||
<div className='w-0 grow truncate text-xs text-text-secondary'>{fileItem.file.name}</div>
|
||||
</div>
|
||||
<div className='w-full truncate text-2xs leading-3 text-text-tertiary'>
|
||||
<span className='uppercase'>{getFileType(fileItem.file)}</span>
|
||||
<span className='px-1 text-text-quaternary'>·</span>
|
||||
<span>{getFileSize(fileItem.file.size)}</span>
|
||||
</div>
|
||||
</div>
|
||||
<div className='flex w-16 shrink-0 items-center justify-end gap-1 pr-3'>
|
||||
{isUploading && (
|
||||
<SimplePieChart percentage={fileItem.progress} stroke={chartColor} fill={chartColor} animationDuration={0} />
|
||||
)}
|
||||
{
|
||||
isError && (
|
||||
<RiErrorWarningFill className='size-4 text-text-destructive' />
|
||||
)
|
||||
}
|
||||
<span className='flex h-6 w-6 cursor-pointer items-center justify-center' onClick={(e) => {
|
||||
e.stopPropagation()
|
||||
removeFile(fileItem.fileID)
|
||||
}}>
|
||||
<RiDeleteBinLine className='size-4 text-text-tertiary' />
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
})}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default FileUploader
|
||||
@ -0,0 +1,36 @@
|
||||
import VectorSpaceFull from '@/app/components/billing/vector-space-full'
|
||||
import type { FileItem } from '@/models/datasets'
|
||||
import FileUploader from './file-uploader'
|
||||
|
||||
type LocalFileProps = {
|
||||
files: FileItem[]
|
||||
updateFileList: (files: FileItem[]) => void
|
||||
updateFile: (fileItem: FileItem, progress: number, list: FileItem[]) => void
|
||||
notSupportBatchUpload: boolean
|
||||
isShowVectorSpaceFull: boolean
|
||||
}
|
||||
|
||||
const LocalFile = ({
|
||||
files,
|
||||
updateFileList,
|
||||
updateFile,
|
||||
notSupportBatchUpload,
|
||||
isShowVectorSpaceFull,
|
||||
}: LocalFileProps) => {
|
||||
return (
|
||||
<>
|
||||
<FileUploader
|
||||
fileList={files}
|
||||
prepareFileList={updateFileList}
|
||||
onFileListUpdate={updateFileList}
|
||||
onFileUpdate={updateFile}
|
||||
notSupportBatchUpload={notSupportBatchUpload}
|
||||
/>
|
||||
{isShowVectorSpaceFull && (
|
||||
<VectorSpaceFull />
|
||||
)}
|
||||
</>
|
||||
)
|
||||
}
|
||||
|
||||
export default LocalFile
|
||||
@ -0,0 +1,51 @@
|
||||
import { useDataSources } from '@/service/use-common'
|
||||
import { useCallback, useMemo } from 'react'
|
||||
import { NotionPageSelector } from '@/app/components/base/notion-page-selector'
|
||||
import type { NotionPage } from '@/models/common'
|
||||
import VectorSpaceFull from '@/app/components/billing/vector-space-full'
|
||||
import { NotionConnector } from '@/app/components/base/notion-connector'
|
||||
import { useModalContextSelector } from '@/context/modal-context'
|
||||
|
||||
type NotionProps = {
|
||||
notionPages: NotionPage[]
|
||||
updateNotionPages: (value: NotionPage[]) => void
|
||||
isShowVectorSpaceFull: boolean
|
||||
}
|
||||
|
||||
const Notion = ({
|
||||
notionPages,
|
||||
updateNotionPages,
|
||||
isShowVectorSpaceFull,
|
||||
}: NotionProps) => {
|
||||
const { data: dataSources } = useDataSources()
|
||||
const setShowAccountSettingModal = useModalContextSelector(state => state.setShowAccountSettingModal)
|
||||
|
||||
const hasConnection = useMemo(() => {
|
||||
const notionDataSources = dataSources?.data.filter(item => item.provider === 'notion') || []
|
||||
return notionDataSources.length > 0
|
||||
}, [dataSources])
|
||||
|
||||
const handleConnect = useCallback(() => {
|
||||
setShowAccountSettingModal({ payload: 'data-source' })
|
||||
}, [setShowAccountSettingModal])
|
||||
|
||||
return (
|
||||
<>
|
||||
{!hasConnection && <NotionConnector onSetting={handleConnect} />}
|
||||
{hasConnection && (
|
||||
<>
|
||||
<NotionPageSelector
|
||||
value={notionPages.map(page => page.page_id)}
|
||||
onSelect={updateNotionPages}
|
||||
canPreview={false}
|
||||
/>
|
||||
{isShowVectorSpaceFull && (
|
||||
<VectorSpaceFull />
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
</>
|
||||
)
|
||||
}
|
||||
|
||||
export default Notion
|
||||
@ -0,0 +1,60 @@
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import type { DataSourceOption } from './types'
|
||||
import { TestRunStep } from './types'
|
||||
import { DataSourceType } from '@/models/datasets'
|
||||
import { DataSourceProvider } from '@/models/common'
|
||||
|
||||
export const useTestRunSteps = () => {
|
||||
// TODO: i18n
|
||||
const { t } = useTranslation()
|
||||
const steps = [
|
||||
{
|
||||
label: 'DATA SOURCE',
|
||||
value: TestRunStep.dataSource,
|
||||
},
|
||||
{
|
||||
label: 'DOCUMENT PROCESSING',
|
||||
value: TestRunStep.documentProcessing,
|
||||
},
|
||||
]
|
||||
return steps
|
||||
}
|
||||
|
||||
export const useDataSourceOptions = (dataSources: string[]) => {
|
||||
// TODO: i18n
|
||||
const { t } = useTranslation()
|
||||
const options: DataSourceOption[] = []
|
||||
dataSources.forEach((source) => {
|
||||
if (source === DataSourceType.FILE) {
|
||||
options.push({
|
||||
label: 'Local Files',
|
||||
value: DataSourceType.FILE,
|
||||
})
|
||||
}
|
||||
if (source === DataSourceType.NOTION) {
|
||||
options.push({
|
||||
label: 'Notion',
|
||||
value: DataSourceType.NOTION,
|
||||
})
|
||||
}
|
||||
if (source === DataSourceProvider.fireCrawl) {
|
||||
options.push({
|
||||
label: 'Firecrawl',
|
||||
value: DataSourceProvider.fireCrawl,
|
||||
})
|
||||
}
|
||||
if (source === DataSourceProvider.jinaReader) {
|
||||
options.push({
|
||||
label: 'Jina Reader',
|
||||
value: DataSourceProvider.jinaReader,
|
||||
})
|
||||
}
|
||||
if (source === DataSourceProvider.waterCrawl) {
|
||||
options.push({
|
||||
label: 'Water Crawl',
|
||||
value: DataSourceProvider.waterCrawl,
|
||||
})
|
||||
}
|
||||
})
|
||||
return options
|
||||
}
|
||||
@ -0,0 +1,134 @@
|
||||
import { useStore } from '@/app/components/workflow/store'
|
||||
import { RiCloseLine } from '@remixicon/react'
|
||||
import { useCallback, useMemo, useState } from 'react'
|
||||
import StepIndicator from './step-indicator'
|
||||
import { useTestRunSteps } from './hooks'
|
||||
import DataSourceOptions from './data-source-options'
|
||||
import type { FileItem } from '@/models/datasets'
|
||||
import { DataSourceType } from '@/models/datasets'
|
||||
import LocalFile from './data-source/local-file'
|
||||
import produce from 'immer'
|
||||
import Button from '@/app/components/base/button'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import { useProviderContextSelector } from '@/context/provider-context'
|
||||
import type { NotionPage } from '@/models/common'
|
||||
import Notion from './data-source/notion'
|
||||
|
||||
const TestRunPanel = () => {
|
||||
const { t } = useTranslation()
|
||||
const [currentStep, setCurrentStep] = useState(0)
|
||||
const [dataSourceType, setDataSourceType] = useState<string>(DataSourceType.FILE)
|
||||
const [fileList, setFiles] = useState<FileItem[]>([])
|
||||
const [notionPages, setNotionPages] = useState<NotionPage[]>([])
|
||||
|
||||
const setShowTestRunPanel = useStore(s => s.setShowTestRunPanel)
|
||||
const plan = useProviderContextSelector(state => state.plan)
|
||||
const enableBilling = useProviderContextSelector(state => state.enableBilling)
|
||||
|
||||
const steps = useTestRunSteps()
|
||||
const dataSources = ['upload_file', 'notion_import', 'firecrawl', 'jinareader', 'watercrawl'] // TODO: replace with real data sources
|
||||
|
||||
const allFileLoaded = (fileList.length > 0 && fileList.every(file => file.file.id))
|
||||
const isVectorSpaceFull = plan.usage.vectorSpace >= plan.total.vectorSpace
|
||||
const isShowVectorSpaceFull = allFileLoaded && isVectorSpaceFull && enableBilling
|
||||
const notSupportBatchUpload = enableBilling && plan.type === 'sandbox'
|
||||
const nextDisabled = useMemo(() => {
|
||||
if (!fileList.length)
|
||||
return true
|
||||
if (fileList.some(file => !file.file.id))
|
||||
return true
|
||||
return isShowVectorSpaceFull
|
||||
}, [fileList, isShowVectorSpaceFull])
|
||||
|
||||
const handleClose = () => {
|
||||
setShowTestRunPanel?.(false)
|
||||
}
|
||||
|
||||
const handleDataSourceSelect = useCallback((option: string) => {
|
||||
setDataSourceType(option)
|
||||
}, [])
|
||||
|
||||
const updateFile = (fileItem: FileItem, progress: number, list: FileItem[]) => {
|
||||
const newList = produce(list, (draft) => {
|
||||
const targetIndex = draft.findIndex(file => file.fileID === fileItem.fileID)
|
||||
draft[targetIndex] = {
|
||||
...draft[targetIndex],
|
||||
progress,
|
||||
}
|
||||
})
|
||||
setFiles(newList)
|
||||
}
|
||||
|
||||
const updateFileList = (preparedFiles: FileItem[]) => {
|
||||
setFiles(preparedFiles)
|
||||
}
|
||||
|
||||
const updateNotionPages = (value: NotionPage[]) => {
|
||||
setNotionPages(value)
|
||||
}
|
||||
|
||||
const handleNextStep = useCallback(() => {
|
||||
setCurrentStep(preStep => preStep + 1)
|
||||
}, [])
|
||||
|
||||
return (
|
||||
<div className='relative flex h-full w-[480px] flex-col rounded-l-2xl border-y-[0.5px] border-l-[0.5px] border-components-panel-border bg-components-panel-bg shadow-xl shadow-shadow-shadow-1'>
|
||||
<button
|
||||
type='button'
|
||||
className='absolute right-2.5 top-2.5 flex size-8 items-center justify-center p-1.5'
|
||||
onClick={handleClose}
|
||||
>
|
||||
<RiCloseLine className='size-4 text-text-tertiary' />
|
||||
</button>
|
||||
<div className='flex flex-col gap-y-0.5 px-3 pb-2 pt-3.5'>
|
||||
<div className='system-md-semibold-uppercase flex items-center justify-between pl-1 pr-8 text-text-primary'>
|
||||
TEST RUN
|
||||
</div>
|
||||
<StepIndicator steps={steps} currentStep={currentStep} />
|
||||
</div>
|
||||
{
|
||||
currentStep === 0 && (
|
||||
<>
|
||||
<div className='flex flex-col gap-y-4 px-4 py-2'>
|
||||
<DataSourceOptions
|
||||
dataSources={dataSources}
|
||||
dataSourceType={dataSourceType}
|
||||
onSelect={handleDataSourceSelect}
|
||||
/>
|
||||
{dataSourceType === DataSourceType.FILE && (
|
||||
<LocalFile
|
||||
files={fileList}
|
||||
updateFile={updateFile}
|
||||
updateFileList={updateFileList}
|
||||
notSupportBatchUpload={notSupportBatchUpload}
|
||||
isShowVectorSpaceFull={isShowVectorSpaceFull}
|
||||
/>
|
||||
)}
|
||||
{dataSourceType === DataSourceType.NOTION && (
|
||||
<Notion
|
||||
notionPages={notionPages}
|
||||
updateNotionPages={updateNotionPages}
|
||||
isShowVectorSpaceFull={isShowVectorSpaceFull}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
<div className='flex justify-end p-4 pt-2'>
|
||||
{dataSourceType === DataSourceType.FILE && (
|
||||
<Button disabled={nextDisabled} variant='primary' onClick={handleNextStep}>
|
||||
<span className='px-0.5'>{t('datasetCreation.stepOne.button')}</span>
|
||||
</Button>
|
||||
)}
|
||||
{dataSourceType === DataSourceType.NOTION && (
|
||||
<Button disabled={isShowVectorSpaceFull || !notionPages.length} variant='primary' onClick={handleNextStep}>
|
||||
<span className="px-0.5">{t('datasetCreation.stepOne.button')}</span>
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
</>
|
||||
)
|
||||
}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default TestRunPanel
|
||||
@ -0,0 +1,44 @@
|
||||
import Divider from '@/app/components/base/divider'
|
||||
import cn from '@/utils/classnames'
|
||||
import React from 'react'
|
||||
|
||||
type Step = {
|
||||
label: string
|
||||
value: string
|
||||
}
|
||||
|
||||
type StepIndicatorProps = {
|
||||
currentStep: number
|
||||
steps: Step[]
|
||||
}
|
||||
|
||||
const StepIndicator = ({
|
||||
currentStep,
|
||||
steps,
|
||||
}: StepIndicatorProps) => {
|
||||
return (
|
||||
<div className='flex items-center gap-x-2 px-1'>
|
||||
{steps.map((step, index) => {
|
||||
const isCurrentStep = index === currentStep
|
||||
const isLastStep = index === steps.length - 1
|
||||
return (
|
||||
<div key={index} className='flex items-center gap-x-2'>
|
||||
<div
|
||||
className={cn('flex items-center gap-x-1', isCurrentStep ? 'text-state-accent-solid' : 'text-text-tertiary')}
|
||||
>
|
||||
{isCurrentStep && <div className='size-1 rounded-full bg-state-accent-solid' />}
|
||||
<span className='system-2xs-semibold-uppercase'>{step.label}</span>
|
||||
</div>
|
||||
{!isLastStep && (
|
||||
<div className='flex items-center'>
|
||||
<Divider type='horizontal' className='h-px w-3 bg-divider-deep' />
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
})}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default React.memo(StepIndicator)
|
||||
@ -0,0 +1,9 @@
|
||||
export enum TestRunStep {
|
||||
dataSource = 'dataSource',
|
||||
documentProcessing = 'documentProcessing',
|
||||
}
|
||||
|
||||
export type DataSourceOption = {
|
||||
label: string
|
||||
value: string
|
||||
}
|
||||
@ -5,6 +5,8 @@ export type RagPipelineSliceShape = {
|
||||
setShowInputFieldDialog: (showInputFieldPanel: boolean) => void
|
||||
nodesDefaultConfigs: Record<string, any>
|
||||
setNodesDefaultConfigs: (nodesDefaultConfigs: Record<string, any>) => void
|
||||
showTestRunPanel: boolean
|
||||
setShowTestRunPanel: (showTestRunPanel: boolean) => void
|
||||
}
|
||||
|
||||
export type CreateRagPipelineSliceSlice = StateCreator<RagPipelineSliceShape>
|
||||
@ -13,4 +15,6 @@ export const createRagPipelineSliceSlice: StateCreator<RagPipelineSliceShape> =
|
||||
setShowInputFieldDialog: showInputFieldDialog => set(() => ({ showInputFieldDialog })),
|
||||
nodesDefaultConfigs: {},
|
||||
setNodesDefaultConfigs: nodesDefaultConfigs => set(() => ({ nodesDefaultConfigs })),
|
||||
showTestRunPanel: false,
|
||||
setShowTestRunPanel: showTestRunPanel => set(() => ({ showTestRunPanel })),
|
||||
})
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
import { get, post } from './base'
|
||||
import type {
|
||||
DataSourceNotion,
|
||||
FileUploadConfigResponse,
|
||||
StructuredOutputRulesRequestBody,
|
||||
StructuredOutputRulesResponse,
|
||||
@ -9,11 +10,10 @@ import type { FileTypesRes } from './datasets'
|
||||
|
||||
const NAME_SPACE = 'common'
|
||||
|
||||
export const useFileUploadConfig = (enabled?: true) => {
|
||||
export const useFileUploadConfig = () => {
|
||||
return useQuery<FileUploadConfigResponse>({
|
||||
queryKey: [NAME_SPACE, 'file-upload-config'],
|
||||
queryFn: () => get<FileUploadConfigResponse>('/files/upload'),
|
||||
enabled,
|
||||
})
|
||||
}
|
||||
|
||||
@ -35,3 +35,14 @@ export const useFileSupportTypes = () => {
|
||||
queryFn: () => get<FileTypesRes>('/files/support-type'),
|
||||
})
|
||||
}
|
||||
|
||||
type DataSourcesResponse = {
|
||||
data: DataSourceNotion[]
|
||||
}
|
||||
|
||||
export const useDataSources = () => {
|
||||
return useQuery<DataSourcesResponse>({
|
||||
queryKey: [NAME_SPACE, 'data-sources'],
|
||||
queryFn: () => get<DataSourcesResponse>('/data-source/integrates'),
|
||||
})
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user