feat: implement document settings and pipeline settings components with localization support

web/app/components/datasets/documents/create-from-pipeline/index.tsx

@@ -32,10 +32,7 @@ const CreateFormPipeline = () => {
   const { t } = useTranslation()
   const plan = useProviderContextSelector(state => state.plan)
   const enableBilling = useProviderContextSelector(state => state.enableBilling)
-  const datasetId = useDatasetDetailContextWithSelector(s => s.dataset?.id)
   const pipelineId = useDatasetDetailContextWithSelector(s => s.dataset?.pipeline_id)
-  const indexingType = useDatasetDetailContextWithSelector(s => s.dataset?.indexing_technique)
-  const retrievalMethod = useDatasetDetailContextWithSelector(s => s.dataset?.retrieval_model_dict.search_method)
   const [datasource, setDatasource] = useState<Datasource>()
   const [estimateData, setEstimateData] = useState<FileIndexingEstimateResponse | undefined>(undefined)
   const [batchId, setBatchId] = useState('')
@@ -302,11 +299,8 @@ const CreateFormPipeline = () => {
           {
             currentStep === 3 && (
               <Processing
-                datasetId={datasetId!}
                 batchId={batchId}
                 documents={documents}
-                indexingType={indexingType!}
-                retrievalMethod={retrievalMethod!}
               />
             )
           }
@@ -326,7 +320,7 @@ const CreateFormPipeline = () => {
         currentStep === 2 && (
           <div className='flex h-full w-[752px] shrink-0 pl-2 pt-2'>
             <ChunkPreview
-              datasource={datasource!}
+              dataSourceType={datasource!.type}
               files={fileList.map(file => file.file)}
               onlineDocuments={onlineDocuments}
               websitePages={websitePages}

web/app/components/datasets/documents/create-from-pipeline/preview/chunk-preview.tsx

@@ -2,7 +2,6 @@ import React, { useState } from 'react'
 import { useTranslation } from 'react-i18next'
 import { PreviewContainer } from '../../../preview/container'
 import { PreviewHeader } from '../../../preview/header'
-import type { Datasource } from '@/app/components/rag-pipeline/components/panel/test-run/types'
 import type { CrawlResultItem, CustomFile, DocumentItem, FileIndexingEstimateResponse } from '@/models/datasets'
 import { ChunkingMode } from '@/models/datasets'
 import type { NotionPage } from '@/models/common'
@@ -18,7 +17,7 @@ import Button from '@/app/components/base/button'
 import { DatasourceType } from '@/models/pipeline'
 
 type ChunkPreviewProps = {
-  datasource: Datasource
+  dataSourceType: DatasourceType
   files: CustomFile[]
   onlineDocuments: NotionPage[]
   websitePages: CrawlResultItem[]
@@ -32,7 +31,7 @@ type ChunkPreviewProps = {
 }
 
 const ChunkPreview = ({
-  datasource,
+  dataSourceType,
   files,
   onlineDocuments,
   websitePages,
@@ -51,8 +50,6 @@ const ChunkPreview = ({
   const [previewOnlineDocument, setPreviewOnlineDocument] = useState<NotionPage>(onlineDocuments[0])
   const [previewWebsitePage, setPreviewWebsitePage] = useState<CrawlResultItem>(websitePages[0])
 
-  const dataSourceType = datasource?.type
-
   return (
     <PreviewContainer
       header={<PreviewHeader
@@ -185,9 +182,7 @@ const ChunkPreview = ({
             <p className='text-sm text-text-tertiary'>
               {t('datasetCreation.stepTwo.previewChunkTip')}
             </p>
-            <Button
-              onClick={onPreview}
-            >
+            <Button onClick={onPreview}>
               {t('datasetPipeline.addDocuments.stepTwo.previewChunks')}
             </Button>
           </div>

web/app/components/datasets/documents/create-from-pipeline/processing/index.tsx

@@ -4,34 +4,30 @@ import { useTranslation } from 'react-i18next'
 import { RiBookOpenLine } from '@remixicon/react'
 import { useGetDocLanguage } from '@/context/i18n'
 import EmbeddingProcess from './embedding-process'
-import type { IndexingType } from '../../../create/step-two'
-import type { RETRIEVE_METHOD } from '@/types/app'
 import type { InitialDocumentDetail } from '@/models/pipeline'
+import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
 
 type ProcessingProps = {
-  datasetId: string
-  indexingType: IndexingType
-  retrievalMethod: RETRIEVE_METHOD
   batchId: string
   documents: InitialDocumentDetail[]
 }
 
 const Processing = ({
-  datasetId,
   batchId,
   documents,
-  indexingType,
-  retrievalMethod,
 }: ProcessingProps) => {
   const { t } = useTranslation()
   const docLanguage = useGetDocLanguage()
+  const datasetId = useDatasetDetailContextWithSelector(s => s.dataset?.id)
+  const indexingType = useDatasetDetailContextWithSelector(s => s.dataset?.indexing_technique)
+  const retrievalMethod = useDatasetDetailContextWithSelector(s => s.dataset?.retrieval_model_dict.search_method)
 
   return (
     <div className='flex h-full w-full justify-center overflow-hidden'>
       <div className='h-full w-3/5 overflow-y-auto pb-8 pt-10'>
         <div className='max-w-[640px]'>
           <EmbeddingProcess
-            datasetId={datasetId}
+            datasetId={datasetId!}
             batchId={batchId}
             documents={documents}
             indexingType={indexingType}

web/app/components/datasets/documents/detail/settings/document-settings.tsx

@@ -0,0 +1,94 @@
+import React, { useMemo } from 'react'
+import { useTranslation } from 'react-i18next'
+import { useBoolean } from 'ahooks'
+import { useContext } from 'use-context-selector'
+import { useRouter } from 'next/navigation'
+import DatasetDetailContext from '@/context/dataset-detail'
+import type { CrawlOptions, CustomFile, DataSourceType } from '@/models/datasets'
+import Loading from '@/app/components/base/loading'
+import StepTwo from '@/app/components/datasets/create/step-two'
+import AccountSetting from '@/app/components/header/account-setting'
+import AppUnavailable from '@/app/components/base/app-unavailable'
+import { useDefaultModel } from '@/app/components/header/account-setting/model-provider-page/hooks'
+import { ModelTypeEnum } from '@/app/components/header/account-setting/model-provider-page/declarations'
+import type { NotionPage } from '@/models/common'
+import { useDocumentDetail, useInvalidDocumentDetailKey } from '@/service/knowledge/use-document'
+
+type DocumentSettingsProps = {
+  datasetId: string
+  documentId: string
+}
+
+const DocumentSettings = ({ datasetId, documentId }: DocumentSettingsProps) => {
+  const { t } = useTranslation()
+  const router = useRouter()
+  const [isShowSetAPIKey, { setTrue: showSetAPIKey, setFalse: hideSetAPIkey }] = useBoolean()
+  const { indexingTechnique, dataset } = useContext(DatasetDetailContext)
+  const { data: embeddingsDefaultModel } = useDefaultModel(ModelTypeEnum.textEmbedding)
+
+  const invalidDocumentDetail = useInvalidDocumentDetailKey()
+  const saveHandler = () => {
+    invalidDocumentDetail()
+    router.push(`/datasets/${datasetId}/documents/${documentId}`)
+  }
+
+  const cancelHandler = () => router.back()
+
+  const { data: documentDetail, error } = useDocumentDetail({
+    datasetId,
+    documentId,
+    params: { metadata: 'without' },
+  })
+
+  const currentPage = useMemo(() => {
+    return {
+      workspace_id: documentDetail?.data_source_info.notion_workspace_id,
+      page_id: documentDetail?.data_source_info.notion_page_id,
+      page_name: documentDetail?.name,
+      page_icon: documentDetail?.data_source_info.notion_page_icon,
+      type: documentDetail?.data_source_type,
+    }
+  }, [documentDetail])
+
+  if (error)
+    return <AppUnavailable code={500} unknownReason={t('datasetCreation.error.unavailable') as string} />
+
+  return (
+    <div className='flex' style={{ height: 'calc(100vh - 56px)' }}>
+      <div className='grow'>
+        {!documentDetail && <Loading type='app' />}
+        {dataset && documentDetail && (
+          <StepTwo
+            isAPIKeySet={!!embeddingsDefaultModel}
+            onSetting={showSetAPIKey}
+            datasetId={datasetId}
+            dataSourceType={documentDetail.data_source_type as DataSourceType}
+            notionPages={[currentPage as unknown as NotionPage]}
+            websitePages={[
+              {
+                title: documentDetail.name,
+                source_url: documentDetail.data_source_info?.url,
+                markdown: '',
+                description: '',
+              },
+            ]}
+            websiteCrawlProvider={documentDetail.data_source_info?.provider}
+            websiteCrawlJobId={documentDetail.data_source_info?.job_id}
+            crawlOptions={documentDetail.data_source_info as unknown as CrawlOptions}
+            indexingType={indexingTechnique}
+            isSetting
+            documentDetail={documentDetail}
+            files={[documentDetail.data_source_info.upload_file as CustomFile]}
+            onSave={saveHandler}
+            onCancel={cancelHandler}
+          />
+        )}
+      </div>
+      {isShowSetAPIKey && <AccountSetting activeTab='provider' onCancel={async () => {
+        hideSetAPIkey()
+      }} />}
+    </div>
+  )
+}
+
+export default DocumentSettings

web/app/components/datasets/documents/detail/settings/index.tsx

@@ -1,96 +1,35 @@
 'use client'
-import React, { useMemo } from 'react'
-import { useTranslation } from 'react-i18next'
-import { useBoolean } from 'ahooks'
-import { useContext } from 'use-context-selector'
-import { useRouter } from 'next/navigation'
-import DatasetDetailContext from '@/context/dataset-detail'
-import type { CrawlOptions, CustomFile } from '@/models/datasets'
+import React from 'react'
+import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
+import DocumentSettings from './document-settings'
+import PipelineSettings from './pipeline-settings'
 
-import Loading from '@/app/components/base/loading'
-import StepTwo from '@/app/components/datasets/create/step-two'
-import AccountSetting from '@/app/components/header/account-setting'
-import AppUnavailable from '@/app/components/base/app-unavailable'
-import { useDefaultModel } from '@/app/components/header/account-setting/model-provider-page/hooks'
-import { ModelTypeEnum } from '@/app/components/header/account-setting/model-provider-page/declarations'
-import type { NotionPage } from '@/models/common'
-import { useDocumentDetail, useInvalidDocumentDetailKey } from '@/service/knowledge/use-document'
-
-type DocumentSettingsProps = {
+type SettingsProps = {
   datasetId: string
   documentId: string
 }
 
-const DocumentSettings = ({ datasetId, documentId }: DocumentSettingsProps) => {
-  const { t } = useTranslation()
-  const router = useRouter()
-  const [isShowSetAPIKey, { setTrue: showSetAPIKey, setFalse: hideSetAPIkey }] = useBoolean()
-  const { indexingTechnique, dataset } = useContext(DatasetDetailContext)
-  const { data: embeddingsDefaultModel } = useDefaultModel(ModelTypeEnum.textEmbedding)
+const Settings = ({
+  datasetId,
+  documentId,
+}: SettingsProps) => {
+  const pipelineId = useDatasetDetailContextWithSelector(s => s.dataset?.pipeline_id)
 
-  const invalidDocumentDetail = useInvalidDocumentDetailKey()
-  const saveHandler = () => {
-    invalidDocumentDetail()
-    router.push(`/datasets/${datasetId}/documents/${documentId}`)
+  if (!pipelineId) {
+    return (
+      <DocumentSettings
+        datasetId={datasetId}
+        documentId={documentId}
+      />
+    )
   }
 
-  const cancelHandler = () => router.back()
-
-  const { data: documentDetail, error } = useDocumentDetail({
-    datasetId,
-    documentId,
-    params: { metadata: 'without' },
-  })
-
-  const currentPage = useMemo(() => {
-    return {
-      workspace_id: documentDetail?.data_source_info.notion_workspace_id,
-      page_id: documentDetail?.data_source_info.notion_page_id,
-      page_name: documentDetail?.name,
-      page_icon: documentDetail?.data_source_info.notion_page_icon,
-      type: documentDetail?.data_source_type,
-    }
-  }, [documentDetail])
-
-  if (error)
-    return <AppUnavailable code={500} unknownReason={t('datasetCreation.error.unavailable') as string} />
-
   return (
-    <div className='flex' style={{ height: 'calc(100vh - 56px)' }}>
-      <div className="grow">
-        {!documentDetail && <Loading type='app' />}
-        {dataset && documentDetail && (
-          <StepTwo
-            isAPIKeySet={!!embeddingsDefaultModel}
-            onSetting={showSetAPIKey}
-            datasetId={datasetId}
-            dataSourceType={documentDetail.data_source_type}
-            notionPages={[currentPage as unknown as NotionPage]}
-            websitePages={[
-              {
-                title: documentDetail.name,
-                source_url: documentDetail.data_source_info?.url,
-                markdown: '',
-                description: '',
-              },
-            ]}
-            websiteCrawlProvider={documentDetail.data_source_info?.provider}
-            websiteCrawlJobId={documentDetail.data_source_info?.job_id}
-            crawlOptions={documentDetail.data_source_info as unknown as CrawlOptions}
-            indexingType={indexingTechnique}
-            isSetting
-            documentDetail={documentDetail}
-            files={[documentDetail.data_source_info.upload_file as CustomFile]}
-            onSave={saveHandler}
-            onCancel={cancelHandler}
-          />
-        )}
-      </div>
-      {isShowSetAPIKey && <AccountSetting activeTab="provider" onCancel={async () => {
-        hideSetAPIkey()
-      }} />}
-    </div>
+    <PipelineSettings
+      datasetId={datasetId}
+      documentId={documentId}
+    />
   )
 }
 
-export default DocumentSettings
+export default Settings

web/app/components/datasets/documents/detail/settings/pipeline-settings/index.tsx

@@ -0,0 +1,120 @@
+import { useCallback, useRef, useState } from 'react'
+import type { CrawlResultItem, DocumentItem, FileIndexingEstimateResponse } from '@/models/datasets'
+import type { NotionPage } from '@/models/common'
+import { useTranslation } from 'react-i18next'
+import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
+import { useDocumentDetail } from '@/service/knowledge/use-document'
+import AppUnavailable from '@/app/components/base/app-unavailable'
+import ChunkPreview from '../../../create-from-pipeline/preview/chunk-preview'
+import Loading from '@/app/components/base/loading'
+import type { DatasourceType } from '@/models/pipeline'
+import ProcessDocuments from './process-documents'
+import LeftHeader from './left-header'
+
+type PipelineSettingsProps = {
+  datasetId: string
+  documentId: string
+}
+
+const PipelineSettings = ({
+  datasetId,
+  documentId,
+}: PipelineSettingsProps) => {
+  const { t } = useTranslation()
+  const pipelineId = useDatasetDetailContextWithSelector(s => s.dataset?.pipeline_id)
+  const [estimateData, setEstimateData] = useState<FileIndexingEstimateResponse | undefined>(undefined)
+
+  const isPreview = useRef(false)
+  const formRef = useRef<any>(null)
+
+  const { data: documentDetail, error, isFetching: isFetchingDocumentDetail } = useDocumentDetail({
+    datasetId,
+    documentId,
+    params: { metadata: 'without' },
+  })
+
+  const handlePreviewChunks = useCallback(async (data: Record<string, any>) => {
+    // todo: Preview
+  }, [])
+
+  const handleProcess = useCallback(async (data: Record<string, any>) => {
+    // todo: Process
+  }, [])
+
+  const onClickProcess = useCallback(() => {
+    isPreview.current = false
+    formRef.current?.submit()
+  }, [])
+
+  const onClickPreview = useCallback(() => {
+    isPreview.current = true
+    formRef.current?.submit()
+  }, [])
+
+  const handleSubmit = useCallback((data: Record<string, any>) => {
+    isPreview.current ? handlePreviewChunks(data) : handleProcess(data)
+  }, [handlePreviewChunks, handleProcess])
+
+  const handlePreviewFileChange = useCallback((file: DocumentItem) => {
+    onClickPreview()
+  }, [onClickPreview])
+
+  const handlePreviewOnlineDocumentChange = useCallback((page: NotionPage) => {
+    onClickPreview()
+  }, [onClickPreview])
+
+  const handlePreviewWebsiteChange = useCallback((website: CrawlResultItem) => {
+    onClickPreview()
+  }, [onClickPreview])
+
+  if (isFetchingDocumentDetail) {
+    return (
+      <Loading type='app' />
+    )
+  }
+
+  if (error)
+    return <AppUnavailable code={500} unknownReason={t('datasetCreation.error.unavailable') as string} />
+
+  return (
+    <div
+      className='relative flex h-[calc(100vh-56px)] overflow-x-auto rounded-t-2xl border-t border-effects-highlight bg-background-default-subtle'
+    >
+      <div className='flex h-full flex-1 flex-col px-14'>
+        <LeftHeader title={t('datasetPipeline.documentSettings.title')} />
+        <div className='grow overflow-y-auto'>
+          <ProcessDocuments
+            ref={formRef}
+            documentId={documentId}
+            onProcess={onClickProcess}
+            onPreview={onClickPreview}
+            onSubmit={handleSubmit}
+          />
+        </div>
+      </div>
+      {/* Preview */}
+      <div className='flex h-full flex-1 shrink-0 pl-2 pt-2'>
+        <ChunkPreview
+          dataSourceType={documentDetail!.data_source_type as DatasourceType}
+          // @ts-expect-error mock data // todo: remove mock data
+          files={[{
+            id: '12345678',
+            name: 'test-file',
+            extension: 'txt',
+          }]}
+          onlineDocuments={[]}
+          websitePages={[]}
+          isIdle={true}
+          isPending={true}
+          estimateData={estimateData}
+          onPreview={onClickPreview}
+          handlePreviewFileChange={handlePreviewFileChange}
+          handlePreviewOnlineDocumentChange={handlePreviewOnlineDocumentChange}
+          handlePreviewWebsitePageChange={handlePreviewWebsiteChange}
+        />
+      </div>
+    </div>
+  )
+}
+
+export default PipelineSettings

web/app/components/datasets/documents/detail/settings/pipeline-settings/left-header.tsx

@@ -0,0 +1,42 @@
+import React, { useCallback } from 'react'
+import { RiArrowLeftLine } from '@remixicon/react'
+import Button from '@/app/components/base/button'
+import { useRouter } from 'next/navigation'
+import Effect from '@/app/components/base/effect'
+import { useTranslation } from 'react-i18next'
+
+type LeftHeaderProps = {
+  title: string
+}
+
+const LeftHeader = ({
+  title,
+}: LeftHeaderProps) => {
+  const { t } = useTranslation()
+  const { back } = useRouter()
+
+  const navigateBack = useCallback(() => {
+    back()
+  }, [back])
+
+  return (
+    <div className='relative flex flex-col gap-y-0.5 pb-2 pt-4'>
+      <div className='system-2xs-semibold-uppercase bg-pipeline-add-documents-title-bg bg-clip-text text-transparent'>
+        {title}
+      </div>
+      <div className='system-md-semibold text-text-primary'>
+        {t('datasetPipeline.addDocuments.steps.processDocuments')}
+      </div>
+      <Button
+        variant='secondary-accent'
+        className='absolute -left-11 top-3.5 size-9 rounded-full p-0'
+        onClick={navigateBack}
+      >
+        <RiArrowLeftLine className='size-5 ' />
+      </Button>
+      <Effect className='left-8 top-[-34px] opacity-20' />
+    </div>
+  )
+}
+
+export default React.memo(LeftHeader)

web/app/components/datasets/documents/detail/settings/pipeline-settings/process-documents/actions.tsx

@@ -0,0 +1,26 @@
+import React from 'react'
+import Button from '@/app/components/base/button'
+import { useTranslation } from 'react-i18next'
+
+type ActionsProps = {
+  onProcess: () => void
+}
+
+const Actions = ({
+  onProcess,
+}: ActionsProps) => {
+  const { t } = useTranslation()
+
+  return (
+    <div className='flex items-center justify-end'>
+      <Button
+        variant='primary'
+        onClick={onProcess}
+      >
+        {t('datasetPipeline.operations.saveAndProcess')}
+      </Button>
+    </div>
+  )
+}
+
+export default React.memo(Actions)

web/app/components/datasets/documents/detail/settings/pipeline-settings/process-documents/hooks.ts

@@ -0,0 +1,11 @@
+import type { BaseConfiguration } from '@/app/components/base/form/form-scenarios/base/types'
+
+export const useConfigurations = (documentdId: string) => {
+  const initialData: Record<string, any> = {}
+  const configurations: BaseConfiguration[] = []
+
+  return {
+    initialData,
+    configurations,
+  }
+}

web/app/components/datasets/documents/detail/settings/pipeline-settings/process-documents/index.tsx

@@ -0,0 +1,39 @@
+import { generateZodSchema } from '@/app/components/base/form/form-scenarios/base/utils'
+import { useConfigurations } from './hooks'
+import Actions from './actions'
+import Form from '../../../../create-from-pipeline/process-documents/form'
+
+type ProcessDocumentsProps = {
+  documentId: string
+  ref: React.RefObject<any>
+  onProcess: () => void
+  onPreview: () => void
+  onSubmit: (data: Record<string, any>) => void
+}
+
+const ProcessDocuments = ({
+  documentId,
+  onProcess,
+  onPreview,
+  onSubmit,
+  ref,
+}: ProcessDocumentsProps) => {
+  const { initialData, configurations } = useConfigurations(documentId)
+  const schema = generateZodSchema(configurations)
+
+  return (
+    <div className='flex flex-col gap-y-4 pt-4'>
+      <Form
+        ref={ref}
+        initialData={initialData}
+        configurations={configurations}
+        schema={schema}
+        onSubmit={onSubmit}
+        onPreview={onPreview}
+      />
+      <Actions onProcess={onProcess} />
+    </div>
+  )
+}
+
+export default ProcessDocuments

web/i18n/en-US/dataset-pipeline.ts

@@ -103,6 +103,9 @@ const translation = {
     },
     characters: 'characters',
   },
+  documentSettings: {
+    title: 'Document Settings',
+  },
 }
 
 export default translation

web/i18n/zh-Hans/dataset-pipeline.ts

@@ -103,6 +103,9 @@ const translation = {
     },
     characters: '字符',
   },
+  documentSettings: {
+    title: '文档设置',
+  },
 }
 
 export default translation

web/models/datasets.ts

@@ -6,6 +6,7 @@ import type { MetadataFilteringVariableType } from '@/app/components/workflow/no
 import type { MetadataItemWithValue } from '@/app/components/datasets/metadata/types'
 import { ExternalKnowledgeBase, General, ParentChild, Qa } from '@/app/components/base/icons/src/public/knowledge/dataset-card'
 import { GeneralChunk, ParentChildChunk, QuestionAndAnswer } from '@/app/components/base/icons/src/vender/knowledge'
+import type { DatasourceType } from './pipeline'
 
 export enum DataSourceType {
   FILE = 'upload_file',
@@ -318,7 +319,7 @@ export type InitialDocumentDetail = {
   batch: string
   position: number
   dataset_id: string
-  data_source_type: DataSourceType
+  data_source_type: DataSourceType | DatasourceType
   data_source_info: DataSourceInfo
   dataset_process_rule_id: string
   name: string