diff --git a/api/core/rag/index_processor/processor/parent_child_index_processor.py b/api/core/rag/index_processor/processor/parent_child_index_processor.py index 6183f0900c..bb2efdafe0 100644 --- a/api/core/rag/index_processor/processor/parent_child_index_processor.py +++ b/api/core/rag/index_processor/processor/parent_child_index_processor.py @@ -1,5 +1,6 @@ """Paragraph index processor.""" +import json import uuid from collections.abc import Mapping from typing import Any, Optional @@ -16,7 +17,7 @@ from core.rag.index_processor.index_processor_base import BaseIndexProcessor from core.rag.models.document import ChildDocument, Document, ParentChildStructureChunk from extensions.ext_database import db from libs import helper -from models.dataset import ChildChunk, Dataset, DocumentSegment +from models.dataset import ChildChunk, Dataset, DatasetProcessRule, DocumentSegment from models.dataset import Document as DatasetDocument from services.entities.knowledge_entities.knowledge_entities import ParentMode, Rule @@ -228,13 +229,31 @@ class ParentChildIndexProcessor(BaseIndexProcessor): doc = Document(page_content=parent_child.parent_content, metadata=metadata, children=child_documents) documents.append(doc) if documents: + # update document parent mode + dataset_process_rule = DatasetProcessRule( + dataset_id=dataset.id, + mode="hierarchical", + rules=json.dumps({ + "parent_mode": parent_childs.parent_mode, + }), + created_by=document.created_by, + ) + db.session.add(dataset_process_rule) + db.session.flush() + document.dataset_process_rule_id = dataset_process_rule.id + db.session.commit() # save node to document segment doc_store = DatasetDocumentStore(dataset=dataset, user_id=document.created_by, document_id=document.id) # add document segments doc_store.add_documents(docs=documents, save_child=True) if dataset.indexing_technique == "high_quality": - vector = Vector(dataset) - vector.create(documents) + all_child_documents = [] + for doc in documents: + if doc.children: + all_child_documents.extend(doc.children) + if all_child_documents: + vector = Vector(dataset) + vector.create(all_child_documents) def format_preview(self, chunks: Mapping[str, Any]) -> Mapping[str, Any]: parent_childs = ParentChildStructureChunk(**chunks) diff --git a/api/core/rag/models/document.py b/api/core/rag/models/document.py index e382ff6b54..c37203f1c8 100644 --- a/api/core/rag/models/document.py +++ b/api/core/rag/models/document.py @@ -58,6 +58,7 @@ class ParentChildStructureChunk(BaseModel): """ parent_child_chunks: list[ParentChildChunk] + parent_mode: str = "paragraph" class QAChunk(BaseModel): diff --git a/web/app/components/base/prompt-editor/index.tsx b/web/app/components/base/prompt-editor/index.tsx index a87a51cd50..1ed0a0a004 100644 --- a/web/app/components/base/prompt-editor/index.tsx +++ b/web/app/components/base/prompt-editor/index.tsx @@ -1,6 +1,6 @@ 'use client' -import type { FC } from 'react' +import type { FC, ReactNode } from 'react' import { useEffect } from 'react' import type { EditorState, @@ -66,7 +66,7 @@ export type PromptEditorProps = { compact?: boolean wrapperClassName?: string className?: string - placeholder?: string | JSX.Element + placeholder?: string | ReactNode placeholderClassName?: string style?: React.CSSProperties value?: string diff --git a/web/app/components/base/prompt-editor/plugins/workflow-variable-block/component.tsx b/web/app/components/base/prompt-editor/plugins/workflow-variable-block/component.tsx index 37971e1e2a..e165b93a66 100644 --- a/web/app/components/base/prompt-editor/plugins/workflow-variable-block/component.tsx +++ b/web/app/components/base/prompt-editor/plugins/workflow-variable-block/component.tsx @@ -34,6 +34,7 @@ type WorkflowVariableBlockComponentProps = { workflowNodesMap: WorkflowNodesMap environmentVariables?: Var[] conversationVariables?: Var[] + ragVariables?: Var[] getVarType?: (payload: { nodeId: string, valueSelector: ValueSelector, @@ -47,6 +48,7 @@ const WorkflowVariableBlockComponent = ({ getVarType, environmentVariables, conversationVariables, + ragVariables, }: WorkflowVariableBlockComponentProps) => { const { t } = useTranslation() const [editor] = useLexicalComposerContext() @@ -66,7 +68,6 @@ const WorkflowVariableBlockComponent = ({ const isEnv = isENV(variables) const isChatVar = isConversationVar(variables) const isException = isExceptionVariable(varName, node?.type) - let variableValid = true if (isEnv) { if (environmentVariables) @@ -77,7 +78,8 @@ const WorkflowVariableBlockComponent = ({ variableValid = conversationVariables.some(v => v.variable === `${variables?.[0] ?? ''}.${variables?.[1] ?? ''}`) } else if (isRagVar) { - variableValid = variables[0] === 'rag' + if (ragVariables) + variableValid = ragVariables.some(v => v.variable === `${variables?.[0] ?? ''}.${variables?.[1] ?? ''}.${variables?.[2] ?? ''}`) } else { variableValid = !!node diff --git a/web/app/components/base/prompt-editor/plugins/workflow-variable-block/node.tsx b/web/app/components/base/prompt-editor/plugins/workflow-variable-block/node.tsx index f828bdbc14..bdecb21034 100644 --- a/web/app/components/base/prompt-editor/plugins/workflow-variable-block/node.tsx +++ b/web/app/components/base/prompt-editor/plugins/workflow-variable-block/node.tsx @@ -13,6 +13,7 @@ export type SerializedNode = SerializedLexicalNode & { getVarType?: GetVarType environmentVariables?: Var[] conversationVariables?: Var[] + ragVariables?: Var[] } export class WorkflowVariableBlockNode extends DecoratorNode { @@ -21,20 +22,21 @@ export class WorkflowVariableBlockNode extends DecoratorNode __getVarType?: GetVarType __environmentVariables?: Var[] __conversationVariables?: Var[] + __ragVariables?: Var[] static getType(): string { return 'workflow-variable-block' } static clone(node: WorkflowVariableBlockNode): WorkflowVariableBlockNode { - return new WorkflowVariableBlockNode(node.__variables, node.__workflowNodesMap, node.__getVarType, node.__key, node.__environmentVariables, node.__conversationVariables) + return new WorkflowVariableBlockNode(node.__variables, node.__workflowNodesMap, node.__getVarType, node.__key, node.__environmentVariables, node.__conversationVariables, node.__ragVariables) } isInline(): boolean { return true } - constructor(variables: string[], workflowNodesMap: WorkflowNodesMap, getVarType: any, key?: NodeKey, environmentVariables?: Var[], conversationVariables?: Var[]) { + constructor(variables: string[], workflowNodesMap: WorkflowNodesMap, getVarType: any, key?: NodeKey, environmentVariables?: Var[], conversationVariables?: Var[], ragVariables?: Var[]) { super(key) this.__variables = variables @@ -42,6 +44,7 @@ export class WorkflowVariableBlockNode extends DecoratorNode this.__getVarType = getVarType this.__environmentVariables = environmentVariables this.__conversationVariables = conversationVariables + this.__ragVariables = ragVariables } createDOM(): HTMLElement { @@ -63,12 +66,13 @@ export class WorkflowVariableBlockNode extends DecoratorNode getVarType={this.__getVarType!} environmentVariables={this.__environmentVariables} conversationVariables={this.__conversationVariables} + ragVariables={this.__ragVariables} /> ) } static importJSON(serializedNode: SerializedNode): WorkflowVariableBlockNode { - const node = $createWorkflowVariableBlockNode(serializedNode.variables, serializedNode.workflowNodesMap, serializedNode.getVarType, serializedNode.environmentVariables, serializedNode.conversationVariables) + const node = $createWorkflowVariableBlockNode(serializedNode.variables, serializedNode.workflowNodesMap, serializedNode.getVarType, serializedNode.environmentVariables, serializedNode.conversationVariables, serializedNode.ragVariables) return node } @@ -82,6 +86,7 @@ export class WorkflowVariableBlockNode extends DecoratorNode getVarType: this.getVarType(), environmentVariables: this.getEnvironmentVariables(), conversationVariables: this.getConversationVariables(), + ragVariables: this.getRagVariables(), } } @@ -110,12 +115,17 @@ export class WorkflowVariableBlockNode extends DecoratorNode return self.__conversationVariables } + getRagVariables(): any { + const self = this.getLatest() + return self.__ragVariables + } + getTextContent(): string { return `{{#${this.getVariables().join('.')}#}}` } } -export function $createWorkflowVariableBlockNode(variables: string[], workflowNodesMap: WorkflowNodesMap, getVarType?: GetVarType, environmentVariables?: Var[], conversationVariables?: Var[]): WorkflowVariableBlockNode { - return new WorkflowVariableBlockNode(variables, workflowNodesMap, getVarType, undefined, environmentVariables, conversationVariables) +export function $createWorkflowVariableBlockNode(variables: string[], workflowNodesMap: WorkflowNodesMap, getVarType?: GetVarType, environmentVariables?: Var[], conversationVariables?: Var[], ragVariables?: Var[]): WorkflowVariableBlockNode { + return new WorkflowVariableBlockNode(variables, workflowNodesMap, getVarType, undefined, environmentVariables, conversationVariables, ragVariables) } export function $isWorkflowVariableBlockNode( diff --git a/web/app/components/base/prompt-editor/plugins/workflow-variable-block/workflow-variable-block-replacement-block.tsx b/web/app/components/base/prompt-editor/plugins/workflow-variable-block/workflow-variable-block-replacement-block.tsx index 4d0c80f10f..6a73e5bb02 100644 --- a/web/app/components/base/prompt-editor/plugins/workflow-variable-block/workflow-variable-block-replacement-block.tsx +++ b/web/app/components/base/prompt-editor/plugins/workflow-variable-block/workflow-variable-block-replacement-block.tsx @@ -32,7 +32,7 @@ const WorkflowVariableBlockReplacementBlock = ({ onInsert() const nodePathString = textNode.getTextContent().slice(3, -3) - return $applyNodeReplacement($createWorkflowVariableBlockNode(nodePathString.split('.'), workflowNodesMap, getVarType, variables?.find(o => o.nodeId === 'env')?.vars || [], variables?.find(o => o.nodeId === 'conversation')?.vars || [])) + return $applyNodeReplacement($createWorkflowVariableBlockNode(nodePathString.split('.'), workflowNodesMap, getVarType, variables?.find(o => o.nodeId === 'env')?.vars || [], variables?.find(o => o.nodeId === 'conversation')?.vars || [], variables?.find(o => o.nodeId === 'rag')?.vars || [])) }, [onInsert, workflowNodesMap, getVarType, variables]) const getMatch = useCallback((text: string) => { diff --git a/web/app/components/workflow/nodes/_base/components/variable/var-reference-picker.tsx b/web/app/components/workflow/nodes/_base/components/variable/var-reference-picker.tsx index d57f09ea0d..5a6bd77af3 100644 --- a/web/app/components/workflow/nodes/_base/components/variable/var-reference-picker.tsx +++ b/web/app/components/workflow/nodes/_base/components/variable/var-reference-picker.tsx @@ -45,7 +45,6 @@ import Tooltip from '@/app/components/base/tooltip' import { isExceptionVariable } from '@/app/components/workflow/utils' import VarFullPathPanel from './var-full-path-panel' import { noop } from 'lodash-es' -import { useStore as useWorkflowStore } from '@/app/components/workflow/store' import { useFetchDynamicOptions } from '@/service/use-plugins' import type { Tool } from '@/app/components/tools/types' import { VariableIconWithColor } from '@/app/components/workflow/nodes/_base/components/variable/variable-label' @@ -129,7 +128,6 @@ const VarReferencePicker: FC = ({ }) const node = nodes.find(n => n.id === nodeId) - const ragPipelineVariables = useWorkflowStore(s => s.ragPipelineVariables) const isInIteration = !!(node?.data as any).isInIteration const iterationNode = isInIteration ? nodes.find(n => n.id === node?.parentId) : null diff --git a/web/app/components/workflow/nodes/_base/hooks/use-available-var-list.ts b/web/app/components/workflow/nodes/_base/hooks/use-available-var-list.ts index 38e104c90f..9f974e13f3 100644 --- a/web/app/components/workflow/nodes/_base/hooks/use-available-var-list.ts +++ b/web/app/components/workflow/nodes/_base/hooks/use-available-var-list.ts @@ -75,7 +75,10 @@ const useAvailableVarList = (nodeId: string, { return { availableVars, availableNodes, - availableNodesWithParent: availableNodes, + availableNodesWithParent: [ + ...availableNodes, + ...(isDataSourceNode ? [currNode] : []), + ], } } diff --git a/web/app/components/workflow/nodes/data-source/panel.tsx b/web/app/components/workflow/nodes/data-source/panel.tsx index 8d9d0d17ec..29390b10a9 100644 --- a/web/app/components/workflow/nodes/data-source/panel.tsx +++ b/web/app/components/workflow/nodes/data-source/panel.tsx @@ -1,8 +1,6 @@ import type { FC } from 'react' import { - useCallback, useMemo, - useState, } from 'react' import { useTranslation } from 'react-i18next' import { memo } from 'react' @@ -24,10 +22,8 @@ import { WEBSITE_CRAWL_OUTPUT, } from './constants' import { useStore } from '@/app/components/workflow/store' -import InputVarList from '@/app/components/workflow/nodes/tool/components/input-var-list' import { toolParametersToFormSchemas } from '@/app/components/tools/utils/to-form-schema' -import type { Var } from '@/app/components/workflow/types' -import { VarType } from '@/app/components/workflow/types' +import ToolForm from '../tool/components/tool-form' const Panel: FC> = ({ id, data }) => { const { t } = useTranslation() @@ -52,18 +48,9 @@ const Panel: FC> = ({ id, data }) => { const formSchemas = useMemo(() => { return currentDataSourceItem ? toolParametersToFormSchemas(currentDataSourceItem.parameters) : [] }, [currentDataSourceItem]) - const [currVarIndex, setCurrVarIndex] = useState(-1) - const currVarType = formSchemas[currVarIndex]?._type - const handleOnVarOpen = useCallback((index: number) => { - setCurrVarIndex(index) - }, []) - const filterVar = useCallback((varPayload: Var) => { - if (currVarType) - return varPayload.type === currVarType - - return varPayload.type !== VarType.arrayFile - }, [currVarType]) + const pipelineId = useStore(s => s.pipelineId) + const setShowInputFieldPanel = useStore(s => s.setShowInputFieldPanel) return (
@@ -80,16 +67,19 @@ const Panel: FC> = ({ id, data }) => { supportCollapse: true, }} > - + {formSchemas.length > 0 && ( + setShowInputFieldPanel?.(true)} + /> + )} ) }