yujunjun/dify
1
0
Fork 0
mirror of https://github.com/langgenius/dify.git synced 2025-11-04 04:43:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: split text keep separator (#7930)

Browse Source
This commit is contained in:
Sumkor 2024-09-04 12:59:10 +08:00 committed by GitHub
parent 7b2cf8215f
commit 571415d1a4
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 3 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
api/core/rag/splitter/text_splitter.py
View File

@ -30,15 +30,14 @@ def _split_text_with_regex(
if keep_separator: if keep_separator:
# The parentheses in the pattern keep the delimiters in the result. # The parentheses in the pattern keep the delimiters in the result.
_splits = re.split(f"({re.escape(separator)})", text) _splits = re.split(f"({re.escape(separator)})", text)
splits = [_splits[i] + _splits[i + 1] for i in range(1, len(_splits), 2)] splits = [_splits[i - 1] + _splits[i] for i in range(1, len(_splits), 2)]
if len(_splits) % 2 == 0: if len(_splits) % 2 != 0:
splits += _splits[-1:] splits += _splits[-1:]
splits = [_splits[0]] + splits
else: else:
splits = re.split(separator, text) splits = re.split(separator, text)
else: else:
splits = list(text) splits = list(text)
return [s for s in splits if s != ""] return [s for s in splits if (s != "" and s != '\n')]
class TextSplitter(BaseDocumentTransformer, ABC): class TextSplitter(BaseDocumentTransformer, ABC):