yujunjun/unstructured
1
0
Fork 0
mirror of https://github.com/Unstructured-IO/unstructured.git synced 2025-06-27 02:30:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(chunking): add overlap on chunk-splits (#2305)

Browse Source 
There are two distinct overlap operations with completely different
implementations. This is "intra-chunk" overlap, applying overlap to
chunks resulting from text-splitting an oversized element.

So if an oversized element had text "abcd efgh ijkl mnop qrst" and was
split at 15 chars with overlap of 5, it would produce "abcd efgh ijkl"
and "ijkl mnop qrst". Any inter-chunk overlap from the prior chunk and
applied at the beginning of the string (before "abcd") is handled in a
separate operation in the next PR.
This commit is contained in:
Steve Canny 2023-12-22 12:35:18 -08:00 committed by GitHub
parent 5c0043aa7d
commit eb1b022ff8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 90 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
CHANGELOG.md
View File

@ -1,7 +1,9 @@
## 0.11.7-dev0
## 0.11.7-dev1
### Enhancements
* **Add intra-chunk overlap capability.** Implement overlap for split-chunks where text-splitting is used to divide an oversized chunk into two or more chunks that fit in the chunking window. Note this capability is not yet available from the API but will shortly be made accessible using a new `overlap` kwarg on partition functions.
### Features
### Fixes

1
setup.cfg
View File

@ -2,6 +2,7 @@
license_files = LICENSE.md
[flake8]
ignore = E203,W503
max-line-length = 100
exclude =
.venv

62
test_unstructured/chunking/test_base.py
View File

@ -149,23 +149,33 @@ class Describe_TextSplitter:
"""Unit-test suite for `unstructured.chunking.base._TextSplitter` objects."""
def it_splits_on_a_preferred_separator_when_it_can(self):
opts = ChunkingOptions.new(max_characters=50, text_splitting_separators=("\n", " "))
opts = ChunkingOptions.new(
max_characters=50, text_splitting_separators=("\n", " "), overlap=10
)
split = _TextSplitter(opts)
text = (
"Lorem ipsum dolor amet consectetur adipiscing.\n"
"In rhoncus ipsum sed lectus porta volutpat."
"Lorem ipsum dolor amet consectetur adipiscing. \n "
"In rhoncus ipsum sed lectus porta."
)
s, remainder = split(text)
# -- trailing whitespace is stripped from split --
assert s == "Lorem ipsum dolor amet consectetur adipiscing."
assert remainder == "In rhoncus ipsum sed lectus porta volutpat."
# -- leading whitespace is stripped from remainder
# -- overlap is separated by single space
# -- overlap-prefix is computed on arbitrary character boundary
# -- overlap-prefix len includes space separator (text portion is one less than specified)
assert remainder == "ipiscing. In rhoncus ipsum sed lectus porta."
# --
s, remainder = split(remainder)
assert s == "In rhoncus ipsum sed lectus porta volutpat."
assert s == "ipiscing. In rhoncus ipsum sed lectus porta."
assert remainder == ""
def and_it_splits_on_the_next_available_separator_when_the_first_is_not_available(self):
opts = ChunkingOptions.new(max_characters=40, text_splitting_separators=("\n", " "))
opts = ChunkingOptions.new(
max_characters=40, text_splitting_separators=("\n", " "), overlap=10
)
split = _TextSplitter(opts)
text = (
"Lorem ipsum dolor amet consectetur adipiscing. In rhoncus ipsum sed lectus porta"
@ -174,28 +184,34 @@ class Describe_TextSplitter:
s, remainder = split(text)
assert s == "Lorem ipsum dolor amet consectetur"
assert remainder == "adipiscing. In rhoncus ipsum sed lectus porta volutpat."
assert remainder == "nsectetur adipiscing. In rhoncus ipsum sed lectus porta volutpat."
# --
s, remainder = split(remainder)
assert s == "adipiscing. In rhoncus ipsum sed lectus"
assert remainder == "porta volutpat."
assert s == "nsectetur adipiscing. In rhoncus ipsum"
assert remainder == "cus ipsum sed lectus porta volutpat."
# --
s, remainder = split(remainder)
assert s == "porta volutpat."
assert s == "cus ipsum sed lectus porta volutpat."
assert remainder == ""
def and_it_splits_on_an_arbitrary_character_as_a_last_resort(self):
opts = ChunkingOptions.new(max_characters=40, text_splitting_separators=("\n", " "))
opts = ChunkingOptions.new(
max_characters=30, text_splitting_separators=("\n", " "), overlap=10
)
split = _TextSplitter(opts)
text = "Loremipsumdolorametconsecteturadipiscingelit. In rhoncus ipsum sed lectus porta."
s, remainder = split(text)
assert s == "Loremipsumdolorametconsecteturadipiscing"
assert remainder == "elit. In rhoncus ipsum sed lectus porta."
assert s == "Loremipsumdolorametconsectetur"
assert remainder == "onsecteturadipiscingelit. In rhoncus ipsum sed lectus porta."
# --
s, remainder = split(remainder)
assert s == "elit. In rhoncus ipsum sed lectus porta."
assert remainder == ""
assert s == "onsecteturadipiscingelit. In"
assert remainder == "gelit. In rhoncus ipsum sed lectus porta."
# --
s, remainder = split(remainder)
assert s == "gelit. In rhoncus ipsum sed"
assert remainder == "ipsum sed lectus porta."
@pytest.mark.parametrize(
"text",
@ -205,7 +221,7 @@ class Describe_TextSplitter:
],
)
def it_does_not_split_a_string_that_is_not_longer_than_maxlen(self, text: str):
opts = ChunkingOptions.new(max_characters=46)
opts = ChunkingOptions.new(max_characters=46, overlap=10)
split = _TextSplitter(opts)
s, remainder = split(text)
@ -214,7 +230,7 @@ class Describe_TextSplitter:
assert remainder == ""
def it_fills_the_window_when_falling_back_to_an_arbitrary_character_split(self):
opts = ChunkingOptions.new(max_characters=38)
opts = ChunkingOptions.new(max_characters=38, overlap=10)
split = _TextSplitter(opts)
text = "Loremipsumdolorametconsecteturadipiscingelit. In rhoncus ipsum sed lectus porta."
@ -223,17 +239,19 @@ class Describe_TextSplitter:
assert s == "Loremipsumdolorametconsecteturadipisci"
assert len(s) == 38
@pytest.mark.parametrize("separators", [("\n", " "), ()])
@pytest.mark.parametrize("separators", [("\n", " "), (" ",)])
def it_strips_whitespace_around_the_split(self, separators: Sequence[str]):
opts = ChunkingOptions.new(max_characters=50, text_splitting_separators=separators)
opts = ChunkingOptions.new(
max_characters=50, text_splitting_separators=separators, overlap=10
)
split = _TextSplitter(opts)
text = "Lorem ipsum dolor amet consectetur adipiscing. In rhoncus ipsum sed lectus."
# |------------------------------------------------^ 50-chars
text = "Lorem ipsum dolor amet consectetur adipiscing. \n\n In rhoncus ipsum sed lectus."
# |-------------------------------------------------^ 50-chars
s, remainder = split(text)
assert s == "Lorem ipsum dolor amet consectetur adipiscing."
assert remainder == "In rhoncus ipsum sed lectus."
assert remainder == "ipiscing. In rhoncus ipsum sed lectus."
# ================================================================================================

2
unstructured/__version__.py
View File

@ -1 +1 @@
__version__ = "0.11.7-dev0" # pragma: no cover
__version__ = "0.11.7-dev1" # pragma: no cover

58
unstructured/chunking/base.py
View File

@ -286,17 +286,23 @@ class _TextSplitter:
if len(s) <= maxlen:
return s, ""
for p, length in self._patterns:
for p, sep_len in self._patterns:
# -- length of separator must be added to include that separator when it happens to be
# -- located exactly at maxlen. Otherwise the search-from-end regex won't find it.
fragment, remainder = self._split_from_maxlen(p, maxlen + length, s)
if not fragment:
fragment, remainder = self._split_from_maxlen(p, sep_len, s)
if (
# -- no available split with this separator --
not fragment
# -- split did not progress, consuming part of the string --
or len(remainder) >= len(s)
):
continue
return fragment.rstrip(), remainder.lstrip()
# -- the terminal "" pattern is not actually executed via regex since its implementation is
# -- trivial and provides a hard back-stop here in this method.
return s[:maxlen].rstrip(), s[maxlen:].lstrip()
# -- trivial and provides a hard back-stop here in this method. No separator is used between
# -- tail and remainder on arb-char split.
return s[:maxlen].rstrip(), s[maxlen - self._opts.overlap :].lstrip()
@lazyproperty
def _patterns(self) -> Tuple[Tuple[regex.Pattern[str], int], ...]:
@ -312,21 +318,47 @@ class _TextSplitter:
separators = self._opts.text_splitting_separators
return tuple((regex.compile(f"(?r){sep}"), len(sep)) for sep in separators)
@staticmethod
def _split_from_maxlen(pattern: regex.Pattern[str], maxlen: int, s: str) -> Tuple[str, str]:
def _split_from_maxlen(
self, pattern: regex.Pattern[str], sep_len: int, s: str
) -> Tuple[str, str]:
"""Return (split, remainder) pair split from `s` on the right-most match before `maxlen`.
Returns `"", s` if no suitable match was found. The first string in the pair will never be
longer than `maxlen` and there is no longer split available using `pattern`.
Returns `"", s` if no suitable match was found. Also returns `"", s` if splitting on this
separator produces a split shorter than the required overlap (which would produce an
infinite loop).
`split` will never be longer than `maxlen` and there is no longer split available using
`pattern`.
The separator is removed and does not appear in either the split or remainder.
"""
match = pattern.search(s[:maxlen])
maxlen, overlap = self._opts.hard_max, self._opts.overlap
# -- A split not longer than overlap will not progress (infinite loop). On the right side,
# -- need to extend search range to include a separator located exactly at maxlen.
match = pattern.search(s, pos=overlap + 1, endpos=maxlen + sep_len)
if match is None:
return "", s
start: int = match.start()
end: int = match.end()
return s[:start], s[end:]
# -- characterize match location
match_start, match_end = match.span()
# -- matched separator is replaced by single-space in overlap string --
separator = " "
# -- in multi-space situation, fragment may have trailing whitespace because match is from
# -- right to left
fragment = s[:match_start].rstrip()
# -- remainder can have leading space when match is on "\n" followed by spaces --
raw_remainder = s[match_end:].lstrip()
if overlap <= len(separator):
return fragment, raw_remainder
# -- compute overlap --
tail_len = overlap - len(separator)
tail = fragment[-tail_len:].lstrip()
overlapped_remainder = tail + separator + raw_remainder
return fragment, overlapped_remainder
# ================================================================================================