mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-08-05 15:20:30 +08:00
fix: align pipeline delimiter chunking (#17723)
This commit is contained in:
buua436
@@ -19,7 +19,6 @@ import re
|
||||
|
||||
from common.token_utils import num_tokens_from_string
|
||||
from deepdoc.parser.utils import get_text
|
||||
from rag.nlp import _split_oversized_unit
|
||||
from rag.nlp.delim import (
|
||||
compile_delimiter_pattern,
|
||||
normalize_text_newlines,
|
||||
@@ -28,12 +27,12 @@ from rag.nlp.delim import (
|
||||
|
||||
|
||||
class RAGFlowTxtParser:
|
||||
def __call__(self, fnm, binary=None, chunk_token_num=128, delimiter="\n!?;。；！？"):
|
||||
def __call__(self, fnm, binary=None, chunk_token_num=128, delimiter="\n!?;。；！？", keep_delimiters=False):
|
||||
txt = get_text(fnm, binary)
|
||||
return self.parser_txt(txt, chunk_token_num, delimiter)
|
||||
return self.parser_txt(txt, chunk_token_num, delimiter, keep_delimiters)
|
||||
|
||||
@classmethod
|
||||
def parser_txt(cls, txt, chunk_token_num=128, delimiter="\n!?;。；！？"):
|
||||
def parser_txt(cls, txt, chunk_token_num=128, delimiter="\n!?;。；！？", keep_delimiters=False):
|
||||
if not isinstance(txt, str):
|
||||
raise TypeError("txt type should be str!")
|
||||
cks = [""]
|
||||
@@ -42,21 +41,15 @@ class RAGFlowTxtParser:
|
||||
def add_chunk(t):
|
||||
nonlocal cks, tk_nums
|
||||
tnum = num_tokens_from_string(t)
|
||||
|
||||
if cks[-1] == "":
|
||||
cks[-1] = t
|
||||
tk_nums[-1] = tnum
|
||||
return
|
||||
|
||||
merged = cks[-1] + "\n" + t
|
||||
merged_tnum = num_tokens_from_string(merged)
|
||||
if merged_tnum <= chunk_token_num:
|
||||
cks[-1] = merged
|
||||
tk_nums[-1] = merged_tnum
|
||||
return
|
||||
|
||||
cks.append(t)
|
||||
tk_nums.append(tnum)
|
||||
if tk_nums[-1] > chunk_token_num:
|
||||
cks.append(t)
|
||||
tk_nums.append(tnum)
|
||||
else:
|
||||
if cks[-1]:
|
||||
cks[-1] += "\n" + t
|
||||
else:
|
||||
cks[-1] += t
|
||||
tk_nums[-1] += tnum
|
||||
|
||||
txt = normalize_text_newlines(txt)
|
||||
parsed_dels = parse_delimiter_field(delimiter)
|
||||
@@ -67,18 +60,14 @@ class RAGFlowTxtParser:
|
||||
bool(dels),
|
||||
)
|
||||
secs = re.split(r"(%s)" % dels, txt) if dels else [txt]
|
||||
for sec in secs:
|
||||
for index, sec in enumerate(secs):
|
||||
if dels and re.match(f"^{dels}$", sec):
|
||||
continue
|
||||
if not sec:
|
||||
continue
|
||||
if num_tokens_from_string(sec) <= chunk_token_num:
|
||||
add_chunk(sec)
|
||||
continue
|
||||
pieces = _split_oversized_unit(sec, chunk_token_num, token_count_fn=num_tokens_from_string)
|
||||
logging.debug("parser_txt: split oversized section (%d tokens) into %d pieces", num_tokens_from_string(sec), len(pieces))
|
||||
for piece in pieces:
|
||||
add_chunk(piece)
|
||||
if keep_delimiters and index + 1 < len(secs) and re.match(f"^{dels}$", secs[index + 1]):
|
||||
sec += secs[index + 1]
|
||||
add_chunk(sec)
|
||||
|
||||
logging.debug("parser_txt: %d sections -> %d chunks (chunk_token_num=%d)", len(secs), len(cks), chunk_token_num)
|
||||
return [[c, ""] for c in cks]
|
||||
|
||||
@@ -77,7 +77,7 @@ def _compile_delimiter_pattern(delimiters):
|
||||
|
||||
|
||||
def _split_text_by_pattern(text, pattern):
|
||||
# Split text by the compiled delimiter pattern and keep delimiter text in each chunk.
|
||||
# Split text by the compiled delimiter pattern and discard delimiters.
|
||||
if not pattern:
|
||||
return [text or ""]
|
||||
|
||||
@@ -85,10 +85,6 @@ def _split_text_by_pattern(text, pattern):
|
||||
chunks = []
|
||||
for i in range(0, len(split_texts), 2):
|
||||
chunk = split_texts[i]
|
||||
if not chunk:
|
||||
continue
|
||||
if i + 1 < len(split_texts):
|
||||
chunk += split_texts[i + 1]
|
||||
if chunk.strip():
|
||||
chunks.append(chunk)
|
||||
return chunks
|
||||
@@ -317,16 +313,17 @@ class TokenChunker(ProcessBase):
|
||||
self.set_output("chunks", [{"text": payload}] if payload.strip() else [])
|
||||
self.callback(1, "Done.")
|
||||
return
|
||||
cks = (
|
||||
_split_text_by_pattern(payload, delimiter_pattern)
|
||||
if delimiter_pattern
|
||||
else naive_merge(
|
||||
if self._param.delimiter_mode == "delimiter":
|
||||
cks = _split_text_by_pattern(payload, delimiter_pattern)
|
||||
elif delimiter_pattern:
|
||||
cks = _split_text_by_pattern(payload, delimiter_pattern)
|
||||
else:
|
||||
cks = naive_merge(
|
||||
payload,
|
||||
self._param.chunk_token_size,
|
||||
"".join(self._param.delimiters),
|
||||
overlapped_percent,
|
||||
)
|
||||
)
|
||||
if custom_pattern:
|
||||
docs = []
|
||||
for c in cks:
|
||||
@@ -357,11 +354,47 @@ class TokenChunker(ProcessBase):
|
||||
self.set_output("chunks", [{"text": merged_text}] if merged_text.strip() else [])
|
||||
self.callback(1, "Done.")
|
||||
return
|
||||
|
||||
if self._param.delimiter_mode == "delimiter":
|
||||
text_chunks = _build_json_chunks(json_result, "")
|
||||
chunks = []
|
||||
text_buffer = []
|
||||
|
||||
def flush_text_buffer():
|
||||
if not text_buffer:
|
||||
return
|
||||
combined_text = "".join(text_buffer)
|
||||
split_texts = _split_text_by_pattern(combined_text, delimiter_pattern)
|
||||
chunks.extend(
|
||||
{
|
||||
"text": text,
|
||||
"doc_type_kwd": "text",
|
||||
"ck_type": "text",
|
||||
"tk_nums": num_tokens_from_string(text),
|
||||
}
|
||||
for text in split_texts
|
||||
if text.strip()
|
||||
)
|
||||
text_buffer.clear()
|
||||
|
||||
for chunk in text_chunks:
|
||||
if chunk["ck_type"] == "text":
|
||||
text_buffer.append(chunk["text"])
|
||||
else:
|
||||
flush_text_buffer()
|
||||
chunks.append(chunk)
|
||||
flush_text_buffer()
|
||||
_attach_context_to_media_chunks(chunks, self._param.table_context_size, self._param.image_context_size)
|
||||
await restore_pdf_text_previews(chunks, from_upstream, self._canvas)
|
||||
self.set_output("chunks", _finalize_json_chunks(chunks))
|
||||
self.callback(1, "Done.")
|
||||
return
|
||||
|
||||
# Structured JSON input is normalized first, then optionally enriched with
|
||||
# media context, and finally merged only when delimiter splitting is inactive.
|
||||
chunks = _build_json_chunks(json_result, delimiter_pattern)
|
||||
_attach_context_to_media_chunks(chunks, self._param.table_context_size, self._param.image_context_size)
|
||||
if not delimiter_pattern:
|
||||
if self._param.delimiter_mode == "token_size" and not delimiter_pattern:
|
||||
chunks = _merge_text_chunks_by_token_size(chunks, self._param.chunk_token_size, overlapped_percent)
|
||||
|
||||
if custom_pattern:
|
||||
|
||||
@@ -1131,6 +1131,7 @@ class Parser(ProcessBase):
|
||||
blob,
|
||||
conf.get("chunk_token_num", 128),
|
||||
conf.get("delimiter", "\n!?;。；！？"),
|
||||
keep_delimiters=True,
|
||||
)
|
||||
if conf.get("output_format") == "json":
|
||||
self.set_output("json", [{"text": section[0], "doc_type_kwd": "text"} for section in sections if section[0]])
|
||||
|
||||
@@ -1,178 +0,0 @@
|
||||
#
|
||||
# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
"""Unit tests for ``RAGFlowTxtParser.parser_txt`` strict-cap behaviour.
|
||||
|
||||
The pre-fix ``add_chunk`` fired its size check *after* the append, so each
|
||||
chunk could overshoot ``chunk_token_num`` by up to the size of one line. These
|
||||
tests assert the proactive projected-total invariant: no produced chunk may
|
||||
contain more than ``chunk_token_num`` tokens.
|
||||
"""
|
||||
|
||||
import importlib.util
|
||||
import os
|
||||
import sys
|
||||
from unittest import mock
|
||||
|
||||
_MOCK_MODULES = [
|
||||
"xgboost",
|
||||
"pdfplumber",
|
||||
"huggingface_hub",
|
||||
"PIL",
|
||||
"PIL.Image",
|
||||
"pypdf",
|
||||
"sklearn",
|
||||
"deepdoc.vision",
|
||||
"deepdoc",
|
||||
"deepdoc.parser",
|
||||
"deepdoc.parser.utils",
|
||||
]
|
||||
_orig_modules = {m: sys.modules.get(m) for m in _MOCK_MODULES}
|
||||
_orig_get_text = getattr(sys.modules.get("deepdoc.parser.utils"), "get_text", None)
|
||||
|
||||
try:
|
||||
for _m in _MOCK_MODULES:
|
||||
if _m not in sys.modules:
|
||||
sys.modules[_m] = mock.MagicMock()
|
||||
|
||||
# ``get_text`` is invoked by ``RAGFlowTxtParser.__call__`` only, not by
|
||||
# ``parser_txt``. Provide a permissive stub so the module loads.
|
||||
sys.modules["deepdoc.parser.utils"].get_text = lambda *a, **kw: ""
|
||||
|
||||
def _find_project_root(marker="pyproject.toml"):
|
||||
d = os.path.dirname(os.path.abspath(__file__))
|
||||
while d != os.path.dirname(d):
|
||||
if os.path.exists(os.path.join(d, marker)):
|
||||
return d
|
||||
d = os.path.dirname(d)
|
||||
return None
|
||||
|
||||
_PROJECT_ROOT = _find_project_root()
|
||||
|
||||
_spec = importlib.util.spec_from_file_location(
|
||||
"deepdoc.parser._txt_parser_under_test",
|
||||
os.path.join(_PROJECT_ROOT, "deepdoc", "parser", "txt_parser.py"),
|
||||
)
|
||||
_mod = importlib.util.module_from_spec(_spec)
|
||||
sys.modules["deepdoc.parser._txt_parser_under_test"] = _mod
|
||||
_spec.loader.exec_module(_mod)
|
||||
|
||||
RAGFlowTxtParser = _mod.RAGFlowTxtParser
|
||||
finally:
|
||||
for _m, _orig in _orig_modules.items():
|
||||
if _orig is None:
|
||||
sys.modules.pop(_m, None)
|
||||
else:
|
||||
sys.modules[_m] = _orig
|
||||
if _orig_modules.get("deepdoc.parser.utils") is not None and _orig_get_text is not None:
|
||||
_orig_modules["deepdoc.parser.utils"].get_text = _orig_get_text
|
||||
if _orig_modules.get("deepdoc.parser") is not None and _orig_modules.get("deepdoc.parser.utils") is not None:
|
||||
_orig_modules["deepdoc.parser"].utils = _orig_modules["deepdoc.parser.utils"]
|
||||
|
||||
|
||||
# A deterministic, tokenizer-free stand-in for ``num_tokens_from_string`` so
|
||||
# the assertions below reason in plain words and are independent of tiktoken.
|
||||
|
||||
|
||||
def _patch_word_count(monkeypatch_module):
|
||||
def fake_num_tokens(s):
|
||||
return len((s or "").split())
|
||||
|
||||
monkeypatch_module.setattr(_mod, "num_tokens_from_string", fake_num_tokens)
|
||||
|
||||
|
||||
def test_no_overshoot_when_packing_short_lines(monkeypatch):
|
||||
"""Lines of 25 tokens, budget 100 — every chunk must be <= 100 tokens."""
|
||||
_patch_word_count(monkeypatch)
|
||||
txt = " ".join(["alpha"] * 25) + "\n" + " ".join(["beta"] * 25) + "\n" + " ".join(["gamma"] * 25)
|
||||
chunks = RAGFlowTxtParser.parser_txt(txt, chunk_token_num=100, delimiter="\n")
|
||||
sizes = [len(c[0].split()) for c in chunks if c[0].strip()]
|
||||
assert all(s <= 100 for s in sizes), sizes
|
||||
# 75 tokens of content, expected a single 75-token chunk.
|
||||
assert sum(sizes) == 75
|
||||
|
||||
|
||||
def test_no_overshoot_at_chunk_boundary(monkeypatch):
|
||||
"""Lines of 30 tokens, budget 100. Pre-fix the boundary chunk was 130 tokens."""
|
||||
_patch_word_count(monkeypatch)
|
||||
lines = [" ".join([f"w{i}"] * 30) for i in range(10)] # 10 lines, 300 tokens
|
||||
chunks = RAGFlowTxtParser.parser_txt("\n".join(lines), chunk_token_num=100, delimiter="\n")
|
||||
sizes = [len(c[0].split()) for c in chunks if c[0].strip()]
|
||||
assert all(s <= 100 for s in sizes), sizes
|
||||
|
||||
|
||||
def test_atomic_oversized_line_is_sub_split_on_whitespace(monkeypatch):
|
||||
"""A single line that exceeds the budget is split on whitespace atoms."""
|
||||
_patch_word_count(monkeypatch)
|
||||
huge_line = " ".join(["alpha"] * 80) # 80 tokens, no internal delimiter
|
||||
chunks = RAGFlowTxtParser.parser_txt(huge_line, chunk_token_num=50, delimiter="\n")
|
||||
sizes = [len(c[0].split()) for c in chunks if c[0].strip()]
|
||||
assert all(s <= 50 for s in sizes), sizes
|
||||
assert sum(sizes) == 80
|
||||
assert len(chunks) >= 2
|
||||
|
||||
|
||||
def test_empty_text_returns_empty(monkeypatch):
|
||||
_patch_word_count(monkeypatch)
|
||||
# Empty input produces a single empty chunk placeholder (existing
|
||||
# behaviour the callers rely on). The hard-cap guarantee is that any
|
||||
# chunk carrying content stays within the budget.
|
||||
result = RAGFlowTxtParser.parser_txt("", chunk_token_num=128, delimiter="\n")
|
||||
non_empty = [c for c in result if c[0].strip()]
|
||||
assert non_empty == []
|
||||
result2 = RAGFlowTxtParser.parser_txt(" \n\n ", chunk_token_num=128, delimiter="\n")
|
||||
non_empty2 = [c for c in result2 if c[0].strip()]
|
||||
assert non_empty2 == []
|
||||
|
||||
|
||||
def test_unbroken_token_exceeding_budget_fallback(monkeypatch):
|
||||
"""A single unbroken non-whitespace string exceeding the budget is split
|
||||
via the character-window/token-slicing fallback.
|
||||
"""
|
||||
|
||||
def char_count_tokens(s):
|
||||
return len(s or "")
|
||||
|
||||
monkeypatch.setattr(_mod, "num_tokens_from_string", char_count_tokens)
|
||||
|
||||
huge_word = "a" * 80 # 80 characters/tokens, no whitespace
|
||||
chunks = RAGFlowTxtParser.parser_txt(huge_word, chunk_token_num=30, delimiter="\n")
|
||||
|
||||
non_empty = [c[0] for c in chunks if c[0].strip()]
|
||||
assert len(non_empty) >= 3
|
||||
assert all(char_count_tokens(c) <= 30 for c in non_empty)
|
||||
assert "".join(non_empty) == huge_word
|
||||
|
||||
|
||||
def test_newline_join_token_count_strict_cap(monkeypatch):
|
||||
"""Verify that joining chunks with newline does not overshoot chunk_token_num
|
||||
even when individual token counts sum to <= budget but the newline pushes it over.
|
||||
"""
|
||||
|
||||
def char_count_tokens(s):
|
||||
return len(s or "")
|
||||
|
||||
monkeypatch.setattr(_mod, "num_tokens_from_string", char_count_tokens)
|
||||
|
||||
# Two lines of 10 chars each. Budget = 20.
|
||||
# line1 + "\n" + line2 = 10 + 1 + 10 = 21 chars/tokens, exceeding budget of 20.
|
||||
line1 = "a" * 10
|
||||
line2 = "b" * 10
|
||||
txt = f"{line1}\n{line2}"
|
||||
chunks = RAGFlowTxtParser.parser_txt(txt, chunk_token_num=20, delimiter="\n")
|
||||
non_empty = [c[0] for c in chunks if c[0].strip()]
|
||||
assert all(char_count_tokens(c) <= 20 for c in non_empty)
|
||||
assert len(non_empty) == 2
|
||||
@@ -34,9 +34,7 @@ describe('parseDelimitersForDisplay', () => {
|
||||
});
|
||||
|
||||
it('normalizes CRLF before parsing', () => {
|
||||
expect(parseDelimitersForDisplay('\r\n').map((d) => d.raw)).toEqual([
|
||||
'\n',
|
||||
]);
|
||||
expect(parseDelimitersForDisplay('\r\n').map((d) => d.raw)).toEqual(['\n']);
|
||||
expect(parseDelimitersForDisplay('`\r\n`').map((d) => d.raw)).toEqual([
|
||||
'\n',
|
||||
]);
|
||||
|
||||
Reference in New Issue
Block a user