fix: align pipeline delimiter chunking (#17723)

deepdoc/parser/txt_parser.py

@@ -19,7 +19,6 @@ import re
 
 from common.token_utils import num_tokens_from_string
 from deepdoc.parser.utils import get_text
-from rag.nlp import _split_oversized_unit
 from rag.nlp.delim import (
     compile_delimiter_pattern,
     normalize_text_newlines,
@@ -28,12 +27,12 @@ from rag.nlp.delim import (
 
 
 class RAGFlowTxtParser:
-    def __call__(self, fnm, binary=None, chunk_token_num=128, delimiter="\n!?;。；！？"):
+    def __call__(self, fnm, binary=None, chunk_token_num=128, delimiter="\n!?;。；！？", keep_delimiters=False):
         txt = get_text(fnm, binary)
-        return self.parser_txt(txt, chunk_token_num, delimiter)
+        return self.parser_txt(txt, chunk_token_num, delimiter, keep_delimiters)
 
     @classmethod
-    def parser_txt(cls, txt, chunk_token_num=128, delimiter="\n!?;。；！？"):
+    def parser_txt(cls, txt, chunk_token_num=128, delimiter="\n!?;。；！？", keep_delimiters=False):
         if not isinstance(txt, str):
             raise TypeError("txt type should be str!")
         cks = [""]
@@ -42,21 +41,15 @@ class RAGFlowTxtParser:
         def add_chunk(t):
             nonlocal cks, tk_nums
             tnum = num_tokens_from_string(t)
-
+            if tk_nums[-1] > chunk_token_num:
-            if cks[-1] == "":
+                cks.append(t)
-                cks[-1] = t
+                tk_nums.append(tnum)
-                tk_nums[-1] = tnum
+            else:
-                return
+                if cks[-1]:
-
+                    cks[-1] += "\n" + t
-            merged = cks[-1] + "\n" + t
+                else:
-            merged_tnum = num_tokens_from_string(merged)
+                    cks[-1] += t
-            if merged_tnum <= chunk_token_num:
+                tk_nums[-1] += tnum
-                cks[-1] = merged
-                tk_nums[-1] = merged_tnum
-                return
-
-            cks.append(t)
-            tk_nums.append(tnum)
 
         txt = normalize_text_newlines(txt)
         parsed_dels = parse_delimiter_field(delimiter)
@@ -67,18 +60,14 @@ class RAGFlowTxtParser:
             bool(dels),
         )
         secs = re.split(r"(%s)" % dels, txt) if dels else [txt]
-        for sec in secs:
+        for index, sec in enumerate(secs):
             if dels and re.match(f"^{dels}$", sec):
                 continue
             if not sec:
                 continue
-            if num_tokens_from_string(sec) <= chunk_token_num:
+            if keep_delimiters and index + 1 < len(secs) and re.match(f"^{dels}$", secs[index + 1]):
-                add_chunk(sec)
+                sec += secs[index + 1]
-                continue
+            add_chunk(sec)
-            pieces = _split_oversized_unit(sec, chunk_token_num, token_count_fn=num_tokens_from_string)
-            logging.debug("parser_txt: split oversized section (%d tokens) into %d pieces", num_tokens_from_string(sec), len(pieces))
-            for piece in pieces:
-                add_chunk(piece)
 
         logging.debug("parser_txt: %d sections -> %d chunks (chunk_token_num=%d)", len(secs), len(cks), chunk_token_num)
         return [[c, ""] for c in cks]

rag/flow/chunker/token_chunker.py

@@ -77,7 +77,7 @@ def _compile_delimiter_pattern(delimiters):
 
 
 def _split_text_by_pattern(text, pattern):
-    # Split text by the compiled delimiter pattern and keep delimiter text in each chunk.
+    # Split text by the compiled delimiter pattern and discard delimiters.
     if not pattern:
         return [text or ""]
 
@@ -85,10 +85,6 @@ def _split_text_by_pattern(text, pattern):
     chunks = []
     for i in range(0, len(split_texts), 2):
         chunk = split_texts[i]
-        if not chunk:
-            continue
-        if i + 1 < len(split_texts):
-            chunk += split_texts[i + 1]
         if chunk.strip():
             chunks.append(chunk)
     return chunks
@@ -317,16 +313,17 @@ class TokenChunker(ProcessBase):
                 self.set_output("chunks", [{"text": payload}] if payload.strip() else [])
                 self.callback(1, "Done.")
                 return
-            cks = (
+            if self._param.delimiter_mode == "delimiter":
-                _split_text_by_pattern(payload, delimiter_pattern)
+                cks = _split_text_by_pattern(payload, delimiter_pattern)
-                if delimiter_pattern
+            elif delimiter_pattern:
-                else naive_merge(
+                cks = _split_text_by_pattern(payload, delimiter_pattern)
+            else:
+                cks = naive_merge(
                     payload,
                     self._param.chunk_token_size,
                     "".join(self._param.delimiters),
                     overlapped_percent,
                 )
-            )
             if custom_pattern:
                 docs = []
                 for c in cks:
@@ -357,11 +354,47 @@ class TokenChunker(ProcessBase):
             self.set_output("chunks", [{"text": merged_text}] if merged_text.strip() else [])
             self.callback(1, "Done.")
             return
+
+        if self._param.delimiter_mode == "delimiter":
+            text_chunks = _build_json_chunks(json_result, "")
+            chunks = []
+            text_buffer = []
+
+            def flush_text_buffer():
+                if not text_buffer:
+                    return
+                combined_text = "".join(text_buffer)
+                split_texts = _split_text_by_pattern(combined_text, delimiter_pattern)
+                chunks.extend(
+                    {
+                        "text": text,
+                        "doc_type_kwd": "text",
+                        "ck_type": "text",
+                        "tk_nums": num_tokens_from_string(text),
+                    }
+                    for text in split_texts
+                    if text.strip()
+                )
+                text_buffer.clear()
+
+            for chunk in text_chunks:
+                if chunk["ck_type"] == "text":
+                    text_buffer.append(chunk["text"])
+                else:
+                    flush_text_buffer()
+                    chunks.append(chunk)
+            flush_text_buffer()
+            _attach_context_to_media_chunks(chunks, self._param.table_context_size, self._param.image_context_size)
+            await restore_pdf_text_previews(chunks, from_upstream, self._canvas)
+            self.set_output("chunks", _finalize_json_chunks(chunks))
+            self.callback(1, "Done.")
+            return
+
         # Structured JSON input is normalized first, then optionally enriched with
         # media context, and finally merged only when delimiter splitting is inactive.
         chunks = _build_json_chunks(json_result, delimiter_pattern)
         _attach_context_to_media_chunks(chunks, self._param.table_context_size, self._param.image_context_size)
-        if not delimiter_pattern:
+        if self._param.delimiter_mode == "token_size" and not delimiter_pattern:
             chunks = _merge_text_chunks_by_token_size(chunks, self._param.chunk_token_size, overlapped_percent)
 
         if custom_pattern:

rag/flow/parser/parser.py

@@ -1131,6 +1131,7 @@ class Parser(ProcessBase):
             blob,
             conf.get("chunk_token_num", 128),
             conf.get("delimiter", "\n!?;。；！？"),
+            keep_delimiters=True,
         )
         if conf.get("output_format") == "json":
             self.set_output("json", [{"text": section[0], "doc_type_kwd": "text"} for section in sections if section[0]])

test/unit_test/deepdoc/parser/test_txt_parser.py

@@ -1,178 +0,0 @@
-#
-#  Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-#
-
-"""Unit tests for ``RAGFlowTxtParser.parser_txt`` strict-cap behaviour.
-
-The pre-fix ``add_chunk`` fired its size check *after* the append, so each
-chunk could overshoot ``chunk_token_num`` by up to the size of one line. These
-tests assert the proactive projected-total invariant: no produced chunk may
-contain more than ``chunk_token_num`` tokens.
-"""
-
-import importlib.util
-import os
-import sys
-from unittest import mock
-
-_MOCK_MODULES = [
-    "xgboost",
-    "pdfplumber",
-    "huggingface_hub",
-    "PIL",
-    "PIL.Image",
-    "pypdf",
-    "sklearn",
-    "deepdoc.vision",
-    "deepdoc",
-    "deepdoc.parser",
-    "deepdoc.parser.utils",
-]
-_orig_modules = {m: sys.modules.get(m) for m in _MOCK_MODULES}
-_orig_get_text = getattr(sys.modules.get("deepdoc.parser.utils"), "get_text", None)
-
-try:
-    for _m in _MOCK_MODULES:
-        if _m not in sys.modules:
-            sys.modules[_m] = mock.MagicMock()
-
-    # ``get_text`` is invoked by ``RAGFlowTxtParser.__call__`` only, not by
-    # ``parser_txt``. Provide a permissive stub so the module loads.
-    sys.modules["deepdoc.parser.utils"].get_text = lambda *a, **kw: ""
-
-    def _find_project_root(marker="pyproject.toml"):
-        d = os.path.dirname(os.path.abspath(__file__))
-        while d != os.path.dirname(d):
-            if os.path.exists(os.path.join(d, marker)):
-                return d
-            d = os.path.dirname(d)
-        return None
-
-    _PROJECT_ROOT = _find_project_root()
-
-    _spec = importlib.util.spec_from_file_location(
-        "deepdoc.parser._txt_parser_under_test",
-        os.path.join(_PROJECT_ROOT, "deepdoc", "parser", "txt_parser.py"),
-    )
-    _mod = importlib.util.module_from_spec(_spec)
-    sys.modules["deepdoc.parser._txt_parser_under_test"] = _mod
-    _spec.loader.exec_module(_mod)
-
-    RAGFlowTxtParser = _mod.RAGFlowTxtParser
-finally:
-    for _m, _orig in _orig_modules.items():
-        if _orig is None:
-            sys.modules.pop(_m, None)
-        else:
-            sys.modules[_m] = _orig
-    if _orig_modules.get("deepdoc.parser.utils") is not None and _orig_get_text is not None:
-        _orig_modules["deepdoc.parser.utils"].get_text = _orig_get_text
-    if _orig_modules.get("deepdoc.parser") is not None and _orig_modules.get("deepdoc.parser.utils") is not None:
-        _orig_modules["deepdoc.parser"].utils = _orig_modules["deepdoc.parser.utils"]
-
-
-# A deterministic, tokenizer-free stand-in for ``num_tokens_from_string`` so
-# the assertions below reason in plain words and are independent of tiktoken.
-
-
-def _patch_word_count(monkeypatch_module):
-    def fake_num_tokens(s):
-        return len((s or "").split())
-
-    monkeypatch_module.setattr(_mod, "num_tokens_from_string", fake_num_tokens)
-
-
-def test_no_overshoot_when_packing_short_lines(monkeypatch):
-    """Lines of 25 tokens, budget 100 — every chunk must be <= 100 tokens."""
-    _patch_word_count(monkeypatch)
-    txt = " ".join(["alpha"] * 25) + "\n" + " ".join(["beta"] * 25) + "\n" + " ".join(["gamma"] * 25)
-    chunks = RAGFlowTxtParser.parser_txt(txt, chunk_token_num=100, delimiter="\n")
-    sizes = [len(c[0].split()) for c in chunks if c[0].strip()]
-    assert all(s <= 100 for s in sizes), sizes
-    # 75 tokens of content, expected a single 75-token chunk.
-    assert sum(sizes) == 75
-
-
-def test_no_overshoot_at_chunk_boundary(monkeypatch):
-    """Lines of 30 tokens, budget 100. Pre-fix the boundary chunk was 130 tokens."""
-    _patch_word_count(monkeypatch)
-    lines = [" ".join([f"w{i}"] * 30) for i in range(10)]  # 10 lines, 300 tokens
-    chunks = RAGFlowTxtParser.parser_txt("\n".join(lines), chunk_token_num=100, delimiter="\n")
-    sizes = [len(c[0].split()) for c in chunks if c[0].strip()]
-    assert all(s <= 100 for s in sizes), sizes
-
-
-def test_atomic_oversized_line_is_sub_split_on_whitespace(monkeypatch):
-    """A single line that exceeds the budget is split on whitespace atoms."""
-    _patch_word_count(monkeypatch)
-    huge_line = " ".join(["alpha"] * 80)  # 80 tokens, no internal delimiter
-    chunks = RAGFlowTxtParser.parser_txt(huge_line, chunk_token_num=50, delimiter="\n")
-    sizes = [len(c[0].split()) for c in chunks if c[0].strip()]
-    assert all(s <= 50 for s in sizes), sizes
-    assert sum(sizes) == 80
-    assert len(chunks) >= 2
-
-
-def test_empty_text_returns_empty(monkeypatch):
-    _patch_word_count(monkeypatch)
-    # Empty input produces a single empty chunk placeholder (existing
-    # behaviour the callers rely on). The hard-cap guarantee is that any
-    # chunk carrying content stays within the budget.
-    result = RAGFlowTxtParser.parser_txt("", chunk_token_num=128, delimiter="\n")
-    non_empty = [c for c in result if c[0].strip()]
-    assert non_empty == []
-    result2 = RAGFlowTxtParser.parser_txt("   \n\n  ", chunk_token_num=128, delimiter="\n")
-    non_empty2 = [c for c in result2 if c[0].strip()]
-    assert non_empty2 == []
-
-
-def test_unbroken_token_exceeding_budget_fallback(monkeypatch):
-    """A single unbroken non-whitespace string exceeding the budget is split
-    via the character-window/token-slicing fallback.
-    """
-
-    def char_count_tokens(s):
-        return len(s or "")
-
-    monkeypatch.setattr(_mod, "num_tokens_from_string", char_count_tokens)
-
-    huge_word = "a" * 80  # 80 characters/tokens, no whitespace
-    chunks = RAGFlowTxtParser.parser_txt(huge_word, chunk_token_num=30, delimiter="\n")
-
-    non_empty = [c[0] for c in chunks if c[0].strip()]
-    assert len(non_empty) >= 3
-    assert all(char_count_tokens(c) <= 30 for c in non_empty)
-    assert "".join(non_empty) == huge_word
-
-
-def test_newline_join_token_count_strict_cap(monkeypatch):
-    """Verify that joining chunks with newline does not overshoot chunk_token_num
-    even when individual token counts sum to <= budget but the newline pushes it over.
-    """
-
-    def char_count_tokens(s):
-        return len(s or "")
-
-    monkeypatch.setattr(_mod, "num_tokens_from_string", char_count_tokens)
-
-    # Two lines of 10 chars each. Budget = 20.
-    # line1 + "\n" + line2 = 10 + 1 + 10 = 21 chars/tokens, exceeding budget of 20.
-    line1 = "a" * 10
-    line2 = "b" * 10
-    txt = f"{line1}\n{line2}"
-    chunks = RAGFlowTxtParser.parser_txt(txt, chunk_token_num=20, delimiter="\n")
-    non_empty = [c[0] for c in chunks if c[0].strip()]
-    assert all(char_count_tokens(c) <= 20 for c in non_empty)
-    assert len(non_empty) == 2

web/src/utils/__tests__/delimiter-preview.test.ts

@@ -34,9 +34,7 @@ describe('parseDelimitersForDisplay', () => {
   });
 
   it('normalizes CRLF before parsing', () => {
-    expect(parseDelimitersForDisplay('\r\n').map((d) => d.raw)).toEqual([
+    expect(parseDelimitersForDisplay('\r\n').map((d) => d.raw)).toEqual(['\n']);
-      '\n',
-    ]);
     expect(parseDelimitersForDisplay('`\r\n`').map((d) => d.raw)).toEqual([
       '\n',
     ]);