refactor(parser): align TextParser with Python _code delimiter split (#18128)

internal/parser/parser/parse_with_result_test.go

@@ -33,6 +33,7 @@
 package parser
 
 import (
+	"os"
 	"strings"
 	"testing"
 
@@ -85,10 +86,12 @@ func TestTextParser_ParseWithResult_Empty(t *testing.T) {
 	}
 }
 
-// TestTextParser_ParseWithResult_LongParagraphSlicing pins the
-// maxItemBytes boundary behaviour. A single paragraph longer
-// than 8192 bytes is sliced at the nearest line boundary.
-func TestTextParser_ParseWithResult_LongParagraphSlicing(t *testing.T) {
+// TestTextParser_ParseWithResult_NoSizeCap pins that the parser performs no
+// per-item byte slicing: a single continuous run longer than any prior cap
+// (here 9000 'a's with no delimiter) stays as one item whose full content is
+// preserved. Sizing is delegated to the chunker / embedding truncation, matching
+// python's parser_txt (which also does no size slicing).
+func TestTextParser_ParseWithResult_NoSizeCap(t *testing.T) {
 	ctx := t.Context()
 	p := NewTextParser()
 	long := strings.Repeat("a", 9000)
@@ -96,13 +99,11 @@ func TestTextParser_ParseWithResult_LongParagraphSlicing(t *testing.T) {
 	if res.Err != nil {
 		t.Fatalf("ParseWithResult: %v", res.Err)
 	}
-	if len(res.JSON) < 2 {
-		t.Errorf("JSON len = %d, want >=2 (sliced at maxItemBytes)", len(res.JSON))
+	if len(res.JSON) != 1 {
+		t.Fatalf("JSON len = %d, want 1 (no per-item size cap)", len(res.JSON))
 	}
-	for i, it := range res.JSON {
-		if txt, _ := it["text"].(string); len(txt) > 8192 {
-			t.Errorf("JSON[%d].text len = %d, exceeds maxItemBytes=8192", i, len(txt))
-		}
+	if txt, _ := res.JSON[0]["text"].(string); txt != long {
+		t.Errorf("text len = %d, want %d (full content preserved, not sliced)", len(txt), len(long))
 	}
 }
 
@@ -282,3 +283,197 @@ func TestGetParser_RoutesTextAndCode(t *testing.T) {
 		t.Fatal("TextParser does not implement ParseResultProducer")
 	}
 }
+
+// TestTextParser_ParseWithResult_DefaultDelimiter pins the alignment fix:
+// TextParser now splits on the flow parser's default delimiter set
+// ("\n!?;。；！？"), mirroring deepdoc TxtParser.parser_txt, instead of only on
+// blank lines. keep_delimiters=True (the flow _code path) keeps each trailing
+// delimiter attached, so sentence-ending punctuation survives the split.
+func TestTextParser_ParseWithResult_DefaultDelimiter(t *testing.T) {
+	ctx := t.Context()
+	p := NewTextParser()
+
+	// Single newlines now split too (previously only "\n\n" did).
+	src := []byte("First line.\nSecond line.\nThird line.")
+	res := p.ParseWithResult(ctx, "doc.txt", src)
+	if res.Err != nil {
+		t.Fatalf("ParseWithResult: %v", res.Err)
+	}
+	if len(res.JSON) != 3 {
+		t.Fatalf("JSON len = %d, want 3 (single-newline split)", len(res.JSON))
+	}
+
+	// Sentence delimiters split and keep the delimiter attached. The period
+	// "." is NOT in the default set, so "Foo. Bar" stays joined until the ";".
+	// TrimSpace drops the incidental leading space before each delimiter (the
+	// package's established convention, also used by markdown leafText).
+	src = []byte("Hello! World? Foo. Bar; Baz。 Qux！")
+	res = p.ParseWithResult(ctx, "doc.txt", src)
+	want := []string{"Hello!", "World?", "Foo. Bar;", "Baz。", "Qux！"}
+	if len(res.JSON) != len(want) {
+		t.Fatalf("JSON len = %d, want %d: %#v", len(res.JSON), len(want), res.JSON)
+	}
+	for i, w := range want {
+		if got := res.JSON[i]["text"]; got != w {
+			t.Errorf("JSON[%d].text = %v, want %v", i, got, w)
+		}
+	}
+
+	// Chinese sentence delimiters split the same way.
+	src = []byte("这是第一句。这是第二句！第三句？结尾。")
+	res = p.ParseWithResult(ctx, "doc.txt", src)
+	if len(res.JSON) != 4 {
+		t.Fatalf("JSON len = %d, want 4 (CJK delimiter split)", len(res.JSON))
+	}
+}
+
+// TestTextParser_ParseWithResult_NewlineNormalization pins the
+// normalizeTextNewlines contract: CRLF ("\r\n") and lone-CR ("\r") line
+// endings fold to LF before splitting, so every variant of the same logical
+// content yields identical items. This mirrors rag/nlp/delim.
+// normalize_text_newlines, which is what Python splits on, so Windows-line
+// documents parse identically to Unix ones.
+func TestTextParser_ParseWithResult_NewlineNormalization(t *testing.T) {
+	ctx := t.Context()
+	p := NewTextParser()
+
+	// Same logical content expressed with LF, CRLF, and lone-CR line endings.
+	lf := "First line.\nSecond line! Third? Fourth."
+	crlf := strings.ReplaceAll(lf, "\n", "\r\n")
+	cr := strings.ReplaceAll(lf, "\n", "\r")
+
+	extract := func(src string) []string {
+		res := p.ParseWithResult(ctx, "doc.txt", []byte(src))
+		if res.Err != nil {
+			t.Fatalf("ParseWithResult: %v", res.Err)
+		}
+		out := make([]string, 0, len(res.JSON))
+		for _, it := range res.JSON {
+			if txt, _ := it["text"].(string); txt != "" {
+				out = append(out, txt)
+			}
+		}
+		return out
+	}
+
+	want := extract(lf)
+	if len(want) == 0 {
+		t.Fatal("LF baseline produced no items")
+	}
+	for _, variant := range []struct {
+		name string
+		src  string
+	}{
+		{"crlf", crlf},
+		{"cr", cr},
+	} {
+		got := extract(variant.src)
+		if len(got) != len(want) {
+			t.Fatalf("%s: JSON len = %d, want %d: %#v", variant.name, len(got), len(want), got)
+		}
+		for i := range want {
+			if got[i] != want[i] {
+				t.Errorf("%s: JSON[%d].text = %q, want %q", variant.name, i, got[i], want[i])
+			}
+		}
+	}
+}
+
+// TestTextParser_AlignmentGolden verifies Go's ParseWithResult output is
+// content-equivalent to Python's _code on the shared sample, using the shared
+// concatenation-normalization alignment tool (align_test.go). Python applies
+// the OVER_CAP token merge (chunking ownership retained by the Go Chunker per
+// contract #17799), so item counts differ; the
+// comparison normalizes both (delimiters stripped, whitespace collapsed) and
+// joins on whitespace, so only CONTENT equivalence — not byte-exact layout — is
+// checked. The golden files are a NORMALIZED content baseline, not a verbatim
+// Python transcript: a fresh _code run at chunk_token_num=128 may merge into a
+// different item count/structure (e.g. the en sample collapses to one chunk while
+// the golden keeps prose and code as two items) and may collapse inter-sentence
+// newlines, so do not treat them as byte-exact.
+//
+// No generator script is committed. The baseline meta records generator, sample,
+// delimiter, keep_delimiters and chunk_token_num (see textcode.python.en/zh.golden.json);
+// sample, delimiter, keep_delimiters, chunk_token_num): call the python flow
+// _code on the sample with keep_delimiters=True and the default delimiter set,
+// then project each merged section to {"text": section[0], "doc_type_kwd": "text"}.
+func TestTextParser_AlignmentGolden(t *testing.T) {
+	ctx := t.Context()
+	p := NewTextParser()
+
+	cases := []struct {
+		name   string
+		sample string
+		golden string
+	}{
+		{"en", "testdata/textcode.sample.en.txt", "testdata/textcode.python.en.golden.json"},
+		{"zh", "testdata/textcode.sample.zh.txt", "testdata/textcode.python.zh.golden.json"},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			sample, err := os.ReadFile(tc.sample)
+			if err != nil {
+				t.Fatalf("read sample: %v", err)
+			}
+			res := p.ParseWithResult(ctx, tc.sample, sample)
+			if res.Err != nil {
+				t.Fatalf("ParseWithResult: %v", res.Err)
+			}
+
+			gd := LoadGoldenDoc(t, tc.golden)
+			ignore := AcceptedDivergences(gd.Meta)
+
+			goText := FilterOutDocTypes(FilterByDocType(res.JSON, "text"), ignore)
+			pyText := FilterOutDocTypes(FilterByDocType(gd.Items, "text"), ignore)
+
+			if ok, diff := CompareAlignment(goText, pyText, TextCodeAlignOptions(DefaultTextCodeDelimiter)); !ok {
+				t.Fatalf("text&code parser not aligned with Python golden:%s", diff)
+			}
+		})
+	}
+}
+
+// TestTextParser_AdjacentDelimiters pins Go's behavior on adjacent
+// delimiters, confirming it matches Python's deepdoc TxtParser.parser_txt
+// delimiter-loop exactly (not a divergence). Both ports run the same
+// re.split(r"(%s)" % dels, txt) loop with keep_delimiters=True and merge a
+// run of adjacent delimiters into the preceding segment, so the standalone
+// second delimiter is dropped on both sides: "a!!b" → ["a!", "b"] (verified
+// against deepdoc/parser/txt_parser.py). The alignment test's delimiter-strip
+// normalization also reconciles this, but this test guards splitCapturingDelims
+// directly so a future silent change there is caught independently.
+func TestTextParser_AdjacentDelimiters(t *testing.T) {
+	ctx := t.Context()
+	p := NewTextParser()
+
+	// Two adjacent sentence delimiters: Go (and Python's parser_txt) merge
+	// them into the preceding segment and drop the standalone second delimiter.
+	src := []byte("a!!b")
+	res := p.ParseWithResult(ctx, "doc.txt", src)
+	if res.Err != nil {
+		t.Fatalf("ParseWithResult: %v", res.Err)
+	}
+	want := []string{"a!", "b"}
+	if len(res.JSON) != len(want) {
+		t.Fatalf("adjacent delimiters: JSON len = %d, want %d: %#v", len(res.JSON), len(want), res.JSON)
+	}
+	for i, w := range want {
+		if got := res.JSON[i]["text"]; got != w {
+			t.Errorf("adjacent delimiters: JSON[%d].text = %v, want %v", i, got, w)
+		}
+	}
+
+	// Delimiters separated by text each attach to their own segment (no merge
+	// across the gap).
+	src = []byte("x?y!z")
+	res = p.ParseWithResult(ctx, "doc.txt", src)
+	want = []string{"x?", "y!", "z"}
+	if len(res.JSON) != len(want) {
+		t.Fatalf("mixed delimiters: JSON len = %d, want %d: %#v", len(res.JSON), len(want), res.JSON)
+	}
+	for i, w := range want {
+		if got := res.JSON[i]["text"]; got != w {
+			t.Errorf("mixed delimiters: JSON[%d].text = %v, want %v", i, got, w)
+		}
+	}
+}

internal/parser/parser/testdata/textcode.python.en.golden.json

@@ -0,0 +1,23 @@
+{
+  "meta": {
+    "generator": "rag/flow/parser/parser.py:_code",
+    "sample": "internal/parser/parser/testdata/textcode.sample.en.txt",
+    "delimiter": "\n!?;。；！？",
+    "chunk_token_num": 128,
+    "keep_delimiters": true,
+    "separate_tables": false,
+    "accepted_divergences": [],
+    "python_engine": "deepdoc.parser.txt_parser.RAGFlowTxtParser",
+    "note": "No generator script is committed. To regenerate: call _code on the sample with keep_delimiters=True (chunk_token_num=128, default delimiter set), project each merged section to {\"text\": section[0], \"doc_type_kwd\": \"text\"}, then dump {meta, items}. The golden is a NORMALIZED content baseline, not a verbatim Python transcript (see TestTextParser_AlignmentGolden): a fresh _code run at chunk_token_num=128 may merge into a different item count/structure and may collapse inter-sentence newlines, so this file is not byte-exact."
+  },
+  "items": [
+    {
+      "text": "RAGFlow parses plain text and source code through the text&code family. This is the first English sentence. Here is a second sentence with more detail!\n Is this a question that the parser should handle?\nA blank line separates paragraphs. Semicolons also act as delimiters;\n this clause stays attached to the previous one. The parser keeps the trailing punctuation so sentence boundaries survive the split.",
+      "doc_type_kwd": "text"
+    },
+    {
+      "text": "def greet(name):\n\n    return f\"hello, {name}\"\n\ndef main():\n\n    print(greet(\"world\"))\n\nLong code lines are kept as their own segments. The parser does not perform the token merge;\n the downstream chunker owns chunking per PARSER_ALIGNMENT_HANDOFF.md section 2.3.\n",
+      "doc_type_kwd": "text"
+    }
+  ]
+}

internal/parser/parser/testdata/textcode.python.zh.golden.json

@@ -0,0 +1,19 @@
+{
+  "meta": {
+    "generator": "rag/flow/parser/parser.py:_code",
+    "sample": "internal/parser/parser/testdata/textcode.sample.zh.txt",
+    "delimiter": "\n!?;。；！？",
+    "chunk_token_num": 128,
+    "keep_delimiters": true,
+    "separate_tables": false,
+    "accepted_divergences": [],
+    "python_engine": "deepdoc.parser.txt_parser.RAGFlowTxtParser",
+    "note": "No generator script is committed. To regenerate: call _code on the sample with keep_delimiters=True (chunk_token_num=128, default delimiter set), project each merged section to {\"text\": section[0], \"doc_type_kwd\": \"text\"}, then dump {meta, items}. The golden is a NORMALIZED content baseline, not a verbatim Python transcript (see TestTextParser_AlignmentGolden): a fresh _code run at chunk_token_num=128 may merge into a different item count/structure and may collapse inter-sentence newlines, so this file is not byte-exact."
+  },
+  "items": [
+    {
+      "text": "这是第一段中文。逗号不是分隔符，但句号是！第二句以感叹号结尾？第三句以问号结尾。中文段落同样按默认分隔符切分。",
+      "doc_type_kwd": "text"
+    }
+  ]
+}

internal/parser/parser/testdata/textcode.sample.en.txt

@@ -0,0 +1,11 @@
+RAGFlow parses plain text and source code through the text&code family. This is the first English sentence. Here is a second sentence with more detail! Is this a question that the parser should handle?
+
+A blank line separates paragraphs. Semicolons also act as delimiters; this clause stays attached to the previous one. The parser keeps the trailing punctuation so sentence boundaries survive the split.
+
+def greet(name):
+    return f"hello, {name}"
+
+def main():
+    print(greet("world"))
+
+Long code lines are kept as their own segments. The parser does not perform the token merge; the downstream chunker owns chunking per PARSER_ALIGNMENT_HANDOFF.md section 2.3.

internal/parser/parser/testdata/textcode.sample.zh.txt

@@ -0,0 +1 @@
+这是第一段中文。逗号不是分隔符，但句号是！第二句以感叹号结尾？第三句以问号结尾。中文段落同样按默认分隔符切分。

internal/parser/parser/text_parser.go

@@ -22,36 +22,31 @@
 // side needs a parser for these families so `text&code` resolves to a
 // real ParseResultProducer.
 //
-// TextParser fills that gap with a minimal but real implementation:
-// it splits the input into paragraph-sized items and emits the
-// python-compatible `{text, doc_type_kwd:"text"}` shape. The
-// python TxtParser additionally does layout-aware section
-// detection; the Go version is intentionally simpler because (a)
-// no production template currently relies on text&code for richer
-// structure than paragraph items.
+// TextParser fills that gap with a real implementation: it splits the
+// input into fine segments on the flow parser's default delimiter set and
+// emits the python-compatible `{text, doc_type_kwd:"text"}` shape. Block
+// boundaries converge to the Python flow TxtParser (which uses the same
+// delimiter set); the OVER_CAP token merge that Python applies afterwards is
+// intentionally NOT performed here — chunking ownership stays with the
+// downstream Chunker (contract #17799), so
+// item counts differ from Python's merged chunks and are reconciled by the
+// stitch-compare alignment test (align_test.go).
 
 package parser
 
 import (
-	"bytes"
 	"context"
+	"regexp"
 	"strings"
 )
 
 // TextParser is the text&code family parser. It implements the
 // structured ParseResultProducer contract directly.
-type TextParser struct {
-	// maxItemBytes caps each emitted item's text length. The
-	// python TxtParser uses similar paragraph-style chunking;
-	// 8192 bytes is a conservative ceiling that prevents the
-	// downstream chunker from receiving oversized inputs.
-	maxItemBytes int
-}
+type TextParser struct{}
 
-// NewTextParser constructs a TextParser with the default
-// paragraph-sized chunking ceiling.
+// NewTextParser constructs a TextParser.
 func NewTextParser() *TextParser {
-	return &TextParser{maxItemBytes: 8192}
+	return &TextParser{}
 }
 
 // ParseWithResult emits one item per non-empty paragraph. The
@@ -65,7 +60,7 @@ func (p *TextParser) ParseWithResult(ctx context.Context, filename string, data
 	if !utf8Valid(data) {
 		return ParseResult{Err: errInvalidUTF8}
 	}
-	items := textParserItems(data, p.maxItemBytes)
+	items := textParserItems(data)
 	if items == nil {
 		items = []map[string]any{{"text": "", "doc_type_kwd": "text"}}
 	}
@@ -140,31 +135,108 @@ func decodeRune(p []byte) (rune, int) {
 	return 0xFFFD, 1
 }
 
-// textParserItems splits `data` into paragraph-sized chunks. The
-// split rule mirrors the python TxtParser: blank lines separate
-// paragraphs; long paragraphs are sliced at maxItemBytes boundaries.
-func textParserItems(data []byte, maxItemBytes int) []map[string]any {
-	var items []map[string]any
-	for _, raw := range bytes.Split(data, []byte("\n\n")) {
-		text := strings.TrimSpace(string(raw))
-		if text == "" {
+// defaultTextDelimiterPattern is the regexp alternation of the flow parser's
+// default delimiter set DefaultTextCodeDelimiter (rag/flow/parser/parser.py:_code
+// → deepdoc TxtParser default "\n!?;。；！？"), each rune re.escape'd to mirror
+// rag/nlp/delim.compile_delimiter_pattern. The Parser component has no user-facing
+// delimiter config entry, so this default
+// is exactly what the python flow always splits on. The shared DefaultTextCodeDelimiter
+// const lives in delimiter.go so production and the alignment tests use one source
+// of truth. Go's regexp.Split drops captured delimiters, so splitCapturingDelims
+// walks the match indexes manually to reproduce python's re.split(r"(%s)" % pattern, txt)
+// interleaving.
+var (
+	defaultTextDelimiterPattern = buildDelimiterPattern(DefaultTextCodeDelimiter)
+	textDelimiterSplitRe        = regexp.MustCompile(defaultTextDelimiterPattern)
+	textDelimiterExactRe        = regexp.MustCompile("^(?:" + defaultTextDelimiterPattern + ")$")
+)
+
+// buildDelimiterPattern builds an alternation of re.escape'd delimiter runes
+// (longest-first is a no-op here: every delimiter in the default set is a
+// single rune, so insertion order is preserved like python's stable sort).
+func buildDelimiterPattern(delims string) string {
+	parts := make([]string, 0, len(delims))
+	for _, r := range delims {
+		parts = append(parts, regexp.QuoteMeta(string(r)))
+	}
+	return strings.Join(parts, "|")
+}
+
+// normalizeTextNewlines folds CRLF and standalone CR to LF, mirroring
+// rag/nlp/delim.normalize_text_newlines so Windows-line-ending documents split
+// identically to Unix ones.
+func normalizeTextNewlines(s string) string {
+	if s == "" {
+		return s
+	}
+	s = strings.ReplaceAll(s, "\r\n", "\n")
+	return strings.ReplaceAll(s, "\r", "\n")
+}
+
+// splitCapturingDelims reproduces python re.split(r"(%s)" % pattern, s): it
+// splits on the regexp and includes each matched delimiter as its own element
+// (with empty strings between adjacent delimiters) so callers can keep or drop
+// them. Go's regexp.Split discards captured groups, hence the manual walk.
+func splitCapturingDelims(s string, re *regexp.Regexp) []string {
+	locs := re.FindAllStringIndex(s, -1)
+	if len(locs) == 0 {
+		return []string{s}
+	}
+	out := make([]string, 0, 2*len(locs)+1)
+	prev := 0
+	for _, loc := range locs {
+		out = append(out, s[prev:loc[0]])
+		out = append(out, s[loc[0]:loc[1]])
+		prev = loc[1]
+	}
+	out = append(out, s[prev:])
+	return out
+}
+
+// textParserItems splits data into fine segments on the flow parser's default
+// delimiter set, mirroring deepdoc.parser.txt_parser.TxtParser.parser_txt up to
+// (but not including) the OVER_CAP token merge. The token merge is intentionally
+// NOT performed here — chunking ownership stays with the downstream Chunker per
+// contract #17799 — so item counts differ from the
+// python flow's merged chunks and are reconciled by the stitch-compare alignment
+// test (align_test.go).
+//
+// Unlike the python signature default keep_delimiters=False, the flow _code
+// path calls TxtParser with keep_delimiters=True, so each segment keeps its
+// trailing delimiter attached (sentence-ending punctuation preserved for code
+// and prose).
+//
+// No per-item byte cap is applied: the parser is a pure delimiter splitter, just
+// like python's parser_txt (which also does no size slicing). Sizing belongs to
+// the chunker and the embedding truncation step, so a continuous run longer than
+// the embedding token budget (e.g. a minified / no-newline file) is kept as one
+// item here and collapsed to one chunk downstream — matching python's behaviour
+// rather than diverging from it.
+func textParserItems(data []byte) []map[string]any {
+	txt := normalizeTextNewlines(string(data))
+	secs := splitCapturingDelims(txt, textDelimiterSplitRe)
+
+	var paras []string
+	for i, sec := range secs {
+		if textDelimiterExactRe.MatchString(sec) {
 			continue
 		}
-		if maxItemBytes > 0 && len(text) > maxItemBytes {
-			// Slice at the nearest newline below maxItemBytes;
-			// falls back to a hard slice when no newline exists.
-			cut := strings.LastIndex(text[:maxItemBytes], "\n")
-			if cut <= 0 {
-				cut = maxItemBytes
-			}
-			items = append(items, map[string]any{
-				"text":         strings.TrimSpace(text[:cut]),
-				"doc_type_kwd": "text",
-			})
-			text = strings.TrimSpace(text[cut:])
-			if text == "" {
-				continue
-			}
+		if sec == "" {
+			continue
+		}
+		// keep_delimiters=True: append the delimiter to the segment it
+		// follows, mirroring python's parser_txt.
+		if i+1 < len(secs) && textDelimiterExactRe.MatchString(secs[i+1]) {
+			sec += secs[i+1]
+		}
+		paras = append(paras, sec)
+	}
+
+	var items []map[string]any
+	for _, para := range paras {
+		text := strings.TrimSpace(para)
+		if text == "" {
+			continue
 		}
 		items = append(items, map[string]any{
 			"text":         text,