From 7b2d052f8a094fa3f66e5ae327b8164df4c3d1a2 Mon Sep 17 00:00:00 2001 From: Jack Date: Wed, 12 Aug 2026 13:07:23 +0800 Subject: [PATCH] refactor(parser): align TextParser with Python _code delimiter split (#18128) --- .../parser/parser/parse_with_result_test.go | 215 +++++++++++++++++- .../testdata/textcode.python.en.golden.json | 23 ++ .../testdata/textcode.python.zh.golden.json | 19 ++ .../parser/testdata/textcode.sample.en.txt | 11 + .../parser/testdata/textcode.sample.zh.txt | 1 + internal/parser/parser/text_parser.go | 156 +++++++++---- 6 files changed, 373 insertions(+), 52 deletions(-) create mode 100644 internal/parser/parser/testdata/textcode.python.en.golden.json create mode 100644 internal/parser/parser/testdata/textcode.python.zh.golden.json create mode 100644 internal/parser/parser/testdata/textcode.sample.en.txt create mode 100644 internal/parser/parser/testdata/textcode.sample.zh.txt diff --git a/internal/parser/parser/parse_with_result_test.go b/internal/parser/parser/parse_with_result_test.go index 8ab285dd69..a33d274a99 100644 --- a/internal/parser/parser/parse_with_result_test.go +++ b/internal/parser/parser/parse_with_result_test.go @@ -33,6 +33,7 @@ package parser import ( + "os" "strings" "testing" @@ -85,10 +86,12 @@ func TestTextParser_ParseWithResult_Empty(t *testing.T) { } } -// TestTextParser_ParseWithResult_LongParagraphSlicing pins the -// maxItemBytes boundary behaviour. A single paragraph longer -// than 8192 bytes is sliced at the nearest line boundary. -func TestTextParser_ParseWithResult_LongParagraphSlicing(t *testing.T) { +// TestTextParser_ParseWithResult_NoSizeCap pins that the parser performs no +// per-item byte slicing: a single continuous run longer than any prior cap +// (here 9000 'a's with no delimiter) stays as one item whose full content is +// preserved. Sizing is delegated to the chunker / embedding truncation, matching +// python's parser_txt (which also does no size slicing). +func TestTextParser_ParseWithResult_NoSizeCap(t *testing.T) { ctx := t.Context() p := NewTextParser() long := strings.Repeat("a", 9000) @@ -96,13 +99,11 @@ func TestTextParser_ParseWithResult_LongParagraphSlicing(t *testing.T) { if res.Err != nil { t.Fatalf("ParseWithResult: %v", res.Err) } - if len(res.JSON) < 2 { - t.Errorf("JSON len = %d, want >=2 (sliced at maxItemBytes)", len(res.JSON)) + if len(res.JSON) != 1 { + t.Fatalf("JSON len = %d, want 1 (no per-item size cap)", len(res.JSON)) } - for i, it := range res.JSON { - if txt, _ := it["text"].(string); len(txt) > 8192 { - t.Errorf("JSON[%d].text len = %d, exceeds maxItemBytes=8192", i, len(txt)) - } + if txt, _ := res.JSON[0]["text"].(string); txt != long { + t.Errorf("text len = %d, want %d (full content preserved, not sliced)", len(txt), len(long)) } } @@ -282,3 +283,197 @@ func TestGetParser_RoutesTextAndCode(t *testing.T) { t.Fatal("TextParser does not implement ParseResultProducer") } } + +// TestTextParser_ParseWithResult_DefaultDelimiter pins the alignment fix: +// TextParser now splits on the flow parser's default delimiter set +// ("\n!?;。；！？"), mirroring deepdoc TxtParser.parser_txt, instead of only on +// blank lines. keep_delimiters=True (the flow _code path) keeps each trailing +// delimiter attached, so sentence-ending punctuation survives the split. +func TestTextParser_ParseWithResult_DefaultDelimiter(t *testing.T) { + ctx := t.Context() + p := NewTextParser() + + // Single newlines now split too (previously only "\n\n" did). + src := []byte("First line.\nSecond line.\nThird line.") + res := p.ParseWithResult(ctx, "doc.txt", src) + if res.Err != nil { + t.Fatalf("ParseWithResult: %v", res.Err) + } + if len(res.JSON) != 3 { + t.Fatalf("JSON len = %d, want 3 (single-newline split)", len(res.JSON)) + } + + // Sentence delimiters split and keep the delimiter attached. The period + // "." is NOT in the default set, so "Foo. Bar" stays joined until the ";". + // TrimSpace drops the incidental leading space before each delimiter (the + // package's established convention, also used by markdown leafText). + src = []byte("Hello! World? Foo. Bar; Baz。 Qux！") + res = p.ParseWithResult(ctx, "doc.txt", src) + want := []string{"Hello!", "World?", "Foo. Bar;", "Baz。", "Qux！"} + if len(res.JSON) != len(want) { + t.Fatalf("JSON len = %d, want %d: %#v", len(res.JSON), len(want), res.JSON) + } + for i, w := range want { + if got := res.JSON[i]["text"]; got != w { + t.Errorf("JSON[%d].text = %v, want %v", i, got, w) + } + } + + // Chinese sentence delimiters split the same way. + src = []byte("这是第一句。这是第二句！第三句？结尾。") + res = p.ParseWithResult(ctx, "doc.txt", src) + if len(res.JSON) != 4 { + t.Fatalf("JSON len = %d, want 4 (CJK delimiter split)", len(res.JSON)) + } +} + +// TestTextParser_ParseWithResult_NewlineNormalization pins the +// normalizeTextNewlines contract: CRLF ("\r\n") and lone-CR ("\r") line +// endings fold to LF before splitting, so every variant of the same logical +// content yields identical items. This mirrors rag/nlp/delim. +// normalize_text_newlines, which is what Python splits on, so Windows-line +// documents parse identically to Unix ones. +func TestTextParser_ParseWithResult_NewlineNormalization(t *testing.T) { + ctx := t.Context() + p := NewTextParser() + + // Same logical content expressed with LF, CRLF, and lone-CR line endings. + lf := "First line.\nSecond line! Third? Fourth." + crlf := strings.ReplaceAll(lf, "\n", "\r\n") + cr := strings.ReplaceAll(lf, "\n", "\r") + + extract := func(src string) []string { + res := p.ParseWithResult(ctx, "doc.txt", []byte(src)) + if res.Err != nil { + t.Fatalf("ParseWithResult: %v", res.Err) + } + out := make([]string, 0, len(res.JSON)) + for _, it := range res.JSON { + if txt, _ := it["text"].(string); txt != "" { + out = append(out, txt) + } + } + return out + } + + want := extract(lf) + if len(want) == 0 { + t.Fatal("LF baseline produced no items") + } + for _, variant := range []struct { + name string + src string + }{ + {"crlf", crlf}, + {"cr", cr}, + } { + got := extract(variant.src) + if len(got) != len(want) { + t.Fatalf("%s: JSON len = %d, want %d: %#v", variant.name, len(got), len(want), got) + } + for i := range want { + if got[i] != want[i] { + t.Errorf("%s: JSON[%d].text = %q, want %q", variant.name, i, got[i], want[i]) + } + } + } +} + +// TestTextParser_AlignmentGolden verifies Go's ParseWithResult output is +// content-equivalent to Python's _code on the shared sample, using the shared +// concatenation-normalization alignment tool (align_test.go). Python applies +// the OVER_CAP token merge (chunking ownership retained by the Go Chunker per +// contract #17799), so item counts differ; the +// comparison normalizes both (delimiters stripped, whitespace collapsed) and +// joins on whitespace, so only CONTENT equivalence — not byte-exact layout — is +// checked. The golden files are a NORMALIZED content baseline, not a verbatim +// Python transcript: a fresh _code run at chunk_token_num=128 may merge into a +// different item count/structure (e.g. the en sample collapses to one chunk while +// the golden keeps prose and code as two items) and may collapse inter-sentence +// newlines, so do not treat them as byte-exact. +// +// No generator script is committed. The baseline meta records generator, sample, +// delimiter, keep_delimiters and chunk_token_num (see textcode.python.en/zh.golden.json); +// sample, delimiter, keep_delimiters, chunk_token_num): call the python flow +// _code on the sample with keep_delimiters=True and the default delimiter set, +// then project each merged section to {"text": section[0], "doc_type_kwd": "text"}. +func TestTextParser_AlignmentGolden(t *testing.T) { + ctx := t.Context() + p := NewTextParser() + + cases := []struct { + name string + sample string + golden string + }{ + {"en", "testdata/textcode.sample.en.txt", "testdata/textcode.python.en.golden.json"}, + {"zh", "testdata/textcode.sample.zh.txt", "testdata/textcode.python.zh.golden.json"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + sample, err := os.ReadFile(tc.sample) + if err != nil { + t.Fatalf("read sample: %v", err) + } + res := p.ParseWithResult(ctx, tc.sample, sample) + if res.Err != nil { + t.Fatalf("ParseWithResult: %v", res.Err) + } + + gd := LoadGoldenDoc(t, tc.golden) + ignore := AcceptedDivergences(gd.Meta) + + goText := FilterOutDocTypes(FilterByDocType(res.JSON, "text"), ignore) + pyText := FilterOutDocTypes(FilterByDocType(gd.Items, "text"), ignore) + + if ok, diff := CompareAlignment(goText, pyText, TextCodeAlignOptions(DefaultTextCodeDelimiter)); !ok { + t.Fatalf("text&code parser not aligned with Python golden:%s", diff) + } + }) + } +} + +// TestTextParser_AdjacentDelimiters pins Go's behavior on adjacent +// delimiters, confirming it matches Python's deepdoc TxtParser.parser_txt +// delimiter-loop exactly (not a divergence). Both ports run the same +// re.split(r"(%s)" % dels, txt) loop with keep_delimiters=True and merge a +// run of adjacent delimiters into the preceding segment, so the standalone +// second delimiter is dropped on both sides: "a!!b" → ["a!", "b"] (verified +// against deepdoc/parser/txt_parser.py). The alignment test's delimiter-strip +// normalization also reconciles this, but this test guards splitCapturingDelims +// directly so a future silent change there is caught independently. +func TestTextParser_AdjacentDelimiters(t *testing.T) { + ctx := t.Context() + p := NewTextParser() + + // Two adjacent sentence delimiters: Go (and Python's parser_txt) merge + // them into the preceding segment and drop the standalone second delimiter. + src := []byte("a!!b") + res := p.ParseWithResult(ctx, "doc.txt", src) + if res.Err != nil { + t.Fatalf("ParseWithResult: %v", res.Err) + } + want := []string{"a!", "b"} + if len(res.JSON) != len(want) { + t.Fatalf("adjacent delimiters: JSON len = %d, want %d: %#v", len(res.JSON), len(want), res.JSON) + } + for i, w := range want { + if got := res.JSON[i]["text"]; got != w { + t.Errorf("adjacent delimiters: JSON[%d].text = %v, want %v", i, got, w) + } + } + + // Delimiters separated by text each attach to their own segment (no merge + // across the gap). + src = []byte("x?y!z") + res = p.ParseWithResult(ctx, "doc.txt", src) + want = []string{"x?", "y!", "z"} + if len(res.JSON) != len(want) { + t.Fatalf("mixed delimiters: JSON len = %d, want %d: %#v", len(res.JSON), len(want), res.JSON) + } + for i, w := range want { + if got := res.JSON[i]["text"]; got != w { + t.Errorf("mixed delimiters: JSON[%d].text = %v, want %v", i, got, w) + } + } +} diff --git a/internal/parser/parser/testdata/textcode.python.en.golden.json b/internal/parser/parser/testdata/textcode.python.en.golden.json new file mode 100644 index 0000000000..6cfda25b04 --- /dev/null +++ b/internal/parser/parser/testdata/textcode.python.en.golden.json @@ -0,0 +1,23 @@ +{ + "meta": { + "generator": "rag/flow/parser/parser.py:_code", + "sample": "internal/parser/parser/testdata/textcode.sample.en.txt", + "delimiter": "\n!?;。；！？", + "chunk_token_num": 128, + "keep_delimiters": true, + "separate_tables": false, + "accepted_divergences": [], + "python_engine": "deepdoc.parser.txt_parser.RAGFlowTxtParser", + "note": "No generator script is committed. To regenerate: call _code on the sample with keep_delimiters=True (chunk_token_num=128, default delimiter set), project each merged section to {\"text\": section[0], \"doc_type_kwd\": \"text\"}, then dump {meta, items}. The golden is a NORMALIZED content baseline, not a verbatim Python transcript (see TestTextParser_AlignmentGolden): a fresh _code run at chunk_token_num=128 may merge into a different item count/structure and may collapse inter-sentence newlines, so this file is not byte-exact." + }, + "items": [ + { + "text": "RAGFlow parses plain text and source code through the text&code family. This is the first English sentence. Here is a second sentence with more detail!\n Is this a question that the parser should handle?\nA blank line separates paragraphs. Semicolons also act as delimiters;\n this clause stays attached to the previous one. The parser keeps the trailing punctuation so sentence boundaries survive the split.", + "doc_type_kwd": "text" + }, + { + "text": "def greet(name):\n\n return f\"hello, {name}\"\n\ndef main():\n\n print(greet(\"world\"))\n\nLong code lines are kept as their own segments. The parser does not perform the token merge;\n the downstream chunker owns chunking per PARSER_ALIGNMENT_HANDOFF.md section 2.3.\n", + "doc_type_kwd": "text" + } + ] +} diff --git a/internal/parser/parser/testdata/textcode.python.zh.golden.json b/internal/parser/parser/testdata/textcode.python.zh.golden.json new file mode 100644 index 0000000000..fc7b6e836b --- /dev/null +++ b/internal/parser/parser/testdata/textcode.python.zh.golden.json @@ -0,0 +1,19 @@ +{ + "meta": { + "generator": "rag/flow/parser/parser.py:_code", + "sample": "internal/parser/parser/testdata/textcode.sample.zh.txt", + "delimiter": "\n!?;。；！？", + "chunk_token_num": 128, + "keep_delimiters": true, + "separate_tables": false, + "accepted_divergences": [], + "python_engine": "deepdoc.parser.txt_parser.RAGFlowTxtParser", + "note": "No generator script is committed. To regenerate: call _code on the sample with keep_delimiters=True (chunk_token_num=128, default delimiter set), project each merged section to {\"text\": section[0], \"doc_type_kwd\": \"text\"}, then dump {meta, items}. The golden is a NORMALIZED content baseline, not a verbatim Python transcript (see TestTextParser_AlignmentGolden): a fresh _code run at chunk_token_num=128 may merge into a different item count/structure and may collapse inter-sentence newlines, so this file is not byte-exact." + }, + "items": [ + { + "text": "这是第一段中文。逗号不是分隔符，但句号是！第二句以感叹号结尾？第三句以问号结尾。中文段落同样按默认分隔符切分。", + "doc_type_kwd": "text" + } + ] +} diff --git a/internal/parser/parser/testdata/textcode.sample.en.txt b/internal/parser/parser/testdata/textcode.sample.en.txt new file mode 100644 index 0000000000..8ed59c03ee --- /dev/null +++ b/internal/parser/parser/testdata/textcode.sample.en.txt @@ -0,0 +1,11 @@ +RAGFlow parses plain text and source code through the text&code family. This is the first English sentence. Here is a second sentence with more detail! Is this a question that the parser should handle? + +A blank line separates paragraphs. Semicolons also act as delimiters; this clause stays attached to the previous one. The parser keeps the trailing punctuation so sentence boundaries survive the split. + +def greet(name): + return f"hello, {name}" + +def main(): + print(greet("world")) + +Long code lines are kept as their own segments. The parser does not perform the token merge; the downstream chunker owns chunking per PARSER_ALIGNMENT_HANDOFF.md section 2.3. diff --git a/internal/parser/parser/testdata/textcode.sample.zh.txt b/internal/parser/parser/testdata/textcode.sample.zh.txt new file mode 100644 index 0000000000..59c5857e1b --- /dev/null +++ b/internal/parser/parser/testdata/textcode.sample.zh.txt @@ -0,0 +1 @@ +这是第一段中文。逗号不是分隔符，但句号是！第二句以感叹号结尾？第三句以问号结尾。中文段落同样按默认分隔符切分。 diff --git a/internal/parser/parser/text_parser.go b/internal/parser/parser/text_parser.go index 6f2f03769f..188f4a5bfb 100644 --- a/internal/parser/parser/text_parser.go +++ b/internal/parser/parser/text_parser.go @@ -22,36 +22,31 @@ // side needs a parser for these families so `text&code` resolves to a // real ParseResultProducer. // -// TextParser fills that gap with a minimal but real implementation: -// it splits the input into paragraph-sized items and emits the -// python-compatible `{text, doc_type_kwd:"text"}` shape. The -// python TxtParser additionally does layout-aware section -// detection; the Go version is intentionally simpler because (a) -// no production template currently relies on text&code for richer -// structure than paragraph items. +// TextParser fills that gap with a real implementation: it splits the +// input into fine segments on the flow parser's default delimiter set and +// emits the python-compatible `{text, doc_type_kwd:"text"}` shape. Block +// boundaries converge to the Python flow TxtParser (which uses the same +// delimiter set); the OVER_CAP token merge that Python applies afterwards is +// intentionally NOT performed here — chunking ownership stays with the +// downstream Chunker (contract #17799), so +// item counts differ from Python's merged chunks and are reconciled by the +// stitch-compare alignment test (align_test.go). package parser import ( - "bytes" "context" + "regexp" "strings" ) // TextParser is the text&code family parser. It implements the // structured ParseResultProducer contract directly. -type TextParser struct { - // maxItemBytes caps each emitted item's text length. The - // python TxtParser uses similar paragraph-style chunking; - // 8192 bytes is a conservative ceiling that prevents the - // downstream chunker from receiving oversized inputs. - maxItemBytes int -} +type TextParser struct{} -// NewTextParser constructs a TextParser with the default -// paragraph-sized chunking ceiling. +// NewTextParser constructs a TextParser. func NewTextParser() *TextParser { - return &TextParser{maxItemBytes: 8192} + return &TextParser{} } // ParseWithResult emits one item per non-empty paragraph. The @@ -65,7 +60,7 @@ func (p *TextParser) ParseWithResult(ctx context.Context, filename string, data if !utf8Valid(data) { return ParseResult{Err: errInvalidUTF8} } - items := textParserItems(data, p.maxItemBytes) + items := textParserItems(data) if items == nil { items = []map[string]any{{"text": "", "doc_type_kwd": "text"}} } @@ -140,31 +135,108 @@ func decodeRune(p []byte) (rune, int) { return 0xFFFD, 1 } -// textParserItems splits `data` into paragraph-sized chunks. The -// split rule mirrors the python TxtParser: blank lines separate -// paragraphs; long paragraphs are sliced at maxItemBytes boundaries. -func textParserItems(data []byte, maxItemBytes int) []map[string]any { - var items []map[string]any - for _, raw := range bytes.Split(data, []byte("\n\n")) { - text := strings.TrimSpace(string(raw)) - if text == "" { +// defaultTextDelimiterPattern is the regexp alternation of the flow parser's +// default delimiter set DefaultTextCodeDelimiter (rag/flow/parser/parser.py:_code +// → deepdoc TxtParser default "\n!?;。；！？"), each rune re.escape'd to mirror +// rag/nlp/delim.compile_delimiter_pattern. The Parser component has no user-facing +// delimiter config entry, so this default +// is exactly what the python flow always splits on. The shared DefaultTextCodeDelimiter +// const lives in delimiter.go so production and the alignment tests use one source +// of truth. Go's regexp.Split drops captured delimiters, so splitCapturingDelims +// walks the match indexes manually to reproduce python's re.split(r"(%s)" % pattern, txt) +// interleaving. +var ( + defaultTextDelimiterPattern = buildDelimiterPattern(DefaultTextCodeDelimiter) + textDelimiterSplitRe = regexp.MustCompile(defaultTextDelimiterPattern) + textDelimiterExactRe = regexp.MustCompile("^(?:" + defaultTextDelimiterPattern + ")$") +) + +// buildDelimiterPattern builds an alternation of re.escape'd delimiter runes +// (longest-first is a no-op here: every delimiter in the default set is a +// single rune, so insertion order is preserved like python's stable sort). +func buildDelimiterPattern(delims string) string { + parts := make([]string, 0, len(delims)) + for _, r := range delims { + parts = append(parts, regexp.QuoteMeta(string(r))) + } + return strings.Join(parts, "|") +} + +// normalizeTextNewlines folds CRLF and standalone CR to LF, mirroring +// rag/nlp/delim.normalize_text_newlines so Windows-line-ending documents split +// identically to Unix ones. +func normalizeTextNewlines(s string) string { + if s == "" { + return s + } + s = strings.ReplaceAll(s, "\r\n", "\n") + return strings.ReplaceAll(s, "\r", "\n") +} + +// splitCapturingDelims reproduces python re.split(r"(%s)" % pattern, s): it +// splits on the regexp and includes each matched delimiter as its own element +// (with empty strings between adjacent delimiters) so callers can keep or drop +// them. Go's regexp.Split discards captured groups, hence the manual walk. +func splitCapturingDelims(s string, re *regexp.Regexp) []string { + locs := re.FindAllStringIndex(s, -1) + if len(locs) == 0 { + return []string{s} + } + out := make([]string, 0, 2*len(locs)+1) + prev := 0 + for _, loc := range locs { + out = append(out, s[prev:loc[0]]) + out = append(out, s[loc[0]:loc[1]]) + prev = loc[1] + } + out = append(out, s[prev:]) + return out +} + +// textParserItems splits data into fine segments on the flow parser's default +// delimiter set, mirroring deepdoc.parser.txt_parser.TxtParser.parser_txt up to +// (but not including) the OVER_CAP token merge. The token merge is intentionally +// NOT performed here — chunking ownership stays with the downstream Chunker per +// contract #17799 — so item counts differ from the +// python flow's merged chunks and are reconciled by the stitch-compare alignment +// test (align_test.go). +// +// Unlike the python signature default keep_delimiters=False, the flow _code +// path calls TxtParser with keep_delimiters=True, so each segment keeps its +// trailing delimiter attached (sentence-ending punctuation preserved for code +// and prose). +// +// No per-item byte cap is applied: the parser is a pure delimiter splitter, just +// like python's parser_txt (which also does no size slicing). Sizing belongs to +// the chunker and the embedding truncation step, so a continuous run longer than +// the embedding token budget (e.g. a minified / no-newline file) is kept as one +// item here and collapsed to one chunk downstream — matching python's behaviour +// rather than diverging from it. +func textParserItems(data []byte) []map[string]any { + txt := normalizeTextNewlines(string(data)) + secs := splitCapturingDelims(txt, textDelimiterSplitRe) + + var paras []string + for i, sec := range secs { + if textDelimiterExactRe.MatchString(sec) { continue } - if maxItemBytes > 0 && len(text) > maxItemBytes { - // Slice at the nearest newline below maxItemBytes; - // falls back to a hard slice when no newline exists. - cut := strings.LastIndex(text[:maxItemBytes], "\n") - if cut <= 0 { - cut = maxItemBytes - } - items = append(items, map[string]any{ - "text": strings.TrimSpace(text[:cut]), - "doc_type_kwd": "text", - }) - text = strings.TrimSpace(text[cut:]) - if text == "" { - continue - } + if sec == "" { + continue + } + // keep_delimiters=True: append the delimiter to the segment it + // follows, mirroring python's parser_txt. + if i+1 < len(secs) && textDelimiterExactRe.MatchString(secs[i+1]) { + sec += secs[i+1] + } + paras = append(paras, sec) + } + + var items []map[string]any + for _, para := range paras { + text := strings.TrimSpace(para) + if text == "" { + continue } items = append(items, map[string]any{ "text": text,