mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-08-27 10:47:44 +08:00
fix: honor parser params and image VLM system_prompt in Go ingestion (#17334)
## Summary Fix the Go ingestion pipeline so that several parser setup switches and the image VLM prompt are actually honored end-to-end (previously the DSL fields existed but the Go code never read them). - **DOCX** (`docx_parser.go`, `docx_postprocess.go`): read `remove_toc` and `remove_header_footer`; apply to both JSON and markdown output paths (outline-based TOC removal with a text-heuristic fallback, plus header/footer section filtering). - **HTML** (`html_parser.go`, `html_postprocess.go`, `text_toc.go`): read `remove_header_footer` (pre-parse strip of `<header>`/`<footer>` and ARIA `banner`/`contentinfo`) and `remove_toc` (post-parse `remove_contents_table` heuristic). - **Markdown** (`markdown_parser.go`): read `flatten_media_to_text` and force media blocks to text when enabled. - **Image VLM** (`media_dispatch.go`): read `system_prompt` instead of `prompt` so the user-configured image VLM prompt is no longer silently dropped (`prompt` remains the video family key). All flags are wired through `ConfigureFromSetup`, which the dispatch layer already invokes for every family, so the behavior is live rather than dead code. ## Test plan - New unit tests: `docx_postprocess_test.go`, `html_parser_test.go`, `text_toc_test.go`, `markdown_parser_test.go`, `media_dispatch_test.go`. - `bash build.sh --test ./internal/parser/parser/... ./internal/ingestion/component/...` ## Notes - The `File` component is excluded from this migration scope. - Relates to the Python→Go parity diff (Parser 1.8–1.11, 1.15).
This commit is contained in:
108
internal/parser/parser/text_toc_test.go
Normal file
108
internal/parser/parser/text_toc_test.go
Normal file
@@ -0,0 +1,108 @@
|
||||
package parser
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestRemoveContentsTable mirrors the Python remove_contents_table
|
||||
// (rag/nlp/__init__.py:937-965). It locates a TOC heading
|
||||
// (contents/目录/目次/...), drops it, then drops the following
|
||||
// entries that share a common prefix with the first TOC entry.
|
||||
func TestRemoveContentsTable(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
items []map[string]any
|
||||
eng bool
|
||||
want []map[string]any
|
||||
}{
|
||||
{
|
||||
name: "no TOC heading → unchanged",
|
||||
items: []map[string]any{{"text": "Intro"}, {"text": "Body"}},
|
||||
eng: false,
|
||||
want: []map[string]any{{"text": "Intro"}, {"text": "Body"}},
|
||||
},
|
||||
{
|
||||
name: "Chinese 目录 heading + first TOC entry removed",
|
||||
items: []map[string]any{
|
||||
{"text": "前言"},
|
||||
{"text": "目录"},
|
||||
{"text": "第一章 概述"},
|
||||
{"text": "第二章 方法"},
|
||||
{"text": "正文开始"},
|
||||
},
|
||||
eng: false,
|
||||
// Python remove_contents_table drops the heading and the
|
||||
// first entry after it (prefix "第一章"). Subsequent
|
||||
// entries with a different prefix ("第二章") are kept
|
||||
// because re.match("第一章", "第二章 方法") fails.
|
||||
want: []map[string]any{
|
||||
{"text": "前言"},
|
||||
{"text": "第二章 方法"},
|
||||
{"text": "正文开始"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "English Contents heading + first TOC entry removed",
|
||||
items: []map[string]any{
|
||||
{"text": "Intro"},
|
||||
{"text": "Contents"},
|
||||
{"text": "Chapter 1 Overview"},
|
||||
{"text": "Chapter 2 Method"},
|
||||
{"text": "Body starts"},
|
||||
},
|
||||
eng: true,
|
||||
want: []map[string]any{
|
||||
{"text": "Intro"},
|
||||
{"text": "Chapter 2 Method"},
|
||||
{"text": "Body starts"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "目录 heading with no following entry → heading dropped only",
|
||||
items: []map[string]any{
|
||||
{"text": "前言"},
|
||||
{"text": "目录"},
|
||||
},
|
||||
eng: false,
|
||||
want: []map[string]any{
|
||||
{"text": "前言"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "prefix-matching subsequent entries collapse non-matching gap",
|
||||
items: []map[string]any{
|
||||
{"text": "前言"},
|
||||
{"text": "目录"},
|
||||
{"text": "第一节 概述"},
|
||||
{"text": "杂项"},
|
||||
{"text": "第一节 背景"},
|
||||
{"text": "正文"},
|
||||
},
|
||||
eng: false,
|
||||
// prefix="第一节"; after dropping heading + first entry,
|
||||
// scan finds "第一节 背景" at j=2 (gap "杂项" at j=1);
|
||||
// Python deletes [i, j) = the gap, keeps the match.
|
||||
want: []map[string]any{
|
||||
{"text": "前言"},
|
||||
{"text": "第一节 背景"},
|
||||
{"text": "正文"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "empty input → unchanged",
|
||||
items: nil,
|
||||
eng: false,
|
||||
want: nil,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got := removeContentsTable(tc.items, tc.eng)
|
||||
if !reflect.DeepEqual(got, tc.want) {
|
||||
t.Errorf("got %v, want %v", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user