mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-08-22 16:23:12 +08:00
fix: honor parser params and image VLM system_prompt in Go ingestion (#17334)
## Summary Fix the Go ingestion pipeline so that several parser setup switches and the image VLM prompt are actually honored end-to-end (previously the DSL fields existed but the Go code never read them). - **DOCX** (`docx_parser.go`, `docx_postprocess.go`): read `remove_toc` and `remove_header_footer`; apply to both JSON and markdown output paths (outline-based TOC removal with a text-heuristic fallback, plus header/footer section filtering). - **HTML** (`html_parser.go`, `html_postprocess.go`, `text_toc.go`): read `remove_header_footer` (pre-parse strip of `<header>`/`<footer>` and ARIA `banner`/`contentinfo`) and `remove_toc` (post-parse `remove_contents_table` heuristic). - **Markdown** (`markdown_parser.go`): read `flatten_media_to_text` and force media blocks to text when enabled. - **Image VLM** (`media_dispatch.go`): read `system_prompt` instead of `prompt` so the user-configured image VLM prompt is no longer silently dropped (`prompt` remains the video family key). All flags are wired through `ConfigureFromSetup`, which the dispatch layer already invokes for every family, so the behavior is live rather than dead code. ## Test plan - New unit tests: `docx_postprocess_test.go`, `html_parser_test.go`, `text_toc_test.go`, `markdown_parser_test.go`, `media_dispatch_test.go`. - `bash build.sh --test ./internal/parser/parser/... ./internal/ingestion/component/...` ## Notes - The `File` component is excluded from this migration scope. - Relates to the Python→Go parity diff (Parser 1.8–1.11, 1.15).
This commit is contained in:
107
internal/parser/parser/text_toc.go
Normal file
107
internal/parser/parser/text_toc.go
Normal file
@@ -0,0 +1,107 @@
|
||||
package parser
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// tocHeadingPattern matches TOC heading text. Mirrors Python
|
||||
// rag/nlp/__init__.py:945 — case-insensitive match against
|
||||
// contents/目录/目次/table of contents/致谢/acknowledge after
|
||||
// stripping all spaces (ASCII + fullwidth).
|
||||
var tocHeadingPattern = regexp.MustCompile(`(?i)^(contents|目录|目次|table of contents|致谢|acknowledge)$`)
|
||||
|
||||
// whitespacePattern matches ASCII spaces and fullwidth space (U+3000)
|
||||
// for normalization before heading match.
|
||||
var whitespacePattern = regexp.MustCompile("[ \t\u3000]+")
|
||||
|
||||
// itemText extracts the text field from a parser item map. Falls
|
||||
// back to empty string when the field is missing or not a string.
|
||||
func itemText(item map[string]any) string {
|
||||
if s, ok := item["text"].(string); ok {
|
||||
return strings.TrimSpace(s)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// removeContentsTable drops the table-of-contents entries from a list
|
||||
// of parser items. Mirrors Python rag/nlp/__init__.py:937-965
|
||||
// (remove_contents_table).
|
||||
//
|
||||
// Algorithm:
|
||||
// 1. Scan for a TOC heading (contents/目录/目次/...).
|
||||
// 2. Drop the heading.
|
||||
// 3. Take the next entry's prefix (first 3 chars for CJK, first 2
|
||||
// words for English) as the TOC entry pattern.
|
||||
// 4. Drop that entry, then drop up to 128 following entries that
|
||||
// start with the prefix.
|
||||
// 5. Non-matching entries are kept.
|
||||
func removeContentsTable(items []map[string]any, eng bool) []map[string]any {
|
||||
i := 0
|
||||
for i < len(items) {
|
||||
text := itemText(items[i])
|
||||
// Strip @@ suffix and whitespace before matching.
|
||||
normalized := whitespacePattern.ReplaceAllString(strings.SplitN(text, "@@", 2)[0], "")
|
||||
if !tocHeadingPattern.MatchString(normalized) {
|
||||
i++
|
||||
continue
|
||||
}
|
||||
// Drop the TOC heading.
|
||||
items = append(items[:i], items[i+1:]...)
|
||||
if i >= len(items) {
|
||||
break
|
||||
}
|
||||
// Determine the prefix from the entry right after the heading.
|
||||
prefix := tocPrefix(itemText(items[i]), eng)
|
||||
for prefix == "" {
|
||||
items = append(items[:i], items[i+1:]...)
|
||||
if i >= len(items) {
|
||||
break
|
||||
}
|
||||
prefix = tocPrefix(itemText(items[i]), eng)
|
||||
}
|
||||
if i >= len(items) || prefix == "" {
|
||||
break
|
||||
}
|
||||
// Drop the first TOC entry (the one that supplied the prefix).
|
||||
items = append(items[:i], items[i+1:]...)
|
||||
if i >= len(items) {
|
||||
break
|
||||
}
|
||||
// Drop up to 128 following entries that start with prefix.
|
||||
limit := i + 128
|
||||
if limit > len(items) {
|
||||
limit = len(items)
|
||||
}
|
||||
for j := i; j < limit; j++ {
|
||||
if !strings.HasPrefix(itemText(items[j]), prefix) {
|
||||
continue
|
||||
}
|
||||
// Drop entries [i, j).
|
||||
items = append(items[:i], items[j:]...)
|
||||
break
|
||||
}
|
||||
}
|
||||
return items
|
||||
}
|
||||
|
||||
// tocPrefix returns the TOC-entry prefix: first 3 chars for CJK,
|
||||
// first 2 whitespace-separated words for English. Mirrors Python
|
||||
// remove_contents_table line 951.
|
||||
func tocPrefix(text string, eng bool) string {
|
||||
if eng {
|
||||
words := strings.Fields(text)
|
||||
if len(words) >= 2 {
|
||||
return words[0] + " " + words[1]
|
||||
}
|
||||
if len(words) == 1 {
|
||||
return words[0]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
runes := []rune(text)
|
||||
if len(runes) < 3 {
|
||||
return string(runes)
|
||||
}
|
||||
return string(runes[:3])
|
||||
}
|
||||
Reference in New Issue
Block a user