From d6f6b6231f9bb25f980524f4f1c7aee5576331ce Mon Sep 17 00:00:00 2001 From: Jack Date: Tue, 11 Aug 2026 10:21:42 +0800 Subject: [PATCH] Fix(parser): don't collapse markdown doc into one item when a table is present (#18014) --- internal/parser/parser/align_test.go | 237 ++++++++++++ internal/parser/parser/markdown_parser.go | 205 +++++++--- .../parser/parser/markdown_parser_test.go | 359 ++++++++++++++++-- .../parser/testdata/gen_markdown_golden.py | 84 ++++ .../testdata/markdown.python.golden.json | 50 +++ .../parser/parser/testdata/markdown.sample.md | 26 ++ 6 files changed, 885 insertions(+), 76 deletions(-) create mode 100644 internal/parser/parser/align_test.go create mode 100644 internal/parser/parser/testdata/gen_markdown_golden.py create mode 100644 internal/parser/parser/testdata/markdown.python.golden.json create mode 100644 internal/parser/parser/testdata/markdown.sample.md diff --git a/internal/parser/parser/align_test.go b/internal/parser/parser/align_test.go new file mode 100644 index 0000000000..d031f4f738 --- /dev/null +++ b/internal/parser/parser/align_test.go @@ -0,0 +1,237 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// Warranties, INCLUDING THE WARRANTIES OF MERCHANTABILITY AND +// FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. +// + +package parser + +import ( + "encoding/json" + "os" + "regexp" + "strings" + "testing" +) + +// Normalizer transforms a single item's text before comparison. Normalizers +// are composed per parser type so the same comparison core is reused across +// every format (sessions A–E of the Go↔Python parser alignment). +type Normalizer func(string) string + +// WithDelimiterStrip returns a Normalizer that replaces every rune present in +// delims with a single space. This normalizes the delimiter-split difference: +// Python splits the text at delimiters into separate items (so the delimiter +// becomes an item boundary, i.e. whitespace), while Go keeps the delimiter +// inline. Replacing with a space — rather than deleting — preserves the token +// separation on the Go side, so after CollapseWhitespace both sides yield the +// same space-joined text. +func WithDelimiterStrip(delims string) Normalizer { + return func(s string) string { + return strings.Map(func(r rune) rune { + if strings.ContainsRune(delims, r) { + return ' ' + } + return r + }, s) + } +} + +// CollapseWhitespace returns a Normalizer that trims and collapses runs of +// whitespace into a single space. Universal normalizer for tolerant compare. +func CollapseWhitespace() Normalizer { + return func(s string) string { + return strings.Join(strings.Fields(s), " ") + } +} + +// htmlTagRE matches an HTML tag so table/HTML markup can be ignored when +// comparing content across the two markdown libraries (goldmark vs +// Python-Markdown serialize tables differently but the cell text is the same). +var htmlTagRE = regexp.MustCompile(`(?is)<[^>]+>`) + +// StripHTMLTags returns a Normalizer that removes HTML tags, leaving the +// visible text. Tags are replaced with a single space (not deleted) so +// adjacent cell text does not fuse — e.g. "AB" becomes +// "A B" rather than "AB". CollapseWhitespace then folds the extra space. +// Used so table-markup differences between markdown libraries don't mask the +// underlying content equivalence. +func StripHTMLTags() Normalizer { + return func(s string) string { + return htmlTagRE.ReplaceAllString(s, " ") + } +} + +// Markdown-syntax regexes removed by StripMarkdownSyntax. The Python flow +// parser keeps raw markdown in its section text ("# Title", "- item", +// ``` fenced ```); the Go parser emits clean per-block text. These are +// representation differences (PARSER_ALIGNMENT_HANDOFF.md §3.1), not content +// divergences, so the markdown alignment strips them before comparing. +var ( + mdHeaderRE = regexp.MustCompile(`(?m)^#{1,6}\s+`) + mdListRE = regexp.MustCompile(`(?m)^\s*[-*+]\s+`) + mdFenceRE = regexp.MustCompile("(?s)```[^\n]*\n(.*?)```") +) + +// StripMarkdownSyntax returns a Normalizer that removes markdown presentation +// characters (ATX headings, list bullets, fenced-code fences) from a section, +// leaving the bare text. It must run before CollapseWhitespace because the +// fence regex relies on the surrounding newlines. +func StripMarkdownSyntax() Normalizer { + return func(s string) string { + s = mdHeaderRE.ReplaceAllString(s, "") + s = mdListRE.ReplaceAllString(s, "") + s = mdFenceRE.ReplaceAllString(s, "$1") + return s + } +} + +// FilterByDocType returns only the items whose doc_type_kwd equals kwd. +// Python emits duplicate table items (separate_tables=False still appends +// them) — excluding doc_type_kwd:"table" lets the comparison focus on the +// inlined textual content, which is what Go produces. +func FilterByDocType(items []map[string]any, kwd string) []map[string]any { + out := make([]map[string]any, 0, len(items)) + for _, it := range items { + if v, _ := it["doc_type_kwd"].(string); v == kwd { + out = append(out, it) + } + } + return out +} + +// AlignOptions configures NormalizeConcat / CompareAlignment. +type AlignOptions struct { + // Normalizers applied (in order) to each item's text before concat. + Normalizers []Normalizer + // ItemKey is the field holding the compared text (default "text"). + ItemKey string +} + +func alignItemText(item map[string]any, key string) string { + if key == "" { + key = "text" + } + if v, ok := item[key].(string); ok { + return v + } + return "" +} + +// NormalizeConcat extracts the text field from each item, applies the +// normalizers in order, and concatenates into one string. Format-agnostic. +// +// Items are joined with a single space (after whitespace is collapsed by the +// normalizers) rather than by newlines: Go emits one item per top-level block +// while Python splits the same text on delimiters into many smaller items, so +// the item *boundaries* legitimately differ. Joining on whitespace makes the +// comparison boundary-agnostic — only the concatenated content (order +// preserved) is compared, which is exactly the alignment guarantee we want. +// Empty items are skipped so Python's trailing/duplicate segments don't mask a +// real content difference. +func NormalizeConcat(items []map[string]any, opts AlignOptions) string { + key := opts.ItemKey + if key == "" { + key = "text" + } + parts := make([]string, 0, len(items)) + for _, it := range items { + t := alignItemText(it, key) + for _, n := range opts.Normalizers { + t = n(t) + } + if strings.TrimSpace(t) == "" { + continue + } + // Trim so a delimiter turned into a trailing space (WithDelimiterStrip) + // doesn't combine with the join space into a double gap. + t = strings.TrimSpace(t) + parts = append(parts, t) + } + return strings.Join(parts, " ") +} + +// CompareAlignment reports whether two parser outputs are aligned after +// normalization. goItems come from Go's ParseResult.JSON; pyItems come from the +// Python golden JSON. Returns (equal, diffReport). +func CompareAlignment(goItems, pyItems []map[string]any, opts AlignOptions) (bool, string) { + g := NormalizeConcat(goItems, opts) + p := NormalizeConcat(pyItems, opts) + if g == p { + return true, "" + } + return false, diffReport(g, p) +} + +func diffReport(g, p string) string { + const max = 2000 + if len(g) > max { + g = g[:max] + "...(truncated)" + } + if len(p) > max { + p = p[:max] + "...(truncated)" + } + return "alignment mismatch after normalization:\n--- GO ---\n" + g + "\n--- PY ---\n" + p +} + +// LoadGolden reads a Python golden JSON file (a JSON list of item objects) +// produced by the Python flow parser for the same input. +func LoadGolden(t *testing.T, path string) []map[string]any { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("load golden %s: %v", path, err) + } + var items []map[string]any + if err := json.Unmarshal(data, &items); err != nil { + t.Fatalf("parse golden %s: %v", path, err) + } + return items +} + +// MarkdownAlignOptions returns the normalizer preset for markdown. The order +// matters: +// - StripMarkdownSyntax first: drops "#"/"-"/fenced-code markup that Python +// keeps inline but Go parses out (relies on the surrounding newlines, so it +// must run before CollapseWhitespace). +// - StripHTMLTags next: replace table/HTML tags with a space (not delete) so +// adjacent cell text does not fuse, e.g. "AB" → "A B". +// - WithDelimiterStrip: replace the delimiter set Python consumes at split +// points with a space while Go keeps it inline, so both sides keep the same +// token separation. Runs before CollapseWhitespace so the introduced space +// is folded normally. +// - CollapseWhitespace last: folds all remaining internal whitespace (the +// inter-tag gaps of an HTML table, the space from delimiter replacement, +// the newlines inside a fenced code block) into single spaces. +// +// Reused by every markdown alignment test; other formats define their own +// preset and share CompareAlignment. +func MarkdownAlignOptions(delimiter string) AlignOptions { + return AlignOptions{ + Normalizers: []Normalizer{ + StripMarkdownSyntax(), + StripHTMLTags(), + WithDelimiterStrip(delimiter), + CollapseWhitespace(), + }, + ItemKey: "text", + } +} + +// DefaultMarkdownDelimiter is the flow parser's default markdown delimiter +// set, used when generating/loading the golden baseline. +const DefaultMarkdownDelimiter = "\n!?;。；！？" diff --git a/internal/parser/parser/markdown_parser.go b/internal/parser/parser/markdown_parser.go index 54f9f838f0..e41897d152 100644 --- a/internal/parser/parser/markdown_parser.go +++ b/internal/parser/parser/markdown_parser.go @@ -25,7 +25,6 @@ import ( "net" "net/http" "net/url" - "regexp" "strings" "sync" "time" @@ -35,9 +34,6 @@ import ( mdparser "github.com/gomarkdown/markdown/parser" ) -// mdImagePattern matches markdown inline image syntax: ![alt](url). -var mdImagePattern = regexp.MustCompile(`!\[[^\]]*\]\(([^)\s]+)\)`) - // dataURIPrefix is the MIME prefix for data URI images. const dataURIPrefix = "data:image/" @@ -96,20 +92,18 @@ func (p *MarkdownParser) ConfigureFromSetup(setup map[string]any) { // been removed; callers consume ParseResult directly. func (p *MarkdownParser) ParseWithResult(ctx context.Context, filename string, data []byte) ParseResult { rawText := string(data) - if rendered, ok := renderMarkdownTablesInline(rawText); ok { - return ParseResult{ - OutputFormat: "json", - File: map[string]any{ - "name": filename, - }, - JSON: []map[string]any{{"text": rendered, "doc_type_kwd": "text"}}, - } - } + // Render any GFM/HTML table inline as an HTML block before parsing. This + // keeps the document as one item per top-level block (the table becomes a + // normal text item) instead of collapsing the whole document into a single + // item. The result mirrors Python's `_markdown` (separate_tables=False), + // which also inlines tables into the surrounding text. When no table is + // present renderMarkdownTablesInlineText returns the input unchanged. + rendered := renderMarkdownTablesInlineText(rawText) - doc := markdownNew().Parse(data) + doc := markdownNew().Parse([]byte(rendered)) var items []map[string]any - walkMarkdownBlocksWithImages(doc, rawText, &items, p.FlattenMediaToText) + walkMarkdownBlocksWithImages(doc, &items, p.FlattenMediaToText) if items == nil { items = []map[string]any{{"text": "", "doc_type_kwd": "text"}} } @@ -165,8 +159,13 @@ func renderMarkdownTablesInline(text string) (string, bool) { i++ } tableHTML := markdownlib.ToHTML([]byte(strings.Join(lines[start:i], "")), markdownNew(), nil) + // Wrap the inlined HTML in blank lines so gomarkdown + // keeps it as a single HTML block (one item) instead of + // re-parsing it into scattered cell text. See + // PARSER_ALIGNMENT_HANDOFF.md §3.1 (markdown session A, 方案 Y). + ensureTrailingBlankLine(&buf) buf.WriteString(strings.TrimRight(string(tableHTML), "\r\n")) - buf.WriteByte('\n') + buf.WriteString("\n\n") changed = true continue } @@ -176,6 +175,32 @@ func renderMarkdownTablesInline(text string) (string, bool) { return buf.String(), changed } +// renderMarkdownTablesInlineText renders every GFM/HTML table inline as an +// HTML block and returns the rewritten text. When no table is present the +// input is returned unchanged. Unlike renderMarkdownTablesInline it always +// returns the full text (ignoring the changed flag) so callers can parse the +// result uniformly and emit one item per top-level block. +func renderMarkdownTablesInlineText(text string) string { + out, _ := renderMarkdownTablesInline(text) + return out +} + +// ensureTrailingBlankLine makes sure b ends with a blank line (two +// newlines) so the next block is separated from what precedes it. gomarkdown +// only treats a
as a standalone HTML block (rather than re-parsing it +// into scattered cell nodes) when it is surrounded by blank lines. +func ensureTrailingBlankLine(b *strings.Builder) { + s := b.String() + switch { + case strings.HasSuffix(s, "\n\n"): + // already separated. + case strings.HasSuffix(s, "\n"): + b.WriteByte('\n') + default: + b.WriteString("\n\n") + } +} + func markdownFenceMarker(line string) (byte, int, bool) { trimmed := strings.TrimLeft(line, " \t") if len(line)-len(trimmed) > 3 || len(trimmed) < 3 { @@ -236,11 +261,25 @@ func markdownTableCells(line string) []string { // top-level block. Headings, paragraphs, lists, and code blocks are // emitted with their text. When a block contains a markdown image // reference (![alt](src)), the image data is resolved via -// resolveMarkdownImage and the item carries `doc_type_kwd: "image"` -// together with the base64-encoded image payload. When flatten is -// true, all items are forced to doc_type_kwd="text" (mirrors Python -// parser.py:1034 flatten_media_to_text). -func walkMarkdownBlocksWithImages(doc ast.Node, rawText string, out *[]map[string]any, flatten bool) { +// findBlockImage (per-block AST walk) and the item carries +// `doc_type_kwd: "image"` together with the base64-encoded image +// payload. When flatten is true, all items are forced to +// doc_type_kwd="text" (mirrors Python parser.py:1034 +// flatten_media_to_text). +// +// Tables: a GFM/HTML table is rendered inline as a single
HTML +// block by renderMarkdownTablesInlineText and kept as one HTML block. +// It is emitted as TWO items, mirroring Python's _markdown +// (separate_tables=False): an inlined copy in the text flow +// (doc_type_kwd:"text") and a separate structured table item +// (doc_type_kwd:"table", ck_type:"table"). The downstream chunker +// consumes doc_type_kwd:"table" to keep the table whole and attach +// table context to neighbouring chunks (chunker/token.go). Non-table +// HTML blocks (
, \n\nAfter.\n" + res := p.ParseWithResult(ctx, "test.md", []byte(md)) + if res.Err != nil { + t.Fatalf("ParseWithResult: %v", res.Err) + } + for _, item := range res.JSON { + text, _ := item["text"].(string) + if strings.Contains(text, " HTML, corrupting the code. renderMarkdownTablesInline +// tracks fence state (inFence) so the table detector must skip lines inside a fence. +func TestMarkdownParser_TableInCodeFenceNotRendered(t *testing.T) { + ctx := t.Context() + p, _ := NewMarkdownParser(GoMarkdown) + md := "# Title\n\n```\n| A | B |\n| --- | --- |\n| x | y |\n```\n\nAfter fence.\n" + res := p.ParseWithResult(ctx, "test.md", []byte(md)) + if res.Err != nil { + t.Fatalf("ParseWithResult: %v", res.Err) + } + // No table item may be emitted — the pipe rows live inside a code block. + for _, item := range res.JSON { + if kd, _ := item["doc_type_kwd"].(string); kd == "table" { + t.Fatalf("code-fence pipe rows wrongly emitted as table item: %q", item["text"]) + } + } + // The code block item must retain the raw pipe text and must NOT contain + // any
markup. + sawCode := false + for _, item := range res.JSON { + text, _ := item["text"].(string) + if strings.Contains(text, "| A | B |") { + sawCode = true + if strings.Contains(text, " HTML +// block (not a GFM pipe table). renderMarkdownTablesInline only rewrites GFM +// pipe tables, so the raw
passes through and is caught by isTableHTML +// as an HTMLBlock, producing the same inlined copy + separate doc_type_kwd:"table" +// item shape as a GFM table. +func TestMarkdownParser_RawHTMLTableHandled(t *testing.T) { + ctx := t.Context() + p, _ := NewMarkdownParser(GoMarkdown) + md := "Before.\n\n
XY
\n\nAfter.\n" + res := p.ParseWithResult(ctx, "test.md", []byte(md)) + if res.Err != nil { + t.Fatalf("ParseWithResult: %v", res.Err) + } + sawInlined, sawTableItem := false, false + for _, item := range res.JSON { + text, _ := item["text"].(string) + switch kd, _ := item["doc_type_kwd"].(string); kd { + case "text": + if strings.Contains(text, " HTML: %q", text) + } + if ck, _ := item["ck_type"].(string); ck != "table" { + t.Fatalf("raw table item ck_type = %q, want \"table\"", ck) + } + } + } + if !sawInlined { + t.Fatal("raw HTML block not emitted as inlined copy in text flow") + } + if !sawTableItem { + t.Fatal("raw
HTML block not emitted as separate doc_type_kwd:\"table\" item") + } +} + +// TestMarkdownParser_MultipleTablesOrdering guards the ordering contract: each +// GFM table emits an inlined copy in its original document position (among the +// surrounding text blocks) and a separate doc_type_kwd:"table" item appended at +// the end of the stream (mirroring Python's _markdown, which appends tables +// after all sections). Both tables' cell text must be present and in source order. +func TestMarkdownParser_MultipleTablesOrdering(t *testing.T) { + ctx := t.Context() + p, _ := NewMarkdownParser(GoMarkdown) + md := "# Title\n\n| A | B |\n| --- | --- |\n| x | y |\n\nMiddle.\n\n| C | D |\n| --- | --- |\n| p | q |\n\nEnd.\n" + res := p.ParseWithResult(ctx, "test.md", []byte(md)) + if res.Err != nil { + t.Fatalf("ParseWithResult: %v", res.Err) + } + + var inlinedTableIdx []int + var tableItemIdx []int + var titleIdx, middleIdx, endIdx = -1, -1, -1 + for i, item := range res.JSON { + text, _ := item["text"].(string) + switch kd, _ := item["doc_type_kwd"].(string); kd { + case "text": + switch text { + case "Title": + titleIdx = i + case "Middle.": + middleIdx = i + case "End.": + endIdx = i + } + if strings.Contains(text, " endIdx && tableItemIdx[0] < tableItemIdx[1]) { + t.Fatalf("table items not appended after text items in source order: %v end=%d", tableItemIdx, endIdx) + } + t1, _ := res.JSON[tableItemIdx[0]]["text"].(string) + t2, _ := res.JSON[tableItemIdx[1]]["text"].(string) + if !strings.Contains(t1, "x") || !strings.Contains(t1, "y") { + t.Fatalf("first table item missing x/y cells: %q", t1) + } + if !strings.Contains(t2, "p") || !strings.Contains(t2, "q") { + t.Fatalf("second table item missing p/q cells: %q", t2) + } +} + +// TestMarkdownParser_AlignmentGolden verifies Go's ParseWithResult output is +// content-equivalent to Python's _markdown on the shared sample, using the +// shared concatenation-normalization alignment tool (align_test.go). Python +// keeps raw markdown and splits on the delimiter set; Go emits clean per-block +// text. The comparison normalizes both (markdown syntax, html tags, delimiters +// stripped; whitespace collapsed) and ignores "table"/"image" items, which are +// accepted representation differences (PARSER_ALIGNMENT_HANDOFF.md §3.1). +// +// Regenerate the baseline with: +// +// .venv/bin/python internal/parser/parser/testdata/gen_markdown_golden.py +func TestMarkdownParser_AlignmentGolden(t *testing.T) { + ctx := t.Context() + p, _ := NewMarkdownParser(GoMarkdown) + + sample, err := os.ReadFile("testdata/markdown.sample.md") + if err != nil { + t.Fatalf("read sample: %v", err) + } + res := p.ParseWithResult(ctx, "markdown.sample.md", sample) + if res.Err != nil { + t.Fatalf("ParseWithResult: %v", res.Err) + } + + golden := LoadGolden(t, "testdata/markdown.python.golden.json") + + // Ignore "table"/"image" items on both sides (accepted divergences). + goText := FilterByDocType(res.JSON, "text") + pyText := FilterByDocType(golden, "text") + + if ok, diff := CompareAlignment(goText, pyText, MarkdownAlignOptions(DefaultMarkdownDelimiter)); !ok { + t.Fatalf("markdown parser not aligned with Python golden:%s", diff) + } +} diff --git a/internal/parser/parser/testdata/gen_markdown_golden.py b/internal/parser/parser/testdata/gen_markdown_golden.py new file mode 100644 index 0000000000..fc0c113812 --- /dev/null +++ b/internal/parser/parser/testdata/gen_markdown_golden.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +"""Regenerate internal/parser/parser/testdata/markdown.python.golden.json. + +Drives the REAL Python markdown parser (deepdoc.parser.markdown_parser, the +same engine rag/flow/parser/parser.py:_markdown delegates to via +rag/app/naive.Markdown) so the golden is a faithful baseline rather than a +hand approximation. + +Requires the project virtualenv (uv) because deepdoc needs markdown / +beartype / etc.: + + .venv/bin/python internal/parser/parser/testdata/gen_markdown_golden.py + +It mirrors _markdown with separate_tables=False and the default delimiter +set, then assembles json items exactly as _markdown does: + + * each extracted section -> {"text": , "doc_type_kwd": "text"} + * each standalone table -> {"text": , "doc_type_kwd": "table"} + * an image section -> {"text": , "doc_type_kwd": "image"} + +The Go alignment test then strips markdown syntax, html tags, and delimiters +before comparing, and ignores "table"/"image" items (those representations are +accepted divergences per PARSER_ALIGNMENT_HANDOFF.md §3.1). +""" + +import json +import os +import re +import sys + +# Make the repo root importable when run as a standalone script from testdata. +# Script lives at /internal/parser/parser/testdata/, so five dirname hops. +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))))) + +SAMPLE = "internal/parser/parser/testdata/markdown.sample.md" +OUT = "internal/parser/parser/testdata/markdown.python.golden.json" +DELIM = "\n!?;。；！？" +IMG_RE = re.compile(r"!\[([^\]]*)\]\(([^)]+)\)") +SENTINEL = "@@IMAGE@@" + + +def main(): + from deepdoc.parser.markdown_parser import RAGFlowMarkdownParser, MarkdownElementExtractor + + with open(SAMPLE, encoding="utf-8") as f: + raw = f.read() + + # Model _markdown's return_section_images: the image is extracted as its + # own item (alt text only). Replace the markdown with a delimiter-free + # sentinel so the extractor does not split it. + alts = [] + + def _repl(m): + alts.append(m.group(1)) + return SENTINEL + + prepared = IMG_RE.sub(_repl, raw) + + parser = RAGFlowMarkdownParser() + remainder, tables = parser.extract_tables_and_remainder(prepared + "\n", separate_tables=False) + extractor = MarkdownElementExtractor(remainder) + sections = extractor.extract_elements(DELIM, include_meta=True) + + items = [] + for s in sections: + content = s["content"] + if SENTINEL in content: + items.append({"text": alts.pop(0), "doc_type_kwd": "image"}) + continue + items.append({"text": content, "doc_type_kwd": "text"}) + + for tbl in tables: + # _markdown (rag/flow/parser/parser.py:1103-1111) appends each + # extracted table as a duplicate "table" item even when inlined, + # carrying the table's raw text (GFM source or raw
HTML). + items.append({"text": tbl, "doc_type_kwd": "table"}) + + with open(OUT, "w", encoding="utf-8") as f: + json.dump(items, f, ensure_ascii=False, indent=2) + print("wrote %d items to %s" % (len(items), OUT)) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/internal/parser/parser/testdata/markdown.python.golden.json b/internal/parser/parser/testdata/markdown.python.golden.json new file mode 100644 index 0000000000..73f5168110 --- /dev/null +++ b/internal/parser/parser/testdata/markdown.python.golden.json @@ -0,0 +1,50 @@ +[ + { + "text": "# 健康检查套餐对比\n本文比较两种体检套餐，包含表格、列表与代码块", + "doc_type_kwd": "text" + }, + { + "text": "## 套餐明细", + "doc_type_kwd": "text" + }, + { + "text": "
\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
检查项目基础版 699 元进阶版 1299 元
血常规 / 尿常规包含包含
心电图不包含包含
", + "doc_type_kwd": "text" + }, + { + "text": "注意：所有套餐均需空腹", + "doc_type_kwd": "text" + }, + { + "text": "## 注意事项", + "doc_type_kwd": "text" + }, + { + "text": "- 体检前三天清淡饮食", + "doc_type_kwd": "text" + }, + { + "text": "- 避免剧烈运动", + "doc_type_kwd": "text" + }, + { + "text": "下面是示例配置：", + "doc_type_kwd": "text" + }, + { + "text": "```yaml\nname: health-check\nversion: 1\n```", + "doc_type_kwd": "text" + }, + { + "text": "示意图", + "doc_type_kwd": "image" + }, + { + "text": "\n| 检查项目 | 基础版 699 元 | 进阶版 1299 元 |\n| --- | --- | --- |\n| 血常规 / 尿常规 | 包含 | 包含 |\n| 心电图 | 不包含 | 包含 |\n", + "doc_type_kwd": "table" + }, + { + "text": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
检查项目基础版 699 元进阶版 1299 元
血常规 / 尿常规包含包含
心电图不包含包含
\n", + "doc_type_kwd": "table" + } +] diff --git a/internal/parser/parser/testdata/markdown.sample.md b/internal/parser/parser/testdata/markdown.sample.md new file mode 100644 index 0000000000..7aec22ee26 --- /dev/null +++ b/internal/parser/parser/testdata/markdown.sample.md @@ -0,0 +1,26 @@ +# 健康检查套餐对比 + +本文比较两种体检套餐，包含表格、列表与代码块。 + +## 套餐明细 + +| 检查项目 | 基础版 699 元 | 进阶版 1299 元 | +| --- | --- | --- | +| 血常规 / 尿常规 | 包含 | 包含 | +| 心电图 | 不包含 | 包含 | + +注意：所有套餐均需空腹。 + +## 注意事项 + +- 体检前三天清淡饮食。 +- 避免剧烈运动！ + +下面是示例配置： + +```yaml +name: health-check +version: 1 +``` + +![示意图](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+M8AAAMBAQDJ/pLvAAAAAElFTkSuQmCC)