diff --git a/internal/deepdoc/parser/pdf/batch_smoke_test.go b/internal/deepdoc/parser/pdf/batch_smoke_test.go index 6b8678d7bc..e25e6f99d5 100644 --- a/internal/deepdoc/parser/pdf/batch_smoke_test.go +++ b/internal/deepdoc/parser/pdf/batch_smoke_test.go @@ -21,6 +21,7 @@ import ( inf "ragflow/internal/deepdoc/parser/pdf/inference" lyt "ragflow/internal/deepdoc/parser/pdf/layout" + "ragflow/internal/deepdoc/parser/pdf/table" "ragflow/internal/deepdoc/parser/pdf/tool" pdf "ragflow/internal/deepdoc/parser/pdf/type" ) @@ -91,7 +92,7 @@ func variantFromEnv() string { } type outputDirs struct { - text, tables, dla string + text, tables, dla, tsr string } func mkOutputDirs(variant string) outputDirs { @@ -99,10 +100,12 @@ func mkOutputDirs(variant string) outputDirs { text: filepath.Join("testdata", "output", "go", variant, "text"), tables: filepath.Join("testdata", "output", "go", variant, "tables"), dla: filepath.Join("testdata", "output", "go", variant, "dla"), + tsr: filepath.Join("testdata", "output", "go", variant, "tsr_raw"), } os.MkdirAll(d.text, 0755) os.MkdirAll(d.tables, 0755) os.MkdirAll(d.dla, 0755) + os.MkdirAll(d.tsr, 0755) return d } @@ -344,9 +347,68 @@ func writeOutputs(dirs outputDirs, name string, parsed *pdf.ParseResult, res *pa } // ── DLA layout intermediates ── + // DLA dump: post-filter regions (NMS + confidence filter + Y-sort + + // cleanup), matching Python's page_layout. This makes the Go dump + // comparable with the Python parity dump, which also writes post-filter + // regions. + // + // Coordinate space: Go dumps image-pixel coordinates (no scale division; + // see FilteredDLARegions), whereas Python's page_layout divides by + // scale_factor (typically 3, layout_recognizer.py:90-93). So the two + // dumps differ by ~3x in absolute coordinates — this is expected, and the + // parity comparison is count-only (CompareDLAWithPython), so it does not + // affect the match. + // + // Real DLA regions always carry a score, so cleanupLayouts' box-fallback + // never triggers and nil boxes are safe here (count-wise). If a score-0 + // region were ever emitted, the area tie-break would fall back to keeping + // the first region rather than mirroring Python's area-based choice. if parsed.DLARegions != nil { - if b, _ := json.MarshalIndent(parsed.DLARegions, "", " "); b != nil { + filteredPages := make([]pdf.DLAPageRegions, 0, len(parsed.DLARegions)) + for _, pr := range parsed.DLARegions { + filteredPages = append(filteredPages, pdf.DLAPageRegions{ + Page: pr.Page, + Regions: table.FilteredDLARegions(pr.Regions, nil), + }) + } + if b, _ := json.MarshalIndent(filteredPages, "", " "); b != nil { os.WriteFile(filepath.Join(dirs.dla, name+".json"), b, 0644) } } + + // ── TSR raw cells ── (matching Python's tsr_raw dump from parser.tb_cpns). + // Flat list of per-cell records so CompareTSRRawWithPython can diff labels + // and table counts against the Python reference. + type tsrRawCellDump struct { + TableIndex int `json:"table_index"` + Page int `json:"page"` + Label string `json:"label"` + X0 float64 `json:"x0"` + Y0 float64 `json:"y0"` + X1 float64 `json:"x1"` + Y1 float64 `json:"y1"` + Text string `json:"text"` + } + var tsrCells []tsrRawCellDump + for ti, t := range parsed.Tables { + page := 0 + if len(t.Positions) > 0 && len(t.Positions[0].PageNumbers) > 0 { + page = t.Positions[0].PageNumbers[0] + } + for _, c := range t.Cells { + tsrCells = append(tsrCells, tsrRawCellDump{ + TableIndex: ti, + Page: page, + Label: c.Label, + X0: c.X0, + Y0: c.Y0, + X1: c.X1, + Y1: c.Y1, + Text: c.Text, + }) + } + } + if b, _ := json.MarshalIndent(tsrCells, "", " "); b != nil { + os.WriteFile(filepath.Join(dirs.tsr, name+".json"), b, 0644) + } } diff --git a/internal/deepdoc/parser/pdf/table/table_annotate.go b/internal/deepdoc/parser/pdf/table/table_annotate.go index 87d3719cd6..7335e61ca0 100644 --- a/internal/deepdoc/parser/pdf/table/table_annotate.go +++ b/internal/deepdoc/parser/pdf/table/table_annotate.go @@ -3,6 +3,7 @@ package table import ( "fmt" "math" + "sort" pdf "ragflow/internal/deepdoc/parser/pdf/type" "ragflow/internal/deepdoc/parser/pdf/util" @@ -57,63 +58,300 @@ func MatchTableRegions(boxes []pdf.TextBox, regions []pdf.DLARegion, scale float // ── layout annotation ────────────────────────────────────────────────── -// annotateBoxLayouts sets LayoutType and LayoutNo on each box, matching +// annRegion is a layout region in PDF space, used internally by +// AnnotateBoxLayouts. It carries the fields needed for cleanup, sort, and +// annotation bookkeeping. +type annRegion struct { + x0, y0, x1, y1 float64 + label string + score float64 + visited bool + typeIndex int +} + +// regionIntersect returns the intersection area of two regions, or 0. +func regionIntersect(a, b annRegion) float64 { + ix0 := math.Max(a.x0, b.x0) + iy0 := math.Max(a.y0, b.y0) + ix1 := math.Min(a.x1, b.x1) + iy1 := math.Min(a.y1, b.y1) + if ix0 < ix1 && iy0 < iy1 { + return (ix1 - ix0) * (iy1 - iy0) + } + return 0 +} + +// regionArea returns the area of a region, or 0 if degenerate. +func regionArea(a annRegion) float64 { + w := a.x1 - a.x0 + h := a.y1 - a.y0 + if w <= 0 || h <= 0 { + return 0 + } + return w * h +} + +// overlapRatio returns intersection / area(a), matching +// Recognizer.overlapped_area with ratio=True (recognizer.py:106-122). +func overlapRatio(a, b annRegion) float64 { + ar := regionArea(a) + if ar <= 0 { + return 0 + } + return regionIntersect(a, b) / ar +} + +// annNotOverlapped mirrors recognizer.py:126-127 (annRegion variant). +func annNotOverlapped(a, b annRegion) bool { + return a.x1 < b.x0 || a.x0 > b.x1 || a.y1 < b.y0 || a.y0 > b.y1 +} + +func imin(a, b int) int { + if a < b { + return a + } + return b +} + +// sortYFirstly orders regions top-to-bottom (and left-to-right within a +// vertical threshold), matching Recognizer.sort_Y_firstly (recognizer.py:54) +// which LayoutRecognizer.__call__ applies before annotation +// (layout_recognizer.py:99). +func sortYFirstly(regs []annRegion) { + if len(regs) == 0 { + return + } + avgH := 0.0 + for _, r := range regs { + avgH += r.y1 - r.y0 + } + thr := avgH / float64(len(regs)) / 2 + sort.SliceStable(regs, func(i, j int) bool { + di := regs[i].y0 - regs[j].y0 + if di < -thr { + return true + } + if di > thr { + return false + } + return regs[i].x0 < regs[j].x0 + }) +} + +// cleanupLayouts de-duplicates overlapping same-type regions, matching +// Recognizer.layouts_cleanup (recognizer.py:124) called by +// LayoutRecognizer.__call__ at layout_recognizer.py:100. A pair of same-type +// regions whose overlap exceeds thr (0.7) in either direction collapses to a +// single region: the higher-score one, or - when scores are absent - the one +// covering more text-box area. far=2 limits comparison to nearby regions. +func cleanupLayouts(regs []annRegion, boxes []pdf.TextBox) []annRegion { + const far = 2 + const thr = 0.7 + i := 0 + for i+1 < len(regs) { + j := i + 1 + for j < imin(i+far, len(regs)) && (regs[j].label != regs[i].label || annNotOverlapped(regs[i], regs[j])) { + j++ + } + if j >= imin(i+far, len(regs)) { + i++ + continue + } + if overlapRatio(regs[i], regs[j]) < thr && overlapRatio(regs[j], regs[i]) < thr { + i++ + continue + } + // Collapse the pair. Python layouts_cleanup keeps the HIGHER-score + // region; on equal scores it keeps the later one (j) via pop(i). + // Match that exactly so equal-confidence pairs converge with Python. + drop := j + if regs[i].score > 0 && regs[j].score > 0 { + if regs[i].score > regs[j].score { + drop = j + } else { + drop = i + } + } else { + areaI, areaJ := 0.0, 0.0 + for _, b := range boxes { + tb := annRegion{x0: b.X0, y0: b.Top, x1: b.X1, y1: b.Bottom} + if !annNotOverlapped(tb, regs[i]) { + areaI += regionIntersect(tb, regs[i]) + } + if !annNotOverlapped(tb, regs[j]) { + areaJ += regionIntersect(tb, regs[j]) + } + } + if areaJ > areaI { + drop = i + } + } + regs = append(regs[:drop], regs[drop+1:]...) + } + return regs +} + +// nmsDLARegions applies per-class non-maximum suppression to raw DLA regions, +// mirroring Python's layout model postprocess (layout_recognizer.py:246, +// operators.py:667 nms with iou_thresh). For each label, detections are sorted +// by confidence descending; the top one is kept and any other same-label +// detection whose IoU (using the +1 overlap convention from operators.py:685) +// exceeds iouThresh is suppressed. This runs on the raw, pre-scale detections +// just as Python's postprocess does, before cleanup/annotation. +// +// It is idempotent with a server-side NMS: applying it again to already-suppressed +// boxes yields the same set, so it safely converges Go to Python regardless of +// where suppression happens upstream. +func nmsDLARegions(regions []pdf.DLARegion, iouThresh float64) []pdf.DLARegion { + if len(regions) == 0 { + return regions + } + byLabel := map[string][]int{} + for i, r := range regions { + byLabel[r.Label] = append(byLabel[r.Label], i) + } + suppressed := make([]bool, len(regions)) + for _, idxs := range byLabel { + // Highest confidence first (greedy NMS keeps the top box). On equal + // confidence, break the tie by original index so the result is + // deterministic for identical inputs — otherwise the same PDF could + // keep a different region (and emit different LayoutNo) on each run. + sort.SliceStable(idxs, func(a, b int) bool { + ca, cb := regions[idxs[a]].Confidence, regions[idxs[b]].Confidence + if ca != cb { + return ca > cb + } + return idxs[a] < idxs[b] + }) + for k := 0; k < len(idxs); k++ { + i := idxs[k] + if suppressed[i] { + continue + } + for m := k + 1; m < len(idxs); m++ { + j := idxs[m] + if suppressed[j] { + continue + } + if nmsIoU(regions[i], regions[j]) > iouThresh { + suppressed[j] = true + } + } + } + } + out := make([]pdf.DLARegion, 0, len(regions)) + for i, r := range regions { + if !suppressed[i] { + out = append(out, r) + } + } + return out +} + +// nmsIoU computes IoU using the +1 overlap convention from operators.py:685-688 +// (w = max(0, x22-x11+1), h = max(0, y22-y11+1)) while area uses no +1. This +// must match Python exactly so borderline suppressions (around the 0.45 threshold) +// align. +func nmsIoU(a, b pdf.DLARegion) float64 { + w := math.Max(0, math.Min(a.X1, b.X1)-math.Max(a.X0, b.X0)+1) + h := math.Max(0, math.Min(a.Y1, b.Y1)-math.Max(a.Y0, b.Y0)+1) + inter := w * h + areaA := (a.X1 - a.X0) * (a.Y1 - a.Y0) + areaB := (b.X1 - b.X0) * (b.Y1 - b.Y0) + if areaA <= 0 || areaB <= 0 { + return 0 + } + return inter / (areaA + areaB - inter) +} + +// AnnotateBoxLayouts sets LayoutType and LayoutNo on each box, matching // Python's LayoutRecognizer.__call__ which assigns layout types in priority -// order (footer→header→…→equation) with an overlap threshold of 40% of the +// order (footer->header->...->equation) with an overlap threshold of 40% of the // box's area. // -// Python: _layouts_rec (pdf_parser.py:827) → LayoutRecognizer.__call__ → +// Python: _layouts_rec (pdf_parser.py:827) -> LayoutRecognizer.__call__ -> // // for lt in priority_order: findLayout(lt) // // Each findLayout(ty): for each unannotated box, find the DLA region of -// type ty with max overlap ≥ 0.4 × box_area. First type to match wins. +// type ty with max overlap >= 0.4 * box_area. First type to match wins. // // CID-pattern boxes (e.g. "(cid:123)") are skipped as garbage. -// annotateBoxLayouts assigns LayoutType and LayoutNo to boxes based on DLA +// AnnotateBoxLayouts assigns LayoutType and LayoutNo to boxes based on DLA // regions. Returns the filtered slice (Python pops CID-garbled boxes and -// garbage-layout boxes at wrong positions — Go mirrors with compact). +// garbage-layout boxes at wrong positions - Go mirrors with compact). // Also creates synthetic figure boxes for unmatched figure/equation regions. -func AnnotateBoxLayouts(boxes []pdf.TextBox, regions []pdf.DLARegion, scale float64, pageImgHeight float64) []pdf.TextBox { +// +// Before annotation, regions are de-duplicated (layouts_cleanup) and sorted +// top-to-bottom (sort_Y_firstly) to match Python, and unmatched figure and +// equation regions receive SEPARATE synthetic namespaces (figure-N / +// equation-N) so they never collide. +// FilteredDLARegions returns the DLA regions after per-class NMS, the +// confidence filter, Y-sort, and cleanup — i.e. the exact set fed to +// annotation. It mirrors Python's page_layout (layout_recognizer.py:84-100): +// - nmsDLARegions(0.45) == layout model postprocess (operators.py:667) +// - keep if score >= 0.4 OR type not garbage == layout_recognizer.py:97 +// - sortYFirstly == layout_recognizer.py:99 (sort_Y_firstly) +// - cleanupLayouts == layout_recognizer.py:100 (layouts_cleanup) +// +// Regions are returned in image-pixel space (no scale division) so callers +// that only need the region set — e.g. the parity harness dumping post-filter +// regions for comparison with Python's page_layout — get a stable comparison +// point regardless of render DPI. cleanupLayouts only consults boxes when both +// compared regions have score 0, which never happens for real DLA output, so +// passing nil boxes from the harness is safe. +func FilteredDLARegions(regions []pdf.DLARegion, boxes []pdf.TextBox) []pdf.DLARegion { + regions = nmsDLARegions(regions, 0.45) if len(regions) == 0 { + return nil + } + kept := regions[:0] + for _, r := range regions { + if r.Confidence >= 0.4 || !isGarbageLayoutType(r.Label) { + kept = append(kept, r) + } + } + ars := make([]annRegion, len(kept)) + for i, r := range kept { + ars[i] = annRegion{x0: r.X0, y0: r.Y0, x1: r.X1, y1: r.Y1, label: r.Label, score: r.Confidence} + } + sortYFirstly(ars) + ars = cleanupLayouts(ars, boxes) + out := make([]pdf.DLARegion, len(ars)) + for i, a := range ars { + out[i] = pdf.DLARegion{X0: a.x0, Y0: a.y0, X1: a.x1, Y1: a.y1, Label: a.label, Confidence: a.score} + } + return out +} + +func AnnotateBoxLayouts(boxes []pdf.TextBox, regions []pdf.DLARegion, scale float64, pageImgHeight float64) []pdf.TextBox { + // NMS, confidence filter, Y-sort, and cleanup — the exact region set fed + // to annotation. This mirrors Python's layout_recognizer.py:84-100 + // (filter + sort_Y_firstly + layouts_cleanup) and is the same pipeline the + // parity harness uses (via FilteredDLARegions) to dump post-filter regions + // comparable with Python's page_layout. + filtered := FilteredDLARegions(regions, boxes) + if len(filtered) == 0 { return boxes } - // Scale all regions to PDF space once. - type scaledRegion struct { - x0, y0, x1, y1 float64 - label string - } - scaled := make([]scaledRegion, len(regions)) - for i, r := range regions { - scaled[i] = scaledRegion{ + // Scale filtered regions from image-pixel space to PDF space. + cands := make([]annRegion, 0, len(filtered)) + for _, r := range filtered { + cands = append(cands, annRegion{ x0: r.X0 / scale, y0: r.Y0 / scale, x1: r.X1 / scale, y1: r.Y1 / scale, - label: r.Label, - } + label: r.Label, score: r.Confidence, + }) } - // DLA confidence filter — matches Python's `score >= 0.4`. - regionOK := make([]bool, len(regions)) - for i, r := range regions { - regionOK[i] = r.Confidence >= 0.4 || !isGarbageLayoutType(r.Label) - } - - // Pre-compute per-type index for each region (Python: matched index within - // filtered layouts_of_type list). "text" regions get 0,1,2... independent - // of "figure" regions. - typeIndex := make([]int, len(regions)) + // Per-type index in the cleaned, Y-sorted list (Python: ii in lts_). typeCounters := make(map[string]int) - for j, r := range scaled { - if regionOK[j] { - typeIndex[j] = typeCounters[r.label] - typeCounters[r.label]++ - } + for j := range cands { + cands[j].typeIndex = typeCounters[cands[j].label] + typeCounters[cands[j].label]++ } - // Track visited regions (Python: layout["visited"] = True). - visited := make([]bool, len(regions)) - // Marks for Python-style pop removal. dropped := make([]bool, len(boxes)) @@ -139,9 +377,10 @@ func AnnotateBoxLayouts(boxes []pdf.TextBox, regions []pdf.DLARegion, scale floa continue } bestOverlap := 0.0 + bestRegionOverlap := 0.0 bestJ := -1 - for j, r := range scaled { - if r.label != ty || !regionOK[j] { + for j, r := range cands { + if r.label != ty { continue } ix0 := math.Max(r.x0, boxes[i].X0) @@ -149,20 +388,32 @@ func AnnotateBoxLayouts(boxes []pdf.TextBox, regions []pdf.DLARegion, scale floa ix1 := math.Min(r.x1, boxes[i].X1) iy1 := math.Min(r.y1, boxes[i].Bottom) if ix0 < ix1 && iy0 < iy1 { - ov := (ix1 - ix0) * (iy1 - iy0) / boxArea - if ov > bestOverlap { + inter := (ix1 - ix0) * (iy1 - iy0) + ov := inter / boxArea // fraction of the box covered (Python's ov) + rArea := (r.x1 - r.x0) * (r.y1 - r.y0) + ovRegion := 0.0 + if rArea > 0 { + ovRegion = inter / rArea // fraction of the region covered (Python's _ov) + } + // Mirror Python's (ov, _ov) tuple comparison + // (recognizer.py:255-269): primary key is the box + // coverage ratio; on a tie the region-coverage ratio + // wins (prefer the region the box sits more "inside" + // of); on a full tie keep the first (topmost) region. + if ov > bestOverlap || (ov == bestOverlap && ovRegion > bestRegionOverlap) { bestOverlap = ov + bestRegionOverlap = ovRegion bestJ = j } } } if bestJ >= 0 && bestOverlap >= 0.4 { - // Garbage layout not at page edge → pop (Python: bxs.pop(i)). + // Garbage layout not at page edge -> pop (Python: bxs.pop(i)). if isGarbageLayoutType(ty) && pageImgHeight > 0 && !garbageKeepFeat(ty, boxes[i], pageImgHeight/scale) { dropped[i] = true continue } - visited[bestJ] = true + cands[bestJ].visited = true // Python: equation mapped to "figure" for layout_type if ty == pdf.LayoutTypeEquation { boxes[i].LayoutType = pdf.LayoutTypeFigure @@ -170,7 +421,7 @@ func AnnotateBoxLayouts(boxes []pdf.TextBox, regions []pdf.DLARegion, scale floa boxes[i].LayoutType = ty } // Python: f"{layout_type}-{matched}" where matched is per-type index - boxes[i].LayoutNo = fmt.Sprintf("%s-%d", ty, typeIndex[bestJ]) + boxes[i].LayoutNo = fmt.Sprintf("%s-%d", ty, cands[bestJ].typeIndex) } } } @@ -196,25 +447,28 @@ func AnnotateBoxLayouts(boxes []pdf.TextBox, regions []pdf.DLARegion, scale floa boxes = compacted // Synthetic figure boxes for unmatched figure/equation regions (Python: - // dla_cli.py:187-195). Use a fresh per-type counter for synthetic boxes. - synthIdx := 0 - for j, r := range scaled { - if !regionOK[j] || visited[j] { + // layout_recognizer.py:145-155). Python numbers each unmatched region with + // its index WITHIN the per-type list that also includes already-visited + // regions (enumerate([lt for lt in lts if lt["type"] == ty])), so we reuse + // the per-type typeIndex computed above rather than a separate + // unvisited-only counter. Python keeps figure-N / equation-N in SEPARATE + // namespaces, so the typeIndex is keyed by the original type label. + for j := range cands { + if cands[j].visited { continue } - if r.label != pdf.LayoutTypeFigure && r.label != pdf.LayoutTypeEquation { + if cands[j].label != pdf.LayoutTypeFigure && cands[j].label != pdf.LayoutTypeEquation { continue } boxes = append(boxes, pdf.TextBox{ - X0: r.x0, - X1: r.x1, - Top: r.y0, - Bottom: r.y1, + X0: cands[j].x0, + X1: cands[j].x1, + Top: cands[j].y0, + Bottom: cands[j].y1, Text: "", LayoutType: pdf.LayoutTypeFigure, - LayoutNo: fmt.Sprintf("figure-%d", synthIdx), + LayoutNo: fmt.Sprintf("%s-%d", cands[j].label, cands[j].typeIndex), }) - synthIdx++ } return boxes @@ -232,7 +486,7 @@ func isGarbageLayoutType(ty string) bool { // garbageKeepFeat matches Python's keep_feats in LayoutRecognizer.__call__: // footer near page bottom (>90% of page height) or header near page top (<10%) -// are real page decorations — keep them. Others are DLA noise. +// are real page decorations - keep them. Others are DLA noise. func garbageKeepFeat(ty string, box pdf.TextBox, pageImgHeight float64) bool { switch ty { case pdf.LayoutTypeFooter: diff --git a/internal/deepdoc/parser/pdf/table/table_annotate_parity_test.go b/internal/deepdoc/parser/pdf/table/table_annotate_parity_test.go new file mode 100644 index 0000000000..23cb815e1a --- /dev/null +++ b/internal/deepdoc/parser/pdf/table/table_annotate_parity_test.go @@ -0,0 +1,297 @@ +// Tests pinning the Python-equivalent behavior of AnnotateBoxLayouts. +// +// These assert the layout-annotation semantics that Python's +// LayoutRecognizer.__call__ (deepdoc/vision/layout_recognizer.py:68) produces, +// and that the Go implementation now replicates (GREEN): +// +// - #1 layouts_cleanup: Python de-dupes overlapping same-type regions +// (recognizer.py:124-160, called at layout_recognizer.py:100) BEFORE +// annotation. Go matches via cleanupLayouts, so no extra synthetic +// figure/equation boxes are emitted. +// - #2 sort_Y_firstly: Python sorts regions top-to-bottom (recognizer.py:54, +// layout_recognizer.py:99) before numbering them, so layoutno indices +// follow reading order. Go matches via sortYFirstly. +// - #3 synthetic namespace: Python numbers unmatched figure/equation regions +// in SEPARATE counters -> "figure-N" / "equation-N" +// (layout_recognizer.py:145-155). Go matches via a per-type typeIndex +// keyed by the original type label. +// +// These are pure-Go unit tests (no model server, no external service): they +// feed crafted pdf.DLARegion slices and assert the annotated pdf.TextBox +// output. Coordinates use scale=1 so region pixels == PDF space. + +package table + +import ( + "testing" + + pdf "ragflow/internal/deepdoc/parser/pdf/type" +) + +// countFigureBoxes returns the number of boxes whose LayoutType is "figure". +func countFigureBoxes(boxes []pdf.TextBox) int { + n := 0 + for _, b := range boxes { + if b.LayoutType == pdf.LayoutTypeFigure { + n++ + } + } + return n +} + +// countSyntheticFigures returns the number of figure boxes with no text +// (the synthetic placeholders AnnotateBoxLayouts appends for unvisited +// figure/equation regions). +func countSyntheticFigures(boxes []pdf.TextBox) int { + n := 0 + for _, b := range boxes { + if b.LayoutType == pdf.LayoutTypeFigure && b.Text == "" { + n++ + } + } + return n +} + +// TestAnnotateBoxLayouts_DuplicateFigureRegions_Merged pins #1: two heavily +// overlapping same-type (figure) regions must be treated as ONE, so only one +// figure box (the annotated text box) results and NO synthetic figure box is +// produced. Python merges via layouts_cleanup; Go now mirrors that by fusing +// the overlapping pair into one region, so the duplicate synthetic figure is +// avoided. +func TestAnnotateBoxLayouts_DuplicateFigureRegions_Merged(t *testing.T) { + box := pdf.TextBox{X0: 0, X1: 100, Top: 0, Bottom: 50, Text: "Some caption text", PageNumber: 0} + regions := []pdf.DLARegion{ + {X0: 0, Y0: 0, X1: 100, Y1: 50, Confidence: 0.9, Label: pdf.LayoutTypeFigure}, + {X0: 0, Y0: 0, X1: 100, Y1: 50, Confidence: 0.8, Label: pdf.LayoutTypeFigure}, // identical -> IoU 1.0 + } + + out := AnnotateBoxLayouts([]pdf.TextBox{box}, regions, 1.0, 100.0) + + if got := countFigureBoxes(out); got != 1 { + t.Errorf("#1 layouts_cleanup: expected 1 figure box after de-duplicating overlapping figure regions, got %d", got) + } + if got := countSyntheticFigures(out); got != 0 { + t.Errorf("#1 layouts_cleanup: expected 0 synthetic figure boxes (duplicate region must be merged), got %d", got) + } +} + +// TestAnnotateBoxLayouts_SameTypeOutOfYOrder_LayoutNoFollowsY pins #2: when +// same-type regions arrive in non-reading order (here the bottom region is +// first in the wire list), the layoutno index must still follow top-to-bottom +// order, exactly like Python's sort_Y_firstly. The box unambiguously overlaps +// only the bottom region, so the winner is the same in both languages; only +// the assigned index differs. +func TestAnnotateBoxLayouts_SameTypeOutOfYOrder_LayoutNoFollowsY(t *testing.T) { + // Wire order: bottom region first (NOT top-to-bottom). + regions := []pdf.DLARegion{ + {X0: 0, Y0: 30, X1: 100, Y1: 50, Confidence: 0.9, Label: pdf.LayoutTypeTitle}, // bottom strip + {X0: 0, Y0: 0, X1: 100, Y1: 20, Confidence: 0.9, Label: pdf.LayoutTypeTitle}, // top strip + } + // Box overlaps only the bottom region -> unambiguous winner = bottom region. + box := pdf.TextBox{X0: 0, X1: 100, Top: 30, Bottom: 50, Text: "Title text", PageNumber: 0} + + out := AnnotateBoxLayouts([]pdf.TextBox{box}, regions, 1.0, 100.0) + + var annotated *pdf.TextBox + for i := range out { + if out[i].Text == "Title text" { + annotated = &out[i] + } + } + if annotated == nil { + t.Fatal("#2 sort_Y_firstly: text box was not annotated as title") + } + // Python Y-sorts -> [top(ii=0), bottom(ii=1)] -> winner bottom => "title-1". + if annotated.LayoutNo != "title-1" { + t.Errorf("#2 sort_Y_firstly: expected layoutno \"title-1\" (top-to-bottom numbering), got %q", annotated.LayoutNo) + } +} + +// TestAnnotateBoxLayouts_SyntheticFigureEquation_SeparateNamespaces pins #3: +// an unmatched figure region and an unmatched equation region must yield +// distinct synthetic layoutnos "figure-0" and "equation-0". Go now keeps +// per-type synthetic counters so figures and equations are numbered +// independently instead of sharing a single figure-N counter. +func TestAnnotateBoxLayouts_SyntheticFigureEquation_SeparateNamespaces(t *testing.T) { + regions := []pdf.DLARegion{ + {X0: 0, Y0: 0, X1: 50, Y1: 50, Confidence: 0.9, Label: pdf.LayoutTypeFigure}, + {X0: 60, Y0: 0, X1: 110, Y1: 50, Confidence: 0.9, Label: pdf.LayoutTypeEquation}, + } + + out := AnnotateBoxLayouts([]pdf.TextBox{}, regions, 1.0, 100.0) + + layoutNos := map[string]bool{} + for _, b := range out { + layoutNos[b.LayoutNo] = true + } + if !layoutNos["figure-0"] { + t.Errorf("#3 synthetic namespace: expected a synthetic box with layoutno \"figure-0\"") + } + if !layoutNos["equation-0"] { + t.Errorf("#3 synthetic namespace: expected a synthetic box with layoutno \"equation-0\" (separate counter from figure); equation was folded into the shared figure counter") + } +} + +// TestAnnotateBoxLayouts_TieBreakRegionCoverage pins #4: when two same-type +// regions cover the box by the SAME fraction (ov tie), Python's +// find_overlapped_with_threshold (recognizer.py:255-269) breaks the tie by the +// region-coverage ratio (_ov = box∩region / region area), preferring the region +// the box sits more "inside" of. Go mirrors the (ov, _ov) tuple comparison so +// the higher-_ov region wins the tie instead of the first Y-sorted candidate. +// +// Layout: wide top region A and narrow bottom region B. The box spans both +// vertically with EQUAL absolute intersection, so ov_A == ov_B. B is smaller, +// so _ov_B > _ov_A and Python picks B. In Y-sorted order [A, B], B is per-type +// index 1, so the expected layoutno is "table-1". +func TestAnnotateBoxLayouts_TieBreakRegionCoverage(t *testing.T) { + regions := []pdf.DLARegion{ + {X0: 0, Y0: 0, X1: 200, Y1: 50, Confidence: 0.9, Label: pdf.LayoutTypeTable}, // A: wide, top + {X0: 0, Y0: 60, X1: 50, Y1: 110, Confidence: 0.9, Label: pdf.LayoutTypeTable}, // B: narrow, bottom + } + box := pdf.TextBox{X0: 0, X1: 50, Top: 0, Bottom: 110, Text: "tbl", PageNumber: 0} + + out := AnnotateBoxLayouts([]pdf.TextBox{box}, regions, 1.0, 0) + + var annotated *pdf.TextBox + for i := range out { + if out[i].Text == "tbl" { + annotated = &out[i] + } + } + if annotated == nil { + t.Fatal("#4 tie-break: box not annotated as table") + } + if annotated.LayoutNo != "table-1" { + t.Errorf("#4 tie-break: expected layoutno \"table-1\" (smaller region B wins the _ov tie), got %q", annotated.LayoutNo) + } +} + +// TestAnnotateBoxLayouts_NMSDedupOverlappingRegions pins #6: Python's layout +// model postprocess applies per-class NMS with IoU 0.45 (layout_recognizer.py:246, +// operators.py:667) on the RAW detections BEFORE annotation. Go must do the same +// on the raw regions; otherwise same-label detections overlapping between 0.45 +// and 0.7 survive (cleanupLayouts only merges at thr=0.7) and emit extra +// synthetic figure boxes. +// +// Two same-label figure regions with IoU ~0.54: >0.45 so NMS suppresses the +// lower-score one; <0.7 so cleanupLayouts would NOT merge them. With NMS there +// is exactly one figure region -> one synthetic figure box. +func TestAnnotateBoxLayouts_NMSDedupOverlappingRegions(t *testing.T) { + regions := []pdf.DLARegion{ + {X0: 0, Y0: 0, X1: 100, Y1: 100, Confidence: 0.9, Label: pdf.LayoutTypeFigure}, // A + {X0: 35, Y0: 0, X1: 135, Y1: 100, Confidence: 0.8, Label: pdf.LayoutTypeFigure}, // B overlaps A (shift 35) + } + // No text box overlaps -> without NMS both become synthetic figures. + // Overlap is ~0.65 (no-+1 IoU) so cleanupLayouts (thr=0.7) does NOT merge, + // but Python's NMS uses +1 IoU (~0.50) > 0.45 and suppresses the lower-score B. + out := AnnotateBoxLayouts([]pdf.TextBox{}, regions, 1.0, 100.0) + + if got := countFigureBoxes(out); got != 1 { + t.Errorf("#6 NMS: expected 1 figure box after per-class NMS merges overlapping detections (IoU ~0.5), got %d", got) + } +} + +// TestAnnotateBoxLayouts_NMSDeterministic pins that equal-confidence +// same-label detections collapse to the SAME region on every run. Before the +// sort.SliceStable + original-index tie-break, sort.Slice ordered equal scores +// nondeterministically, so the survivor (and its synthetic LayoutNo) could flip +// between otherwise-identical runs — breaking reproducibility and the golden +// comparison. +func TestAnnotateBoxLayouts_NMSDeterministic(t *testing.T) { + regions := []pdf.DLARegion{ + {X0: 0, Y0: 0, X1: 100, Y1: 100, Confidence: 0.9, Label: pdf.LayoutTypeFigure}, // A (lower original index) + {X0: 35, Y0: 0, X1: 135, Y1: 100, Confidence: 0.9, Label: pdf.LayoutTypeFigure}, // B overlaps A, equal score + } + var baseline []pdf.TextBox + for run := 0; run < 30; run++ { + out := AnnotateBoxLayouts([]pdf.TextBox{}, regions, 1.0, 100.0) + if got := countFigureBoxes(out); got != 1 { + t.Fatalf("run %d: expected exactly 1 figure box after NMS, got %d", run, got) + } + if run == 0 { + baseline = out + continue + } + if len(out) != len(baseline) { + t.Fatalf("run %d: box count changed across runs (%d vs %d)", run, len(out), len(baseline)) + } + for i := range out { + if out[i].X0 != baseline[i].X0 || out[i].X1 != baseline[i].X1 || + out[i].Top != baseline[i].Top || out[i].Bottom != baseline[i].Bottom || + out[i].LayoutNo != baseline[i].LayoutNo { + t.Fatalf("run %d: nondeterministic output (box %d differs from run 0)", run, i) + } + } + } + // Lower-index tie-break keeps region A. + if baseline[0].X0 != 0 || baseline[0].X1 != 100 { + t.Errorf("expected the lower-index region A (x0=0,x1=100) to survive, got x0=%v,x1=%v", baseline[0].X0, baseline[0].X1) + } +} + +// TestAnnotateBoxLayouts_CleanupEqualScoreKeepsLater pins that when two +// same-type regions overlap beyond cleanup's 0.7 threshold with EQUAL +// confidence, Go keeps the LATER one (j) — matching Python layouts_cleanup +// (pop(i) on equal scores). Before the fix Go kept the earlier region. +// +// NMS (IoU 0.45) must not pre-remove either: A sits fully inside B, so their +// +1 IoU is tiny (<0.45) while their overlap ratio (inter/area of B) exceeds +// 0.7, so only cleanup is exercised. +func TestAnnotateBoxLayouts_CleanupEqualScoreKeepsLater(t *testing.T) { + regions := []pdf.DLARegion{ + {X0: 0, Y0: 0, X1: 400, Y1: 400, Confidence: 0.9, Label: pdf.LayoutTypeFigure}, // B: large, on top (Y0=0) + {X0: 100, Y0: 200, X1: 200, Y1: 300, Confidence: 0.9, Label: pdf.LayoutTypeFigure}, // A: small, inside B, lower (Y0=200) + } + // No text box overlaps -> both unvisited -> cleanup collapses to one synthetic. + out := AnnotateBoxLayouts([]pdf.TextBox{}, regions, 1.0, 100.0) + if got := countFigureBoxes(out); got != 1 { + t.Fatalf("expected 1 figure box after cleanup merge, got %d", got) + } + // Y-sort puts B (Y0=0) first, A (Y0=200) second; equal score -> keep later (A). + if out[0].X0 != 100 || out[0].X1 != 200 || out[0].Top != 200 || out[0].Bottom != 300 { + t.Errorf("cleanup equal-score: expected later region A (x0=100,x1=200,top=200,bottom=300) to survive, got x0=%v,x1=%v,top=%v,bottom=%v", + out[0].X0, out[0].X1, out[0].Top, out[0].Bottom) + } +} + +// TestAnnotateBoxLayouts_SyntheticVisitedInterleaved pins that an unmatched +// figure region is numbered by its index in the per-type list that ALSO +// includes already-visited figure regions (Python's enumerate over the full +// type-filtered list). Before the fix Go used a separate unvisited-only +// counter, so the unmatched figure became figure-0 instead of figure-1. +func TestAnnotateBoxLayouts_SyntheticVisitedInterleaved(t *testing.T) { + boxes := []pdf.TextBox{ + {X0: 0, X1: 100, Top: 0, Bottom: 50, Text: "caption for A"}, + } + regions := []pdf.DLARegion{ + {X0: 0, Y0: 0, X1: 100, Y1: 50, Confidence: 0.9, Label: pdf.LayoutTypeFigure}, // A: matched to text box -> visited + {X0: 200, Y0: 0, X1: 300, Y1: 50, Confidence: 0.9, Label: pdf.LayoutTypeFigure}, // B: unmatched -> synthetic + } + out := AnnotateBoxLayouts(boxes, regions, 1.0, 100.0) + + var annotated, synthetic *pdf.TextBox + for i := range out { + if out[i].Text == "caption for A" { + annotated = &out[i] + } + if out[i].LayoutType == pdf.LayoutTypeFigure && out[i].Text == "" { + synthetic = &out[i] + } + } + if annotated == nil { + t.Fatal("visited figure A: text box was not annotated as figure") + } + if annotated.LayoutNo != "figure-0" { + t.Errorf("visited figure A: expected LayoutNo figure-0, got %q", annotated.LayoutNo) + } + if synthetic == nil { + t.Fatal("expected a synthetic figure box for unmatched B") + } + // B is the 2nd figure in Y order (after visited A), so Python numbers it figure-1. + if synthetic.LayoutNo != "figure-1" { + t.Errorf("unmatched figure B: expected figure-1 (index in full per-type list), got %q", synthetic.LayoutNo) + } + if synthetic.X0 != 200 || synthetic.X1 != 300 { + t.Errorf("unmatched figure B: expected x0=200,x1=300, got x0=%v,x1=%v", synthetic.X0, synthetic.X1) + } +} diff --git a/internal/deepdoc/parser/pdf/table/table_annotate_test.go b/internal/deepdoc/parser/pdf/table/table_annotate_test.go index 3ac34468f4..6ffd615b99 100644 --- a/internal/deepdoc/parser/pdf/table/table_annotate_test.go +++ b/internal/deepdoc/parser/pdf/table/table_annotate_test.go @@ -154,6 +154,34 @@ func TestAnnotateBoxLayouts_ConfidenceFilter(t *testing.T) { } } +// TestFilteredDLARegions pins the post-filter region set the parity harness +// dumps for comparison with Python's page_layout. The confidence filter keeps +// a region when score >= 0.4 OR its type is not garbage — so a low-confidence +// *non-garbage* region (e.g. text at 0.1) is KEPT, exactly matching Python's +// `score >= 0.4 or type not in garbage_layouts` (layout_recognizer.py:97). +// Returned regions stay in image-pixel space (no scale division). +func TestFilteredDLARegions(t *testing.T) { + regions := []pdf.DLARegion{ + {X0: 0, Y0: 0, X1: 300, Y1: 150, Label: "footer", Confidence: 0.2}, // low-conf garbage → dropped + {X0: 0, Y0: 200, X1: 300, Y1: 350, Label: "text", Confidence: 0.1}, // low-conf non-garbage → kept + {X0: 0, Y0: 400, X1: 500, Y1: 460, Label: "reference", Confidence: 0.5}, // >=0.4 garbage → kept + } + got := FilteredDLARegions(regions, nil) + if len(got) != 2 { + t.Fatalf("FilteredDLARegions() = %d regions, want 2 (got %+v)", len(got), got) + } + labels := map[string]bool{} + for _, r := range got { + labels[r.Label] = true + if r.Confidence == 0.2 { + t.Errorf("low-confidence footer should have been filtered out") + } + } + if !labels["text"] || !labels["reference"] { + t.Errorf("expected text(low-conf) and reference(>=0.4) kept, got labels %v", labels) + } +} + func TestAnnotateBoxLayouts_GarbageFooterRejected(t *testing.T) { // Footer at page bottom: Bottom(290) > 270 (90% of 300px→PDF height 100→90% of 100=90) // → real footer decoration → garbage → pop (Python: bxs.pop(i)). @@ -278,8 +306,11 @@ func TestAnnotateBoxLayouts_SyntheticFigure(t *testing.T) { if b.LayoutType == "figure" && b.Text == "" { if b.LayoutNo == "figure-0" { foundFig0 = true - if b.X0 != 100 || b.X1 != 200 { - t.Errorf("synthetic figure-0: expected x0=100,x1=200 (300/3,600/3), got x0=%v,x1=%v", b.X0, b.X1) + // After sort_Y_firstly, the top figure region (Y0=0 -> PDF + // y0=0, x0=200 from 600/3, x1=300 from 900/3) is figure-0, + // matching Python's top-to-bottom numbering. + if b.X0 != 200 || b.X1 != 300 { + t.Errorf("synthetic figure-0: expected x0=200,x1=300 (top region after Y-sort), got x0=%v,x1=%v", b.X0, b.X1) } } if b.LayoutNo == "figure-1" { diff --git a/internal/deepdoc/parser/pdf/tool/compare.go b/internal/deepdoc/parser/pdf/tool/compare.go index c62ea849be..c8c0c88a77 100644 --- a/internal/deepdoc/parser/pdf/tool/compare.go +++ b/internal/deepdoc/parser/pdf/tool/compare.go @@ -47,6 +47,9 @@ func CompareWithPython(log TLogger, goResults []BatchResult, pyResults []PyResul var diffs []Diff matched, mismatched := 0, 0 + // suppressedAny tracks whether any PDF hit the Py stage-metric phase + // gap (see the guard below) so we can warn once in the summary. + suppressedAny := false for _, r := range goResults { py, ok := pyMap[r.File] @@ -74,6 +77,23 @@ func CompareWithPython(log TLogger, goResults []BatchResult, pyResults []PyResul if py.Sections > 0 { d.SectionsDiffPct = math.Abs(float64(r.Sections-py.Sections)) / float64(py.Sections) * 100 } + + // Phase-consistency guard: the Python harness cannot report + // post-merge pipeline stages (boxes_text_merge / boxes_vertical_merge + // / sections) because the production parser does not expose them, so + // its #@meta falls back to the raw final box count for all three. + // Comparing those against Go's real merged stages would emit + // misleading percentages (e.g. 67% on "sections"). Mark them N/A. + if py.BoxesInitial > 0 && + py.BoxesTextMerge == py.BoxesInitial && + py.BoxesVertMerge == py.BoxesInitial && + py.Sections == py.BoxesInitial { + d.BoxesTMDiffPct = -1 + d.BoxesVMDiffPct = -1 + d.SectionsDiffPct = -1 + suppressedAny = true + } + if py.TextLen > 0 { d.TextLenDiffPct = math.Abs(float64(r.TextLen-py.TextLen)) / float64(py.TextLen) * 100 } @@ -103,7 +123,18 @@ func CompareWithPython(log TLogger, goResults []BatchResult, pyResults []PyResul len(diffs), len(goResults), r.File, 100-d.CharSim, 100-d.LcsSim, 100-d.RawCharSim, 100-d.RawLcsSim) } - sort.Slice(diffs, func(i, j int) bool { return diffs[i].SectionsDiffPct < diffs[j].SectionsDiffPct }) + // Sort worst-first by SectionsDiffPct, but push N/A (-1) rows to the + // bottom so the per-PDF table reads cleanly. + sort.Slice(diffs, func(i, j int) bool { + a, b := diffs[i].SectionsDiffPct, diffs[j].SectionsDiffPct + if a < 0 { + a = math.MaxFloat64 + } + if b < 0 { + b = math.MaxFloat64 + } + return a < b + }) log.Logf("\n=== Go vs Python (%d PDFs) ===", len(diffs)) log.Logf("Pages match: %d/%d", matched, matched+mismatched) @@ -117,10 +148,10 @@ func CompareWithPython(log TLogger, goResults []BatchResult, pyResults []PyResul gr := goMap[d.File] goStages := fmt.Sprintf("%3d->%3d->%3d->%3d", gr.BoxesInitial, gr.BoxesTextMerg, gr.BoxesVertMerg, gr.Sections) pyStages := fmt.Sprintf("%3d->%3d->%3d->%3d", py.BoxesInitial, py.BoxesTextMerge, py.BoxesVertMerge, py.Sections) - log.Logf("%-40s %-18s %-18s %4.0f%% %4.0f%% %4.0f%% %4.0f%% %4.0f%% %+4d %.0f%% %.0f%% %.0f%% %.0f%%", + log.Logf("%-40s %-18s %-18s %5s %5s %5s %5s %5.0f%% %+4d %.0f%% %.0f%% %.0f%% %.0f%%", d.File, goStages, pyStages, - d.BoxesInitDiffPct, d.BoxesTMDiffPct, d.BoxesVMDiffPct, - d.SectionsDiffPct, d.TextLenDiffPct, d.TablesDiff, + pctStr(d.BoxesInitDiffPct), pctStr(d.BoxesTMDiffPct), pctStr(d.BoxesVMDiffPct), + pctStr(d.SectionsDiffPct), d.TextLenDiffPct, d.TablesDiff, 100-d.CharSim, 100-d.LcsSim, 100-d.RawCharSim, 100-d.RawLcsSim) } @@ -134,17 +165,31 @@ func CompareWithPython(log TLogger, goResults []BatchResult, pyResults []PyResul median, mean, max, min float64 over5, over10 int } - computeStats := func(get func(Diff) float64) stats { - sort.Slice(diffs, func(i, j int) bool { return get(diffs[i]) < get(diffs[j]) }) - s := stats{min: 1e9} - if n%2 == 0 { - s.median = (get(diffs[n/2-1]) + get(diffs[n/2])) / 2 - } else { - s.median = get(diffs[n/2]) - } - var sum float64 + // computeStats returns summary statistics over the valid (>=0) samples. + // It returns ok=false when every sample is N/A, so the caller can print + // "N/A" instead of bogus numbers. + computeStats := func(get func(Diff) float64) (s stats, ok bool) { + s.min = math.MaxFloat64 + var vals []float64 for _, d := range diffs { v := get(d) + if v < 0 { + continue + } + vals = append(vals, v) + } + if len(vals) == 0 { + return s, false + } + sort.Float64s(vals) + m := len(vals) + if m%2 == 0 { + s.median = (vals[m/2-1] + vals[m/2]) / 2 + } else { + s.median = vals[m/2] + } + var sum float64 + for _, v := range vals { sum += v if v > s.max { s.max = v @@ -159,8 +204,8 @@ func CompareWithPython(log TLogger, goResults []BatchResult, pyResults []PyResul s.over10++ } } - s.mean = sum / float64(n) - return s + s.mean = sum / float64(m) + return s, true } label := func(name string, s stats) string { @@ -169,15 +214,59 @@ func CompareWithPython(log TLogger, goResults []BatchResult, pyResults []PyResul } log.Logf("\nSummary (n=%d):", n) - log.Logf(" %s", label("BoxesInit ", computeStats(func(d Diff) float64 { return d.BoxesInitDiffPct }))) - log.Logf(" %s", label("TextMerge", computeStats(func(d Diff) float64 { return d.BoxesTMDiffPct }))) - log.Logf(" %s", label("VertMerge", computeStats(func(d Diff) float64 { return d.BoxesVMDiffPct }))) - log.Logf(" %s", label("Sections ", computeStats(func(d Diff) float64 { return d.SectionsDiffPct }))) - log.Logf(" %s", label("TextLen ", computeStats(func(d Diff) float64 { return d.TextLenDiffPct }))) - log.Logf(" %s", label("CharDiff ", computeStats(func(d Diff) float64 { return 100 - d.CharSim }))) - log.Logf(" %s", label("LcsDiff ", computeStats(func(d Diff) float64 { return 100 - d.LcsSim }))) - log.Logf(" %s", label("RawCharDiff", computeStats(func(d Diff) float64 { return 100 - d.RawCharSim }))) - log.Logf(" %s", label("RawLcsDiff ", computeStats(func(d Diff) float64 { return 100 - d.RawLcsSim }))) + if s, ok := computeStats(func(d Diff) float64 { return d.BoxesInitDiffPct }); ok { + log.Logf(" %s", label("BoxesInit ", s)) + } else { + log.Logf(" BoxesInit N/A") + } + if s, ok := computeStats(func(d Diff) float64 { return d.BoxesTMDiffPct }); ok { + log.Logf(" %s", label("TextMerge", s)) + } else { + log.Logf(" TextMerge N/A (Python harness does not expose post-merge stage)") + } + if s, ok := computeStats(func(d Diff) float64 { return d.BoxesVMDiffPct }); ok { + log.Logf(" %s", label("VertMerge", s)) + } else { + log.Logf(" VertMerge N/A (Python harness does not expose post-merge stage)") + } + if s, ok := computeStats(func(d Diff) float64 { return d.SectionsDiffPct }); ok { + log.Logf(" %s", label("Sections ", s)) + } else { + log.Logf(" Sections N/A (Python harness does not expose post-merge stage)") + } + if s, ok := computeStats(func(d Diff) float64 { return d.TextLenDiffPct }); ok { + log.Logf(" %s", label("TextLen ", s)) + } else { + log.Logf(" TextLen N/A") + } + if s, ok := computeStats(func(d Diff) float64 { return 100 - d.CharSim }); ok { + log.Logf(" %s", label("CharDiff ", s)) + } else { + log.Logf(" CharDiff N/A") + } + if s, ok := computeStats(func(d Diff) float64 { return 100 - d.LcsSim }); ok { + log.Logf(" %s", label("LcsDiff ", s)) + } else { + log.Logf(" LcsDiff N/A") + } + if s, ok := computeStats(func(d Diff) float64 { return 100 - d.RawCharSim }); ok { + log.Logf(" %s", label("RawCharDiff", s)) + } else { + log.Logf(" RawCharDiff N/A") + } + if s, ok := computeStats(func(d Diff) float64 { return 100 - d.RawLcsSim }); ok { + log.Logf(" %s", label("RawLcsDiff ", s)) + } else { + log.Logf(" RawLcsDiff N/A") + } + + if suppressedAny { + log.Logf("\nNOTE: 'N/A' in TM%%/VM%%/Sec%% means the Python harness could not") + log.Logf("expose post-merge pipeline stage counts (the production parser does") + log.Logf("not record boxes_text_merge / boxes_vertical_merge / sections), so its") + log.Logf("#@meta falls back to the raw final box count. Those columns are not") + log.Logf("comparable and were excluded from the summary statistics.") + } // Auto-generate xlsx report with timestamp. mode := filepath.Base(filepath.Dir(goTextDir)) // "ocr" @@ -289,11 +378,15 @@ func WriteExcel(path string, diffs []Diff) error { for col := 2; col <= 9; col++ { cell := cellName(col, r) v := vals[col-1] - f.SetCellValue(sheet, cell, v) - // Color: green <5, yellow 5-20, red >=20. if col == 7 { // TabsD is a count, not percentage + f.SetCellValue(sheet, cell, v) continue } + if v < 0 { // N/A: leave the cell blank, no color. + continue + } + f.SetCellValue(sheet, cell, v) + // Color: green <5, yellow 5-20, red >=20. abs := math.Abs(v) switch { case abs < 5: @@ -646,3 +739,12 @@ func abs(x int) int { } return x } + +// pctStr renders a diff-percentage for table output. Negative values encode +// "N/A" (e.g. a Python stage metric that could not be reported). +func pctStr(v float64) string { + if v < 0 { + return "N/A" + } + return fmt.Sprintf("%.0f%%", v) +} diff --git a/internal/deepdoc/parser/pdf/tool/compare_test.go b/internal/deepdoc/parser/pdf/tool/compare_test.go index bd2dee92f8..1fc7951c5c 100644 --- a/internal/deepdoc/parser/pdf/tool/compare_test.go +++ b/internal/deepdoc/parser/pdf/tool/compare_test.go @@ -7,6 +7,8 @@ import ( "os" "path/filepath" "testing" + + "ragflow/internal/common" ) // TestBatchCompareWithPython compares Go output against Python reference diff --git a/internal/parser/parser/pdf_postprocess.go b/internal/parser/parser/pdf_postprocess.go index 7a0ff99b2f..cc1466ff3e 100644 --- a/internal/parser/parser/pdf_postprocess.go +++ b/internal/parser/parser/pdf_postprocess.go @@ -9,7 +9,11 @@ import ( deepdoctype "ragflow/internal/deepdoc/parser/type" ) -var pdfHeaderFooterPattern = regexp.MustCompile(`(?i)^(header|footer|number)$`) +// Substring match to mirror Python's remove_header_footer: +// re.search(r"(header|footer|number)", raw_layout, re.I) (rag/flow/parser/parser.py:754). +// Python matches any layout type CONTAINING one of these words, not just the +// exact token, so a composite label like "page-footer" is also stripped. +var pdfHeaderFooterPattern = regexp.MustCompile(`(?i)header|footer|number`) var pdfTOCTitlePattern = regexp.MustCompile(`(?i)^(contents|目录|目次|table of contents|致谢|acknowledge)$`) type pdfPostProcessOptions struct { diff --git a/internal/parser/parser/pdf_postprocess_test.go b/internal/parser/parser/pdf_postprocess_test.go index dbb62bb12c..4a1e4fdea6 100644 --- a/internal/parser/parser/pdf_postprocess_test.go +++ b/internal/parser/parser/pdf_postprocess_test.go @@ -141,3 +141,38 @@ func TestApplyPDFPostProcess_ReordersMultiColumnText(t *testing.T) { } } } + +// TestFilterPDFHeaderFooter_SubstringMatch pins #5: Python's remove_header_footer +// uses a substring match re.search(r"(header|footer|number)", ...) (rag/flow/parser/parser.py:754), +// while Go used an anchored exact match ^(header|footer|number)$. A layout type +// that merely CONTAINS one of those words (e.g. "page-footer") must be stripped to +// match Python, not silently kept. +func TestFilterPDFHeaderFooter_SubstringMatch(t *testing.T) { + result := &deepdoctype.ParseResult{ + Sections: []deepdoctype.Section{ + {Text: "real header", LayoutType: "header"}, + {Text: "real footer", LayoutType: "footer"}, + {Text: "page 1", LayoutType: "number"}, + {Text: "a page-footer note", LayoutType: "page-footer"}, // composite -> substring match + {Text: "body text", LayoutType: "text"}, + }, + } + filterPDFHeaderFooter(result) + + kept := map[string]bool{} + for _, s := range result.Sections { + kept[s.LayoutType] = true + } + for _, lt := range []string{"header", "footer", "number"} { + if kept[lt] { + t.Errorf("#5 header/footer: %q should be stripped", lt) + } + } + // Composite "page-footer" must be stripped by substring match (Python-equivalent). + if kept["page-footer"] { + t.Errorf("#5 header/footer: composite layout type %q should be stripped by substring match", "page-footer") + } + if !kept["text"] { + t.Errorf("#5 header/footer: body text %q should be kept", "text") + } +}