diff --git a/internal/deepdoc/parser/pdf/table/table_cell_spatial_test.go b/internal/deepdoc/parser/pdf/table/table_cell_spatial_test.go index 36be34552c..82c4cf5f02 100644 --- a/internal/deepdoc/parser/pdf/table/table_cell_spatial_test.go +++ b/internal/deepdoc/parser/pdf/table/table_cell_spatial_test.go @@ -110,3 +110,213 @@ func TestFillCellTextFromBoxes_NoMatchingBox(t *testing.T) { t.Errorf("no match: got %q, want empty", cells[0].Text) } } + +// TestBoxMatchesCell_FilledCellWeakOverlapRejected locks the Go-only 0.85 +// guard: a cell that already carries text (e.g. per-cell OCR in the rotated +// path) rejects a box whose area is only partially inside the cell. +// Cell (0,0)-(100,50); box (40,5)-(140,15). +// +// overlap = (40,100)x(5,15) = 60*10 = 600 +// box area = (140-40)*(15-5) = 100*10 = 1000 → ratio 0.6 +// +// 0.6 < 0.85 → rejected. +func TestBoxMatchesCell_FilledCellWeakOverlapRejected(t *testing.T) { + cell := pdf.TSRCell{X0: 0, Y0: 0, X1: 100, Y1: 50, Text: "ocr"} + box := pdf.TextBox{X0: 40, X1: 140, Top: 5, Bottom: 15, Text: "元"} + if BoxMatchesCell(cell, box, false) { + t.Error("filled cell should reject box overlapping only 60% (needs >=85%)") + } +} + +// TestBoxMatchesCell_FilledCellStrongOverlapAccepted locks that a box almost +// entirely inside an already-filled cell still matches (so per-cell OCR text +// can be overridden by a confident box). Box fully inside → ratio 1.0. +func TestBoxMatchesCell_FilledCellStrongOverlapAccepted(t *testing.T) { + cell := pdf.TSRCell{X0: 0, Y0: 0, X1: 100, Y1: 50, Text: "ocr"} + box := pdf.TextBox{X0: 5, X1: 95, Top: 5, Bottom: 45, Text: "text"} + if !BoxMatchesCell(cell, box, false) { + t.Error("filled cell should accept box fully inside (>=85%)") + } +} + +// TestFillCellTextFromBoxes_EmptyCellJoinsMultiplePartialBoxes proves the 0.85 +// guard only bites cells that ENTER FillCellTextFromBoxes with text. For a cell +// that starts empty, every overlapping box (here each 60%) is matched at the +// 0.3 threshold and all are joined — no in-cell text loss in the normal path. +// cell (0,0)-(100,50); box1 (40,5)-(140,15) ratio 0.6; box2 (-40,30)-(60,45) ratio 0.6. +func TestFillCellTextFromBoxes_EmptyCellJoinsMultiplePartialBoxes(t *testing.T) { + cells := []pdf.TSRCell{{X0: 0, Y0: 0, X1: 100, Y1: 50}} + boxes := []pdf.TextBox{ + {X0: 40, X1: 140, Top: 5, Bottom: 15, Text: "part1"}, + {X0: -40, X1: 60, Top: 30, Bottom: 45, Text: "part2"}, + } + FillCellTextFromBoxes(cells, boxes) + if cells[0].Text != "part1 part2" { + t.Errorf("empty cell should join all overlapping boxes: got %q, want 'part1 part2'", cells[0].Text) + } +} + +// TestFillCellTextFromBoxes_PrefilledCellDropsSecondaryBox documents the +// deliberate go_intentional divergence from Python. +// +// Scenario: a cell already holds per-cell OCR text ("Total"). A separate +// detected text box ("元") physically sits in the same cell but only overlaps +// 60% of its area. Go applies the 0.85 guard (cell entered with text) and +// DROPS the secondary box, keeping "Total". +// +// Python has no per-cell OCR and no filled-cell threshold: both fragments +// would be joined via find_overlapped_with_threshold(thr=0.3), yielding +// "Total 元". This test LOCKS Go's current behavior so the divergence stays +// visible; it is registered as go_intentional, not a regression target. +func TestFillCellTextFromBoxes_PrefilledCellDropsSecondaryBox(t *testing.T) { + cells := []pdf.TSRCell{{X0: 0, Y0: 0, X1: 100, Y1: 50, Text: "Total"}} // pre-filled by per-cell OCR + boxes := []pdf.TextBox{ + {X0: 40, X1: 140, Top: 5, Bottom: 15, Text: "元"}, // 60% inside + } + FillCellTextFromBoxes(cells, boxes) + // Go: 0.85 guard drops the 60%-overlap box. Python would keep "Total 元". + if cells[0].Text != "Total" { + t.Errorf("go_intentional divergence: got %q, want 'Total' (Python would yield 'Total 元')", cells[0].Text) + } +} + +// ============================================================================= +// Implementation divergences in box→cell ASSIGNMENT (NOT architecture: both +// sides build the grid from TSR rows×columns via cross-product — see +// deepdoc_table_builder.go GroupCells vs Python construct_table). The divergences +// are purely in how a box is assigned to a (row,column) cell. Registered in +// testdata/parity/known_diffs.json as go_bug (#1, #2, #3). +// ============================================================================= + +// TestFillCellTextFromBoxes_BoxOverlappingTwoCells_SingleAssignment is the +// regression test for go_bug #1. Python assigns each box to EXACTLY ONE cell +// (greedy best row + tightest column); Go must no longer duplicate a +// straddling box into both cells. +// +// Two side-by-side cells; one box straddles the boundary, overlapping BOTH at +// 50% of its area. cell A (0,0)-(100,50); cell B (100,0)-(200,50). +// box (50,5)-(150,15): both columns are equally tight (dis=50), so Python +// keeps the first (C0). The box lands in exactly ONE cell. +func TestFillCellTextFromBoxes_BoxOverlappingTwoCells_SingleAssignment(t *testing.T) { + cells := []pdf.TSRCell{ + {X0: 0, Y0: 0, X1: 100, Y1: 50}, + {X0: 100, Y0: 0, X1: 200, Y1: 50}, + } + boxes := []pdf.TextBox{ + {X0: 50, X1: 150, Top: 5, Bottom: 15, Text: "shared"}, + } + FillCellTextFromBoxes(cells, boxes) + count := 0 + for _, c := range cells { + if c.Text == "shared" { + count++ + } + } + if count != 1 { + t.Errorf("regression #1: box duplicated into %d cells; Python assigns it to exactly ONE cell", count) + } +} + +// TestFillCellTextFromBoxes_PythonAssignsGoRejects_2DThreshold_FIXED is the +// regression test for go_bug #2. Python's 0.3 tests the 1-D VERTICAL overlap +// with the (full-width) row, and the column is chosen by +// find_horizontally_tightest_fit (no threshold). Go must match: a box whose +// row-vertical overlap is >= 0.3 and that is tightest to a column must be +// assigned, even if its 2-D overlap with that single cell is < 0.3. +// +// Grid: row R0 Y=(0,30); col C0 X=(0,50), col C1 X=(50,200). +// cell (R0,C0) = (0,0)-(50,30); cell (R0,C1) = (50,0)-(200,30). +// box (0,0)-(100,100): vertical overlap with R0 = 30/100 = 30% (≥0.3 row +// match); horizontally tightest to C0 (left edge aligns). Python → (R0,C0). +func TestFillCellTextFromBoxes_PythonAssignsGoRejects_2DThreshold_FIXED(t *testing.T) { + cells := []pdf.TSRCell{ + {X0: 0, Y0: 0, X1: 50, Y1: 30}, // (R0,C0) + {X0: 50, Y0: 0, X1: 200, Y1: 30}, // (R0,C1) + } + boxes := []pdf.TextBox{ + {X0: 0, X1: 100, Top: 0, Bottom: 100, Text: "tall"}, + } + FillCellTextFromBoxes(cells, boxes) + if cells[0].Text != "tall" { + t.Errorf("regression #2: Go dropped the box (cell0=%q); Python fills (R0,C0) via row-vertical-0.3 ∩ column-tightest", cells[0].Text) + } + if cells[1].Text != "" { + t.Errorf("regression #2: box leaked into (R0,C1)=%q; should land only in (R0,C0)", cells[1].Text) + } +} + +// TestFillCellTextFromBoxes_EqualBoxRatioSingleAssignment is the regression +// test for go_bug #3. Go must no longer inject a box into ALL cells it +// overlaps; it assigns the box to the single tightest column, so a box +// overlapping a small cell A and a large cell B lands in ONLY A. +// +// Small cell A (0,0)-(50,25) area 1250; large cell B (50,0)-(200,50) area 7500. +// box (0,0)-(100,25): overlaps both, but is tightest to A (left edge aligns, +// dis=0 vs B's dis=50). Python keeps only A; Go must too. +func TestFillCellTextFromBoxes_EqualBoxRatioSingleAssignment(t *testing.T) { + cells := []pdf.TSRCell{ + {X0: 0, Y0: 0, X1: 50, Y1: 25}, // A (small) + {X0: 50, Y0: 0, X1: 200, Y1: 50}, // B (large) + } + boxes := []pdf.TextBox{ + {X0: 0, X1: 100, Top: 0, Bottom: 25, Text: "wide"}, + } + FillCellTextFromBoxes(cells, boxes) + if cells[0].Text != "wide" { + t.Errorf("regression #3: small cell A should contain 'wide', got %q", cells[0].Text) + } + if cells[1].Text == "wide" { + t.Errorf("regression #3: box leaked into large cell B; Python keeps only small cell A") + } +} + +// TestFillCellTextFromBoxes_RowSelectionIgnores2DCellOverlap strengthens the +// go_bug #2 fix on a 2×2 grid. A tall box overlaps ONLY row R0 vertically +// (≥0.3) but its 2-D intersection with every single cell is < 0.3, so the old +// 2-D 0.3 filter dropped it entirely. The row/column selection must still fill +// (R0,C0) — the tightest column in the matched row — and leave R1 empty. +func TestFillCellTextFromBoxes_RowSelectionIgnores2DCellOverlap(t *testing.T) { + cells := []pdf.TSRCell{ + {X0: 0, Y0: 0, X1: 50, Y1: 30}, // (R0,C0) + {X0: 50, Y0: 0, X1: 200, Y1: 30}, // (R0,C1) + {X0: 0, Y0: 30, X1: 50, Y1: 60}, // (R1,C0) + {X0: 50, Y0: 30, X1: 200, Y1: 60}, // (R1,C1) + } + boxes := []pdf.TextBox{ + {X0: 0, X1: 100, Top: 0, Bottom: 100, Text: "tall"}, // vertical overlap with R0 = 30/100 = 0.3 + } + FillCellTextFromBoxes(cells, boxes) + if cells[0].Text != "tall" { + t.Errorf("regression #2 (2x2): (R0,C0) should be 'tall', got %q", cells[0].Text) + } + if cells[1].Text != "" || cells[2].Text != "" || cells[3].Text != "" { + t.Errorf("regression #2 (2x2): box leaked to R0C1/R1 = %q/%q/%q; must stay only in (R0,C0)", + cells[1].Text, cells[2].Text, cells[3].Text) + } +} + +// TestFillCellTextFromBoxes_RowSelectionTiebreak guards the row-selection +// _ov tie-break (inter/rowArea) in FillCellTextFromBoxes. When a box overlaps +// two rows with the SAME vertical-overlap ratio (ov), the row with the higher +// _ov (smaller row area for equal vertical overlap) wins — mirroring Python's +// (ov, _ov) ordering in find_overlapped_with_threshold. +// +// row0 y[0,40] (height 40), row1 y[40,100] (height 60). Box y[0,80] (height 80) +// overlaps BOTH rows by 40 → ov = 40/80 = 0.5 for each. _ov: row0 = 40/40 = 1.0, +// row1 = 40/60 ≈ 0.667. Python keeps row0; Go must too. +func TestFillCellTextFromBoxes_RowSelectionTiebreak(t *testing.T) { + cells := []pdf.TSRCell{ + {X0: 0, Y0: 0, X1: 100, Y1: 40}, // row0 (higher _ov) + {X0: 0, Y0: 40, X1: 100, Y1: 100}, // row1 (lower _ov) + } + boxes := []pdf.TextBox{ + {X0: 0, X1: 100, Top: 0, Bottom: 80, Text: "row0"}, + } + FillCellTextFromBoxes(cells, boxes) + if cells[0].Text != "row0" { + t.Errorf("row tiebreak: box should land in row0 (higher _ov), got cell0=%q", cells[0].Text) + } + if cells[1].Text != "" { + t.Errorf("row tiebreak: box leaked into row1 (lower _ov), got %q", cells[1].Text) + } +} diff --git a/internal/deepdoc/parser/pdf/table/table_cells.go b/internal/deepdoc/parser/pdf/table/table_cells.go index 4c6d2dce9f..6d1adc8244 100644 --- a/internal/deepdoc/parser/pdf/table/table_cells.go +++ b/internal/deepdoc/parser/pdf/table/table_cells.go @@ -68,42 +68,206 @@ func GroupTSRCellsToRows(cells []pdf.TSRCell) [][]pdf.TSRCell { // ── cell text filling ────────────────────────────────────────────────── +// FillCellTextFromBoxes assigns PDF text boxes to TSR grid cells, mirroring +// Python's construct_table box→cell assignment (pdf_parser.py + +// table_structure_recognizer.py): +// +// 1. For each box, pick the single BEST row by vertical-overlap ratio +// inter(box,rowStrip)/area(box) >= 0.3, tie-broken by inter/rowArea +// (Python find_overlapped_with_threshold on the full-width row strip). +// 2. Within that row, pick the TIGHTEST column by horizontal edge/center +// distance, requiring vertical overlap (Python find_horizontally_tightest_fit, +// NO threshold). The box lands in exactly ONE cell (R,C). +// 3. Multiple boxes mapped to the same (R,C) are concatenated (Python joins +// them in construct_table). +// +// This replaces the old many-to-many 2-D cell-overlap filter +// (inter(box,cell)/area(box) >= 0.3 on every cross-product cell), which +// duplicated a straddling box into two cells (#1), dropped boxes whose 2-D +// cell overlap was < 0.3 even though their row-vertical overlap was >= 0.3 +// (#2), and never applied the inter/cellArea tie-break (#3). All three are +// go_bug in testdata/parity/known_diffs.json. +// +// The Go-only 0.85 guard (BoxMatchesCell) is retained for PRE-FILLED cells +// only: if a cell already carries text (e.g. per-cell OCR in the rotated +// path), a detected box overrides it only when it sits almost entirely inside +// the cell (>= 0.85). Empty cells accept any box the row/column selection +// picked, matching Python. See go_intentional rule +// table-cell-fill-filled-threshold-0.85. func FillCellTextFromBoxes(cells []pdf.TSRCell, boxes []pdf.TextBox) { slog.Debug("fillCellTextFromBoxes", "cells", len(cells), "boxes", len(boxes)) - if len(cells) > 0 && len(boxes) > 0 { - c0 := cells[0] - slog.Debug("fillCellTextFromBoxes cell[0]", "x0", c0.X0, "y0", c0.Y0, "x1", c0.X1, "y1", c0.Y1) - b0 := boxes[0] - slog.Debug("fillCellTextFromBoxes box[0]", "x0", b0.X0, "y0", b0.Top, "x1", b0.X1, "y1", b0.Bottom, "text_len", len(b0.Text)) + if len(cells) == 0 || len(boxes) == 0 { + return } - matched, filled := 0, 0 - for ci := range cells { - var matches []string - for _, b := range boxes { - if IsCaptionBox(b.Text, b.LayoutType) { + + // Group cells into row bands by their top coordinate. The grid is a TSR + // row×column cross-product, so every cell in a row shares the same Y band. + // A row band spans the full table width (union of its cells), matching + // Python's full-width "table row" components. + type rowBand struct { + y0, y1 float64 + stripX0 float64 + stripX1 float64 + cells []int // indices into `cells` + } + var rows []rowBand + // Group cells into the same row band by exact top coordinate. A TSR + // cross-product grid (GroupCells) assigns every cell in a row the SAME + // Y0 value, and distinct rows differ by at least a row height, so a tiny + // epsilon is enough and never merges two real rows. + const yTol = 1e-6 + for i := range cells { + c := &cells[i] + if c.X1 <= c.X0 || c.Y1 <= c.Y0 { + continue // degenerate / span-covered cell: not a fill target + } + rb := &rowBand{} + found := false + for ri := range rows { + if math.Abs(rows[ri].y0-c.Y0) <= yTol { + rb = &rows[ri] + found = true + break + } + } + if !found { + rows = append(rows, rowBand{ + y0: c.Y0, y1: c.Y1, stripX0: c.X0, stripX1: c.X1, + }) + rb = &rows[len(rows)-1] + } + if c.X0 < rb.stripX0 { + rb.stripX0 = c.X0 + } + if c.X1 > rb.stripX1 { + rb.stripX1 = c.X1 + } + if c.Y1 > rb.y1 { + rb.y1 = c.Y1 + } + rb.cells = append(rb.cells, i) + } + // Stable ordering: rows top-to-bottom, cells left-to-right (matches + // Python's first-wins tie-breaking in find_overlapped_with_threshold / + // find_horizontally_tightest_fit). + sort.Slice(rows, func(i, j int) bool { return rows[i].y0 < rows[j].y0 }) + for ri := range rows { + sort.Slice(rows[ri].cells, func(a, b int) bool { + return cells[rows[ri].cells[a]].X0 < cells[rows[ri].cells[b]].X0 + }) + } + + // Accumulate box text per target cell so multiple boxes in one cell join. + cellText := make([]string, len(cells)) + cellFilled := make([]bool, len(cells)) + matched := 0 + + for bi := range boxes { + b := boxes[bi] + if IsCaptionBox(b.Text, b.LayoutType) { + continue + } + boxArea := util.Area(&b) + if boxArea <= 0 { + continue + } + // 1. Best row by vertical-overlap ratio (>= 0.3), tie-broken by _ov. + bestR := -1 + bestOv, bestOv2 := 0.3, 0.0 + for ri := range rows { + rb := &rows[ri] + if math.Min(b.Bottom, rb.y1)-math.Max(b.Top, rb.y0) <= 0 { + continue // no vertical overlap + } + strip := pdf.TSRCell{X0: rb.stripX0, Y0: rb.y0, X1: rb.stripX1, Y1: rb.y1} + inter := util.OverlapInter(&strip, &b) + ov := inter / boxArea + ov2 := 0.0 + if a := util.Area(&strip); a > 0 { + ov2 = inter / a + } + // Skip unless strictly better than the current best, mirroring + // Python's (ov, _ov) tuple ordering in find_overlapped_with_threshold. + if !(ov > bestOv || (ov == bestOv && ov2 > bestOv2)) { continue } - if BoxMatchesCell(cells[ci], b, cells[ci].Text == "") { - matched++ - t := strings.TrimSpace(b.Text) - if t != "" { - matches = append(matches, t) - } + bestR, bestOv, bestOv2 = ri, ov, ov2 + } + if bestR < 0 { + continue + } + // 2. Tightest column within the matched row (no threshold). + rb := &rows[bestR] + bestC := -1 + bestDis := 1e9 + for _, ci := range rb.cells { + c := &cells[ci] + if math.Min(b.Bottom, c.Y1)-math.Max(b.Top, c.Y0) <= 0 { + continue + } + if dis := tightestColumnDistance(&b, c); dis < bestDis { + bestDis, bestC = dis, ci } } - if len(matches) > 0 { - cells[ci].Text = strings.Join(matches, " ") - filled++ + if bestC < 0 { + continue + } + // 3. Assign, preserving the 0.85 guard for pre-filled cells only. + target := &cells[bestC] + if target.Text != "" && !BoxMatchesCell(*target, b, false) { + continue + } + t := strings.TrimSpace(b.Text) + if t == "" { + continue + } + if cellFilled[bestC] { + cellText[bestC] += " " + t + } else { + cellText[bestC] = t + cellFilled[bestC] = true + } + matched++ + } + + for i := range cells { + if cellFilled[i] { + cells[i].Text = cellText[i] } } - slog.Debug("fillCellTextFromBoxes done", "cell_box_matches", matched, "cells_filled", filled) + slog.Debug("fillCellTextFromBoxes done", "box_cell_matches", matched, "cells_filled", matched) } -// boxMatchesCell reports whether a text box's text should be assigned -// to a TSR cell. When the cell already has text (from TSR), the box -// must be mostly inside the cell (≥85% of box area). When the cell -// is empty, any overlap suffices — matching Python's _table_transformer_job -// which fills cells from overlapping PDF boxes with thr=0.3. +// tightestColumnDistance mirrors Python's find_horizontally_tightest_fit +// distance metric: the minimum of the left-edge gap, right-edge gap, and +// half the center gap. Smaller means the box sits tighter against the cell. +func tightestColumnDistance(b *pdf.TextBox, c *pdf.TSRCell) float64 { + dis := math.Min(math.Abs(b.X0-c.X0), math.Abs(b.X1-c.X1)) + if center := math.Abs((b.X0+b.X1)-(c.X0+c.X1)) / 2; center < dis { + dis = center + } + return dis +} + +// BoxMatchesCell reports whether a text box's text may be assigned to a +// TSR cell. The threshold is two-stage: +// - empty cell: inter/boxArea >= 0.3 — matches Python's +// find_overlapped_with_threshold default (thr=0.3), which fills cells from +// overlapping PDF boxes uniformly. +// - cell already has text: inter/boxArea >= 0.85 — Go-only guard, NOT in +// Python. In the rotated-table path (table_extract.go) ocrTableCells +// pre-fills cells with per-cell OCR text; the 0.85 bar stops a +// weakly-overlapping detected box from corrupting/overriding that OCR +// result. Python has no per-cell OCR at this stage, so it never raises the +// threshold. This is a deliberate go_intentional divergence: it can drop a +// legitimate secondary text fragment (a box overlapping 30-85%) that +// Python would keep. +// +// FillCellTextFromBoxes uses only the 0.85 branch (cellIsEmpty=false) as the +// guard for PRE-FILLED cells; for empty cells it relies on the row/column +// selection (which already enforces >= 0.3 vertical overlap, matching Python). +// BoxMatchesCell remains the canonical "does this box match this exact cell" +// primitive and is directly unit-tested. func BoxMatchesCell(cell pdf.TSRCell, box pdf.TextBox, cellIsEmpty bool) bool { inter := util.OverlapInter(&cell, &box) boxArea := util.Area(&box) diff --git a/internal/deepdoc/parser/pdf/table/table_construct_test.go b/internal/deepdoc/parser/pdf/table/table_construct_test.go index 2a269a420a..97d7a62ed5 100644 --- a/internal/deepdoc/parser/pdf/table/table_construct_test.go +++ b/internal/deepdoc/parser/pdf/table/table_construct_test.go @@ -393,8 +393,10 @@ func TestExtractTableAndReplace_OnlyTableBoxes(t *testing.T) { } func TestFillCellText_RCOverSpatial(t *testing.T) { - // Box at X=30-270 overlaps all 3 cells (>30% each — spatial fills ALL). - // With R/C, it belongs only to cell[1] (R=0, C=1). + // Box at X=30-270 overlaps all 3 cells, but Python assigns it to exactly + // ONE cell via greedy best row + tightest column. With R/C it belongs to + // cell[1] (R=0, C=1); spatial fill must now agree (single assignment, the + // go_bug #1 fix) instead of duplicating across all overlapping cells. cells := []pdf.TSRCell{ {X0: 0, Y0: 0, X1: 100, Y1: 30, Label: "table"}, {X0: 90, Y0: 0, X1: 200, Y1: 30, Label: "table"}, @@ -404,7 +406,7 @@ func TestFillCellText_RCOverSpatial(t *testing.T) { {X0: 30, X1: 270, Top: 0, Bottom: 30, Text: "TEXT", LayoutType: "table", R: 0, C: 1}, } - // Spatial fill: fills ALL overlapping cells → duplication. + // Spatial fill: assigns the box to the single tightest column (cell[1]). cellsCopy := make([]pdf.TSRCell, 3) copy(cellsCopy, cells) FillCellTextFromBoxes(cellsCopy, boxes) @@ -414,10 +416,10 @@ func TestFillCellText_RCOverSpatial(t *testing.T) { spatialCount++ } } - if spatialCount <= 1 { - t.Errorf("spatial fill: expected >1 cells with text, got %d", spatialCount) + if spatialCount != 1 { + t.Errorf("spatial fill: expected exactly 1 cell with text, got %d", spatialCount) } - t.Logf("spatial fill: %d cells (WRONG — duplication)", spatialCount) + t.Logf("spatial fill: %d cell (single assignment, matches R/C)", spatialCount) // R/C fill: only cell matching box.R/C gets text. cellsRC := make([]pdf.TSRCell, 3) diff --git a/internal/deepdoc/parser/pdf/table/testdata/parity/known_diffs.json b/internal/deepdoc/parser/pdf/table/testdata/parity/known_diffs.json new file mode 100644 index 0000000000..57a456586d --- /dev/null +++ b/internal/deepdoc/parser/pdf/table/testdata/parity/known_diffs.json @@ -0,0 +1,50 @@ +{ + "version": 1, + "rules": [ + { + "id": "table-cell-fill-filled-threshold-0.85", + "tag": "go_intentional", + "kind": "threshold", + "applies_to": ["*"], + "fields": ["BoxMatchesCell"], + "permanent": true, + "reason": "BoxMatchesCell (internal/deepdoc/parser/pdf/table/table_cells.go) uses a two-stage overlap threshold: empty cell -> inter/boxArea >= 0.3 (matches Python find_overlapped_with_threshold default thr=0.3, which fills cells from overlapping PDF boxes uniformly); filled cell -> >= 0.85. The 0.85 branch has NO Python equivalent. It is reached only in the rotated-table path: table_extract.go calls ocrTableCells (parser_ocr.go) to pre-fill each cell with per-cell OCR text BEFORE FillCellTextFromBoxes runs (table_extract.go), and the 0.85 bar stops a weakly-overlapping detected text box from corrupting/overriding that OCR result. Python has no per-cell OCR at this stage, so it never raises the threshold and would join a 30-85%-overlapping secondary box (e.g. cell='Total' + a box '元' overlapping 60% -> Python 'Total 元', Go 'Total'). This is a deliberate go_intentional divergence, not a regression target. It is locked by TestFillCellTextFromBoxes_PrefilledCellDropsSecondaryBox and TestBoxMatchesCell_FilledCellWeakOverlapRejected in table_cell_spatial_test.go; the Go-only rationale is documented in the BoxMatchesCell doc comment (table_cells.go). NOTE: if we later unify the pipeline to always run FillCellTextFromBoxes (empty cell, 0.3) before ocrTableCells fills only remaining empties — matching the non-rotated path — this threshold can be removed entirely and full parity with Python restored. Until then it stays permanent." + }, + { + "id": "table-cell-fill-multi-assignment", + "tag": "go_bug", + "kind": "assignment_cardinality", + "applies_to": ["*"], + "fields": ["FillCellTextFromBoxes", "BoxMatchesCell"], + "owner_fix_side": "go", + "status": "resolved", + "resolution": "FillCellTextFromBoxes now assigns each box to exactly ONE cell (best row by vertical-overlap ratio >= 0.3, tie-broken by inter/rowArea; then tightest column by horizontal edge/center distance). The many-to-many 2-D cell-overlap filter was removed. Regression: TestFillCellTextFromBoxes_BoxOverlappingTwoCells_SingleAssignment.", + "tracking": "Replicate Python's one-box-to-one-cell greedy assignment: for each box pick the best row (vertical overlap >= 0.3) and the tightest column, then assign to that single (row,column) cell.", + "reason": "FillCellTextFromBoxes (table_cells.go) matched each box against EVERY cross-product cell where inter/boxArea >= 0.3 and injected the box text into ALL of them (many-to-many threshold filter). Python's construct_table assigns each box to exactly ONE cell via greedy best row (find_overlapped_with_threshold, inter/boxArea >= 0.3) intersected with tightest column (find_horizontally_tightest_fit). A box straddling two cell boundaries was therefore DUPLICATED into both cells in Go, but appears once in Python. This was an implementation divergence in the box→cell assignment step (NOT an architecture difference — both sides build the grid from TSR rows×columns via cross-product; see deepdoc_table_builder.go GroupCells vs Python construct_table). Exposed by the now-fixed TestFillCellTextFromBoxes_BoxOverlappingTwoCells_SingleAssignment." + }, + { + "id": "table-cell-fill-column-algorithm", + "tag": "go_bug", + "kind": "match_primitive", + "applies_to": ["*"], + "fields": ["FillCellTextFromBoxes", "BoxMatchesCell"], + "owner_fix_side": "go", + "status": "resolved", + "resolution": "FillCellTextFromBoxes selects the target cell via best row (vertical-overlap ratio >= 0.3 on the full-width row strip) intersected with tightest column (find_horizontally_tightest_fit, no threshold). The single 2-D cell-intersection 0.3 test is gone. Regression: TestFillCellTextFromBoxes_PythonAssignsGoRejects_2DThreshold_FIXED.", + "tracking": "Replace the single 2-D cell-overlap test with Python's two-step assignment: best row by vertical overlap (inter/boxArea >= 0.3, since rows span full width) AND tightest column by horizontal edge/center distance (find_horizontally_tightest_fit, no threshold), then assign to that (row,column) cell.", + "reason": "Go tested the 0.3 threshold against the 2-D cell intersection (inter/boxArea on the cell rectangle). Python tests 0.3 against the 1-D VERTICAL overlap with the row (rows span the full table width, so overlapped_area(box,row)/boxArea ≈ vertical fraction) and chooses the column by find_horizontally_tightest_fit (vertical overlap required + minimal horizontal edge/center distance, NO threshold). Consequently a box Python assigns to (row,column) was REJECTED by Go when its 2-D cell intersection was < 30% of box area even though its vertical row overlap was >= 30% (e.g. a tall narrow box: row overlap 30%, column tightest, but 2-D = 30% * horizontal_fraction < 30%). Implementation divergence only (same grid construction on both sides). Exposed by the now-fixed TestFillCellTextFromBoxes_PythonAssignsGoRejects_2DThreshold_FIXED." + }, + { + "id": "table-cell-fill-no-best-match-tiebreak", + "tag": "go_bug", + "kind": "selection", + "applies_to": ["*"], + "fields": ["FillCellTextFromBoxes", "BoxMatchesCell"], + "owner_fix_side": "go", + "status": "resolved", + "resolution": "FillCellTextFromBoxes assigns each box to the single tightest column, so a box overlapping several cells lands in only the tightest one (matching Python, which keeps the single best via its ov/_ov ordering). Regression: TestFillCellTextFromBoxes_EqualBoxRatioSingleAssignment.", + "tracking": "When several cells qualify for a box at equal box-area ratio, keep only the single best (max inter/boxArea, tie-broken by inter/cellArea, mirroring Python's ov/_ov ordering) instead of filling all.", + "reason": "Python's find_overlapped_with_threshold orders candidates by (ov=inter/boxArea, _ov=inter/cellArea) and returns the single best, using cell-area ratio as a tie-break. Go's FillCellTextFromBoxes filled EVERY qualifying cell and never considered inter/cellArea. So a box overlapping several cells at the same box-area ratio was duplicated into all of them in Go, whereas Python keeps only the one it best fills (highest cell-area ratio). Implementation divergence only (same grid construction on both sides). Exposed by the now-fixed TestFillCellTextFromBoxes_EqualBoxRatioSingleAssignment." + } + ] +}