mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-08-25 01:20:11 +08:00
fix(deepdoc): assign each PDF box to a single table cell matching Python (#18410)
This commit is contained in:
@@ -110,3 +110,213 @@ func TestFillCellTextFromBoxes_NoMatchingBox(t *testing.T) {
|
||||
t.Errorf("no match: got %q, want empty", cells[0].Text)
|
||||
}
|
||||
}
|
||||
|
||||
// TestBoxMatchesCell_FilledCellWeakOverlapRejected locks the Go-only 0.85
|
||||
// guard: a cell that already carries text (e.g. per-cell OCR in the rotated
|
||||
// path) rejects a box whose area is only partially inside the cell.
|
||||
// Cell (0,0)-(100,50); box (40,5)-(140,15).
|
||||
//
|
||||
// overlap = (40,100)x(5,15) = 60*10 = 600
|
||||
// box area = (140-40)*(15-5) = 100*10 = 1000 → ratio 0.6
|
||||
//
|
||||
// 0.6 < 0.85 → rejected.
|
||||
func TestBoxMatchesCell_FilledCellWeakOverlapRejected(t *testing.T) {
|
||||
cell := pdf.TSRCell{X0: 0, Y0: 0, X1: 100, Y1: 50, Text: "ocr"}
|
||||
box := pdf.TextBox{X0: 40, X1: 140, Top: 5, Bottom: 15, Text: "元"}
|
||||
if BoxMatchesCell(cell, box, false) {
|
||||
t.Error("filled cell should reject box overlapping only 60% (needs >=85%)")
|
||||
}
|
||||
}
|
||||
|
||||
// TestBoxMatchesCell_FilledCellStrongOverlapAccepted locks that a box almost
|
||||
// entirely inside an already-filled cell still matches (so per-cell OCR text
|
||||
// can be overridden by a confident box). Box fully inside → ratio 1.0.
|
||||
func TestBoxMatchesCell_FilledCellStrongOverlapAccepted(t *testing.T) {
|
||||
cell := pdf.TSRCell{X0: 0, Y0: 0, X1: 100, Y1: 50, Text: "ocr"}
|
||||
box := pdf.TextBox{X0: 5, X1: 95, Top: 5, Bottom: 45, Text: "text"}
|
||||
if !BoxMatchesCell(cell, box, false) {
|
||||
t.Error("filled cell should accept box fully inside (>=85%)")
|
||||
}
|
||||
}
|
||||
|
||||
// TestFillCellTextFromBoxes_EmptyCellJoinsMultiplePartialBoxes proves the 0.85
|
||||
// guard only bites cells that ENTER FillCellTextFromBoxes with text. For a cell
|
||||
// that starts empty, every overlapping box (here each 60%) is matched at the
|
||||
// 0.3 threshold and all are joined — no in-cell text loss in the normal path.
|
||||
// cell (0,0)-(100,50); box1 (40,5)-(140,15) ratio 0.6; box2 (-40,30)-(60,45) ratio 0.6.
|
||||
func TestFillCellTextFromBoxes_EmptyCellJoinsMultiplePartialBoxes(t *testing.T) {
|
||||
cells := []pdf.TSRCell{{X0: 0, Y0: 0, X1: 100, Y1: 50}}
|
||||
boxes := []pdf.TextBox{
|
||||
{X0: 40, X1: 140, Top: 5, Bottom: 15, Text: "part1"},
|
||||
{X0: -40, X1: 60, Top: 30, Bottom: 45, Text: "part2"},
|
||||
}
|
||||
FillCellTextFromBoxes(cells, boxes)
|
||||
if cells[0].Text != "part1 part2" {
|
||||
t.Errorf("empty cell should join all overlapping boxes: got %q, want 'part1 part2'", cells[0].Text)
|
||||
}
|
||||
}
|
||||
|
||||
// TestFillCellTextFromBoxes_PrefilledCellDropsSecondaryBox documents the
|
||||
// deliberate go_intentional divergence from Python.
|
||||
//
|
||||
// Scenario: a cell already holds per-cell OCR text ("Total"). A separate
|
||||
// detected text box ("元") physically sits in the same cell but only overlaps
|
||||
// 60% of its area. Go applies the 0.85 guard (cell entered with text) and
|
||||
// DROPS the secondary box, keeping "Total".
|
||||
//
|
||||
// Python has no per-cell OCR and no filled-cell threshold: both fragments
|
||||
// would be joined via find_overlapped_with_threshold(thr=0.3), yielding
|
||||
// "Total 元". This test LOCKS Go's current behavior so the divergence stays
|
||||
// visible; it is registered as go_intentional, not a regression target.
|
||||
func TestFillCellTextFromBoxes_PrefilledCellDropsSecondaryBox(t *testing.T) {
|
||||
cells := []pdf.TSRCell{{X0: 0, Y0: 0, X1: 100, Y1: 50, Text: "Total"}} // pre-filled by per-cell OCR
|
||||
boxes := []pdf.TextBox{
|
||||
{X0: 40, X1: 140, Top: 5, Bottom: 15, Text: "元"}, // 60% inside
|
||||
}
|
||||
FillCellTextFromBoxes(cells, boxes)
|
||||
// Go: 0.85 guard drops the 60%-overlap box. Python would keep "Total 元".
|
||||
if cells[0].Text != "Total" {
|
||||
t.Errorf("go_intentional divergence: got %q, want 'Total' (Python would yield 'Total 元')", cells[0].Text)
|
||||
}
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// Implementation divergences in box→cell ASSIGNMENT (NOT architecture: both
|
||||
// sides build the grid from TSR rows×columns via cross-product — see
|
||||
// deepdoc_table_builder.go GroupCells vs Python construct_table). The divergences
|
||||
// are purely in how a box is assigned to a (row,column) cell. Registered in
|
||||
// testdata/parity/known_diffs.json as go_bug (#1, #2, #3).
|
||||
// =============================================================================
|
||||
|
||||
// TestFillCellTextFromBoxes_BoxOverlappingTwoCells_SingleAssignment is the
|
||||
// regression test for go_bug #1. Python assigns each box to EXACTLY ONE cell
|
||||
// (greedy best row + tightest column); Go must no longer duplicate a
|
||||
// straddling box into both cells.
|
||||
//
|
||||
// Two side-by-side cells; one box straddles the boundary, overlapping BOTH at
|
||||
// 50% of its area. cell A (0,0)-(100,50); cell B (100,0)-(200,50).
|
||||
// box (50,5)-(150,15): both columns are equally tight (dis=50), so Python
|
||||
// keeps the first (C0). The box lands in exactly ONE cell.
|
||||
func TestFillCellTextFromBoxes_BoxOverlappingTwoCells_SingleAssignment(t *testing.T) {
|
||||
cells := []pdf.TSRCell{
|
||||
{X0: 0, Y0: 0, X1: 100, Y1: 50},
|
||||
{X0: 100, Y0: 0, X1: 200, Y1: 50},
|
||||
}
|
||||
boxes := []pdf.TextBox{
|
||||
{X0: 50, X1: 150, Top: 5, Bottom: 15, Text: "shared"},
|
||||
}
|
||||
FillCellTextFromBoxes(cells, boxes)
|
||||
count := 0
|
||||
for _, c := range cells {
|
||||
if c.Text == "shared" {
|
||||
count++
|
||||
}
|
||||
}
|
||||
if count != 1 {
|
||||
t.Errorf("regression #1: box duplicated into %d cells; Python assigns it to exactly ONE cell", count)
|
||||
}
|
||||
}
|
||||
|
||||
// TestFillCellTextFromBoxes_PythonAssignsGoRejects_2DThreshold_FIXED is the
|
||||
// regression test for go_bug #2. Python's 0.3 tests the 1-D VERTICAL overlap
|
||||
// with the (full-width) row, and the column is chosen by
|
||||
// find_horizontally_tightest_fit (no threshold). Go must match: a box whose
|
||||
// row-vertical overlap is >= 0.3 and that is tightest to a column must be
|
||||
// assigned, even if its 2-D overlap with that single cell is < 0.3.
|
||||
//
|
||||
// Grid: row R0 Y=(0,30); col C0 X=(0,50), col C1 X=(50,200).
|
||||
// cell (R0,C0) = (0,0)-(50,30); cell (R0,C1) = (50,0)-(200,30).
|
||||
// box (0,0)-(100,100): vertical overlap with R0 = 30/100 = 30% (≥0.3 row
|
||||
// match); horizontally tightest to C0 (left edge aligns). Python → (R0,C0).
|
||||
func TestFillCellTextFromBoxes_PythonAssignsGoRejects_2DThreshold_FIXED(t *testing.T) {
|
||||
cells := []pdf.TSRCell{
|
||||
{X0: 0, Y0: 0, X1: 50, Y1: 30}, // (R0,C0)
|
||||
{X0: 50, Y0: 0, X1: 200, Y1: 30}, // (R0,C1)
|
||||
}
|
||||
boxes := []pdf.TextBox{
|
||||
{X0: 0, X1: 100, Top: 0, Bottom: 100, Text: "tall"},
|
||||
}
|
||||
FillCellTextFromBoxes(cells, boxes)
|
||||
if cells[0].Text != "tall" {
|
||||
t.Errorf("regression #2: Go dropped the box (cell0=%q); Python fills (R0,C0) via row-vertical-0.3 ∩ column-tightest", cells[0].Text)
|
||||
}
|
||||
if cells[1].Text != "" {
|
||||
t.Errorf("regression #2: box leaked into (R0,C1)=%q; should land only in (R0,C0)", cells[1].Text)
|
||||
}
|
||||
}
|
||||
|
||||
// TestFillCellTextFromBoxes_EqualBoxRatioSingleAssignment is the regression
|
||||
// test for go_bug #3. Go must no longer inject a box into ALL cells it
|
||||
// overlaps; it assigns the box to the single tightest column, so a box
|
||||
// overlapping a small cell A and a large cell B lands in ONLY A.
|
||||
//
|
||||
// Small cell A (0,0)-(50,25) area 1250; large cell B (50,0)-(200,50) area 7500.
|
||||
// box (0,0)-(100,25): overlaps both, but is tightest to A (left edge aligns,
|
||||
// dis=0 vs B's dis=50). Python keeps only A; Go must too.
|
||||
func TestFillCellTextFromBoxes_EqualBoxRatioSingleAssignment(t *testing.T) {
|
||||
cells := []pdf.TSRCell{
|
||||
{X0: 0, Y0: 0, X1: 50, Y1: 25}, // A (small)
|
||||
{X0: 50, Y0: 0, X1: 200, Y1: 50}, // B (large)
|
||||
}
|
||||
boxes := []pdf.TextBox{
|
||||
{X0: 0, X1: 100, Top: 0, Bottom: 25, Text: "wide"},
|
||||
}
|
||||
FillCellTextFromBoxes(cells, boxes)
|
||||
if cells[0].Text != "wide" {
|
||||
t.Errorf("regression #3: small cell A should contain 'wide', got %q", cells[0].Text)
|
||||
}
|
||||
if cells[1].Text == "wide" {
|
||||
t.Errorf("regression #3: box leaked into large cell B; Python keeps only small cell A")
|
||||
}
|
||||
}
|
||||
|
||||
// TestFillCellTextFromBoxes_RowSelectionIgnores2DCellOverlap strengthens the
|
||||
// go_bug #2 fix on a 2×2 grid. A tall box overlaps ONLY row R0 vertically
|
||||
// (≥0.3) but its 2-D intersection with every single cell is < 0.3, so the old
|
||||
// 2-D 0.3 filter dropped it entirely. The row/column selection must still fill
|
||||
// (R0,C0) — the tightest column in the matched row — and leave R1 empty.
|
||||
func TestFillCellTextFromBoxes_RowSelectionIgnores2DCellOverlap(t *testing.T) {
|
||||
cells := []pdf.TSRCell{
|
||||
{X0: 0, Y0: 0, X1: 50, Y1: 30}, // (R0,C0)
|
||||
{X0: 50, Y0: 0, X1: 200, Y1: 30}, // (R0,C1)
|
||||
{X0: 0, Y0: 30, X1: 50, Y1: 60}, // (R1,C0)
|
||||
{X0: 50, Y0: 30, X1: 200, Y1: 60}, // (R1,C1)
|
||||
}
|
||||
boxes := []pdf.TextBox{
|
||||
{X0: 0, X1: 100, Top: 0, Bottom: 100, Text: "tall"}, // vertical overlap with R0 = 30/100 = 0.3
|
||||
}
|
||||
FillCellTextFromBoxes(cells, boxes)
|
||||
if cells[0].Text != "tall" {
|
||||
t.Errorf("regression #2 (2x2): (R0,C0) should be 'tall', got %q", cells[0].Text)
|
||||
}
|
||||
if cells[1].Text != "" || cells[2].Text != "" || cells[3].Text != "" {
|
||||
t.Errorf("regression #2 (2x2): box leaked to R0C1/R1 = %q/%q/%q; must stay only in (R0,C0)",
|
||||
cells[1].Text, cells[2].Text, cells[3].Text)
|
||||
}
|
||||
}
|
||||
|
||||
// TestFillCellTextFromBoxes_RowSelectionTiebreak guards the row-selection
|
||||
// _ov tie-break (inter/rowArea) in FillCellTextFromBoxes. When a box overlaps
|
||||
// two rows with the SAME vertical-overlap ratio (ov), the row with the higher
|
||||
// _ov (smaller row area for equal vertical overlap) wins — mirroring Python's
|
||||
// (ov, _ov) ordering in find_overlapped_with_threshold.
|
||||
//
|
||||
// row0 y[0,40] (height 40), row1 y[40,100] (height 60). Box y[0,80] (height 80)
|
||||
// overlaps BOTH rows by 40 → ov = 40/80 = 0.5 for each. _ov: row0 = 40/40 = 1.0,
|
||||
// row1 = 40/60 ≈ 0.667. Python keeps row0; Go must too.
|
||||
func TestFillCellTextFromBoxes_RowSelectionTiebreak(t *testing.T) {
|
||||
cells := []pdf.TSRCell{
|
||||
{X0: 0, Y0: 0, X1: 100, Y1: 40}, // row0 (higher _ov)
|
||||
{X0: 0, Y0: 40, X1: 100, Y1: 100}, // row1 (lower _ov)
|
||||
}
|
||||
boxes := []pdf.TextBox{
|
||||
{X0: 0, X1: 100, Top: 0, Bottom: 80, Text: "row0"},
|
||||
}
|
||||
FillCellTextFromBoxes(cells, boxes)
|
||||
if cells[0].Text != "row0" {
|
||||
t.Errorf("row tiebreak: box should land in row0 (higher _ov), got cell0=%q", cells[0].Text)
|
||||
}
|
||||
if cells[1].Text != "" {
|
||||
t.Errorf("row tiebreak: box leaked into row1 (lower _ov), got %q", cells[1].Text)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -68,42 +68,206 @@ func GroupTSRCellsToRows(cells []pdf.TSRCell) [][]pdf.TSRCell {
|
||||
|
||||
// ── cell text filling ──────────────────────────────────────────────────
|
||||
|
||||
// FillCellTextFromBoxes assigns PDF text boxes to TSR grid cells, mirroring
|
||||
// Python's construct_table box→cell assignment (pdf_parser.py +
|
||||
// table_structure_recognizer.py):
|
||||
//
|
||||
// 1. For each box, pick the single BEST row by vertical-overlap ratio
|
||||
// inter(box,rowStrip)/area(box) >= 0.3, tie-broken by inter/rowArea
|
||||
// (Python find_overlapped_with_threshold on the full-width row strip).
|
||||
// 2. Within that row, pick the TIGHTEST column by horizontal edge/center
|
||||
// distance, requiring vertical overlap (Python find_horizontally_tightest_fit,
|
||||
// NO threshold). The box lands in exactly ONE cell (R,C).
|
||||
// 3. Multiple boxes mapped to the same (R,C) are concatenated (Python joins
|
||||
// them in construct_table).
|
||||
//
|
||||
// This replaces the old many-to-many 2-D cell-overlap filter
|
||||
// (inter(box,cell)/area(box) >= 0.3 on every cross-product cell), which
|
||||
// duplicated a straddling box into two cells (#1), dropped boxes whose 2-D
|
||||
// cell overlap was < 0.3 even though their row-vertical overlap was >= 0.3
|
||||
// (#2), and never applied the inter/cellArea tie-break (#3). All three are
|
||||
// go_bug in testdata/parity/known_diffs.json.
|
||||
//
|
||||
// The Go-only 0.85 guard (BoxMatchesCell) is retained for PRE-FILLED cells
|
||||
// only: if a cell already carries text (e.g. per-cell OCR in the rotated
|
||||
// path), a detected box overrides it only when it sits almost entirely inside
|
||||
// the cell (>= 0.85). Empty cells accept any box the row/column selection
|
||||
// picked, matching Python. See go_intentional rule
|
||||
// table-cell-fill-filled-threshold-0.85.
|
||||
func FillCellTextFromBoxes(cells []pdf.TSRCell, boxes []pdf.TextBox) {
|
||||
slog.Debug("fillCellTextFromBoxes", "cells", len(cells), "boxes", len(boxes))
|
||||
if len(cells) > 0 && len(boxes) > 0 {
|
||||
c0 := cells[0]
|
||||
slog.Debug("fillCellTextFromBoxes cell[0]", "x0", c0.X0, "y0", c0.Y0, "x1", c0.X1, "y1", c0.Y1)
|
||||
b0 := boxes[0]
|
||||
slog.Debug("fillCellTextFromBoxes box[0]", "x0", b0.X0, "y0", b0.Top, "x1", b0.X1, "y1", b0.Bottom, "text_len", len(b0.Text))
|
||||
if len(cells) == 0 || len(boxes) == 0 {
|
||||
return
|
||||
}
|
||||
matched, filled := 0, 0
|
||||
for ci := range cells {
|
||||
var matches []string
|
||||
for _, b := range boxes {
|
||||
if IsCaptionBox(b.Text, b.LayoutType) {
|
||||
|
||||
// Group cells into row bands by their top coordinate. The grid is a TSR
|
||||
// row×column cross-product, so every cell in a row shares the same Y band.
|
||||
// A row band spans the full table width (union of its cells), matching
|
||||
// Python's full-width "table row" components.
|
||||
type rowBand struct {
|
||||
y0, y1 float64
|
||||
stripX0 float64
|
||||
stripX1 float64
|
||||
cells []int // indices into `cells`
|
||||
}
|
||||
var rows []rowBand
|
||||
// Group cells into the same row band by exact top coordinate. A TSR
|
||||
// cross-product grid (GroupCells) assigns every cell in a row the SAME
|
||||
// Y0 value, and distinct rows differ by at least a row height, so a tiny
|
||||
// epsilon is enough and never merges two real rows.
|
||||
const yTol = 1e-6
|
||||
for i := range cells {
|
||||
c := &cells[i]
|
||||
if c.X1 <= c.X0 || c.Y1 <= c.Y0 {
|
||||
continue // degenerate / span-covered cell: not a fill target
|
||||
}
|
||||
rb := &rowBand{}
|
||||
found := false
|
||||
for ri := range rows {
|
||||
if math.Abs(rows[ri].y0-c.Y0) <= yTol {
|
||||
rb = &rows[ri]
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
rows = append(rows, rowBand{
|
||||
y0: c.Y0, y1: c.Y1, stripX0: c.X0, stripX1: c.X1,
|
||||
})
|
||||
rb = &rows[len(rows)-1]
|
||||
}
|
||||
if c.X0 < rb.stripX0 {
|
||||
rb.stripX0 = c.X0
|
||||
}
|
||||
if c.X1 > rb.stripX1 {
|
||||
rb.stripX1 = c.X1
|
||||
}
|
||||
if c.Y1 > rb.y1 {
|
||||
rb.y1 = c.Y1
|
||||
}
|
||||
rb.cells = append(rb.cells, i)
|
||||
}
|
||||
// Stable ordering: rows top-to-bottom, cells left-to-right (matches
|
||||
// Python's first-wins tie-breaking in find_overlapped_with_threshold /
|
||||
// find_horizontally_tightest_fit).
|
||||
sort.Slice(rows, func(i, j int) bool { return rows[i].y0 < rows[j].y0 })
|
||||
for ri := range rows {
|
||||
sort.Slice(rows[ri].cells, func(a, b int) bool {
|
||||
return cells[rows[ri].cells[a]].X0 < cells[rows[ri].cells[b]].X0
|
||||
})
|
||||
}
|
||||
|
||||
// Accumulate box text per target cell so multiple boxes in one cell join.
|
||||
cellText := make([]string, len(cells))
|
||||
cellFilled := make([]bool, len(cells))
|
||||
matched := 0
|
||||
|
||||
for bi := range boxes {
|
||||
b := boxes[bi]
|
||||
if IsCaptionBox(b.Text, b.LayoutType) {
|
||||
continue
|
||||
}
|
||||
boxArea := util.Area(&b)
|
||||
if boxArea <= 0 {
|
||||
continue
|
||||
}
|
||||
// 1. Best row by vertical-overlap ratio (>= 0.3), tie-broken by _ov.
|
||||
bestR := -1
|
||||
bestOv, bestOv2 := 0.3, 0.0
|
||||
for ri := range rows {
|
||||
rb := &rows[ri]
|
||||
if math.Min(b.Bottom, rb.y1)-math.Max(b.Top, rb.y0) <= 0 {
|
||||
continue // no vertical overlap
|
||||
}
|
||||
strip := pdf.TSRCell{X0: rb.stripX0, Y0: rb.y0, X1: rb.stripX1, Y1: rb.y1}
|
||||
inter := util.OverlapInter(&strip, &b)
|
||||
ov := inter / boxArea
|
||||
ov2 := 0.0
|
||||
if a := util.Area(&strip); a > 0 {
|
||||
ov2 = inter / a
|
||||
}
|
||||
// Skip unless strictly better than the current best, mirroring
|
||||
// Python's (ov, _ov) tuple ordering in find_overlapped_with_threshold.
|
||||
if !(ov > bestOv || (ov == bestOv && ov2 > bestOv2)) {
|
||||
continue
|
||||
}
|
||||
if BoxMatchesCell(cells[ci], b, cells[ci].Text == "") {
|
||||
matched++
|
||||
t := strings.TrimSpace(b.Text)
|
||||
if t != "" {
|
||||
matches = append(matches, t)
|
||||
}
|
||||
bestR, bestOv, bestOv2 = ri, ov, ov2
|
||||
}
|
||||
if bestR < 0 {
|
||||
continue
|
||||
}
|
||||
// 2. Tightest column within the matched row (no threshold).
|
||||
rb := &rows[bestR]
|
||||
bestC := -1
|
||||
bestDis := 1e9
|
||||
for _, ci := range rb.cells {
|
||||
c := &cells[ci]
|
||||
if math.Min(b.Bottom, c.Y1)-math.Max(b.Top, c.Y0) <= 0 {
|
||||
continue
|
||||
}
|
||||
if dis := tightestColumnDistance(&b, c); dis < bestDis {
|
||||
bestDis, bestC = dis, ci
|
||||
}
|
||||
}
|
||||
if len(matches) > 0 {
|
||||
cells[ci].Text = strings.Join(matches, " ")
|
||||
filled++
|
||||
if bestC < 0 {
|
||||
continue
|
||||
}
|
||||
// 3. Assign, preserving the 0.85 guard for pre-filled cells only.
|
||||
target := &cells[bestC]
|
||||
if target.Text != "" && !BoxMatchesCell(*target, b, false) {
|
||||
continue
|
||||
}
|
||||
t := strings.TrimSpace(b.Text)
|
||||
if t == "" {
|
||||
continue
|
||||
}
|
||||
if cellFilled[bestC] {
|
||||
cellText[bestC] += " " + t
|
||||
} else {
|
||||
cellText[bestC] = t
|
||||
cellFilled[bestC] = true
|
||||
}
|
||||
matched++
|
||||
}
|
||||
|
||||
for i := range cells {
|
||||
if cellFilled[i] {
|
||||
cells[i].Text = cellText[i]
|
||||
}
|
||||
}
|
||||
slog.Debug("fillCellTextFromBoxes done", "cell_box_matches", matched, "cells_filled", filled)
|
||||
slog.Debug("fillCellTextFromBoxes done", "box_cell_matches", matched, "cells_filled", matched)
|
||||
}
|
||||
|
||||
// boxMatchesCell reports whether a text box's text should be assigned
|
||||
// to a TSR cell. When the cell already has text (from TSR), the box
|
||||
// must be mostly inside the cell (≥85% of box area). When the cell
|
||||
// is empty, any overlap suffices — matching Python's _table_transformer_job
|
||||
// which fills cells from overlapping PDF boxes with thr=0.3.
|
||||
// tightestColumnDistance mirrors Python's find_horizontally_tightest_fit
|
||||
// distance metric: the minimum of the left-edge gap, right-edge gap, and
|
||||
// half the center gap. Smaller means the box sits tighter against the cell.
|
||||
func tightestColumnDistance(b *pdf.TextBox, c *pdf.TSRCell) float64 {
|
||||
dis := math.Min(math.Abs(b.X0-c.X0), math.Abs(b.X1-c.X1))
|
||||
if center := math.Abs((b.X0+b.X1)-(c.X0+c.X1)) / 2; center < dis {
|
||||
dis = center
|
||||
}
|
||||
return dis
|
||||
}
|
||||
|
||||
// BoxMatchesCell reports whether a text box's text may be assigned to a
|
||||
// TSR cell. The threshold is two-stage:
|
||||
// - empty cell: inter/boxArea >= 0.3 — matches Python's
|
||||
// find_overlapped_with_threshold default (thr=0.3), which fills cells from
|
||||
// overlapping PDF boxes uniformly.
|
||||
// - cell already has text: inter/boxArea >= 0.85 — Go-only guard, NOT in
|
||||
// Python. In the rotated-table path (table_extract.go) ocrTableCells
|
||||
// pre-fills cells with per-cell OCR text; the 0.85 bar stops a
|
||||
// weakly-overlapping detected box from corrupting/overriding that OCR
|
||||
// result. Python has no per-cell OCR at this stage, so it never raises the
|
||||
// threshold. This is a deliberate go_intentional divergence: it can drop a
|
||||
// legitimate secondary text fragment (a box overlapping 30-85%) that
|
||||
// Python would keep.
|
||||
//
|
||||
// FillCellTextFromBoxes uses only the 0.85 branch (cellIsEmpty=false) as the
|
||||
// guard for PRE-FILLED cells; for empty cells it relies on the row/column
|
||||
// selection (which already enforces >= 0.3 vertical overlap, matching Python).
|
||||
// BoxMatchesCell remains the canonical "does this box match this exact cell"
|
||||
// primitive and is directly unit-tested.
|
||||
func BoxMatchesCell(cell pdf.TSRCell, box pdf.TextBox, cellIsEmpty bool) bool {
|
||||
inter := util.OverlapInter(&cell, &box)
|
||||
boxArea := util.Area(&box)
|
||||
|
||||
@@ -393,8 +393,10 @@ func TestExtractTableAndReplace_OnlyTableBoxes(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestFillCellText_RCOverSpatial(t *testing.T) {
|
||||
// Box at X=30-270 overlaps all 3 cells (>30% each — spatial fills ALL).
|
||||
// With R/C, it belongs only to cell[1] (R=0, C=1).
|
||||
// Box at X=30-270 overlaps all 3 cells, but Python assigns it to exactly
|
||||
// ONE cell via greedy best row + tightest column. With R/C it belongs to
|
||||
// cell[1] (R=0, C=1); spatial fill must now agree (single assignment, the
|
||||
// go_bug #1 fix) instead of duplicating across all overlapping cells.
|
||||
cells := []pdf.TSRCell{
|
||||
{X0: 0, Y0: 0, X1: 100, Y1: 30, Label: "table"},
|
||||
{X0: 90, Y0: 0, X1: 200, Y1: 30, Label: "table"},
|
||||
@@ -404,7 +406,7 @@ func TestFillCellText_RCOverSpatial(t *testing.T) {
|
||||
{X0: 30, X1: 270, Top: 0, Bottom: 30, Text: "TEXT", LayoutType: "table", R: 0, C: 1},
|
||||
}
|
||||
|
||||
// Spatial fill: fills ALL overlapping cells → duplication.
|
||||
// Spatial fill: assigns the box to the single tightest column (cell[1]).
|
||||
cellsCopy := make([]pdf.TSRCell, 3)
|
||||
copy(cellsCopy, cells)
|
||||
FillCellTextFromBoxes(cellsCopy, boxes)
|
||||
@@ -414,10 +416,10 @@ func TestFillCellText_RCOverSpatial(t *testing.T) {
|
||||
spatialCount++
|
||||
}
|
||||
}
|
||||
if spatialCount <= 1 {
|
||||
t.Errorf("spatial fill: expected >1 cells with text, got %d", spatialCount)
|
||||
if spatialCount != 1 {
|
||||
t.Errorf("spatial fill: expected exactly 1 cell with text, got %d", spatialCount)
|
||||
}
|
||||
t.Logf("spatial fill: %d cells (WRONG — duplication)", spatialCount)
|
||||
t.Logf("spatial fill: %d cell (single assignment, matches R/C)", spatialCount)
|
||||
|
||||
// R/C fill: only cell matching box.R/C gets text.
|
||||
cellsRC := make([]pdf.TSRCell, 3)
|
||||
|
||||
50
internal/deepdoc/parser/pdf/table/testdata/parity/known_diffs.json
vendored
Normal file
50
internal/deepdoc/parser/pdf/table/testdata/parity/known_diffs.json
vendored
Normal file
@@ -0,0 +1,50 @@
|
||||
{
|
||||
"version": 1,
|
||||
"rules": [
|
||||
{
|
||||
"id": "table-cell-fill-filled-threshold-0.85",
|
||||
"tag": "go_intentional",
|
||||
"kind": "threshold",
|
||||
"applies_to": ["*"],
|
||||
"fields": ["BoxMatchesCell"],
|
||||
"permanent": true,
|
||||
"reason": "BoxMatchesCell (internal/deepdoc/parser/pdf/table/table_cells.go) uses a two-stage overlap threshold: empty cell -> inter/boxArea >= 0.3 (matches Python find_overlapped_with_threshold default thr=0.3, which fills cells from overlapping PDF boxes uniformly); filled cell -> >= 0.85. The 0.85 branch has NO Python equivalent. It is reached only in the rotated-table path: table_extract.go calls ocrTableCells (parser_ocr.go) to pre-fill each cell with per-cell OCR text BEFORE FillCellTextFromBoxes runs (table_extract.go), and the 0.85 bar stops a weakly-overlapping detected text box from corrupting/overriding that OCR result. Python has no per-cell OCR at this stage, so it never raises the threshold and would join a 30-85%-overlapping secondary box (e.g. cell='Total' + a box '元' overlapping 60% -> Python 'Total 元', Go 'Total'). This is a deliberate go_intentional divergence, not a regression target. It is locked by TestFillCellTextFromBoxes_PrefilledCellDropsSecondaryBox and TestBoxMatchesCell_FilledCellWeakOverlapRejected in table_cell_spatial_test.go; the Go-only rationale is documented in the BoxMatchesCell doc comment (table_cells.go). NOTE: if we later unify the pipeline to always run FillCellTextFromBoxes (empty cell, 0.3) before ocrTableCells fills only remaining empties — matching the non-rotated path — this threshold can be removed entirely and full parity with Python restored. Until then it stays permanent."
|
||||
},
|
||||
{
|
||||
"id": "table-cell-fill-multi-assignment",
|
||||
"tag": "go_bug",
|
||||
"kind": "assignment_cardinality",
|
||||
"applies_to": ["*"],
|
||||
"fields": ["FillCellTextFromBoxes", "BoxMatchesCell"],
|
||||
"owner_fix_side": "go",
|
||||
"status": "resolved",
|
||||
"resolution": "FillCellTextFromBoxes now assigns each box to exactly ONE cell (best row by vertical-overlap ratio >= 0.3, tie-broken by inter/rowArea; then tightest column by horizontal edge/center distance). The many-to-many 2-D cell-overlap filter was removed. Regression: TestFillCellTextFromBoxes_BoxOverlappingTwoCells_SingleAssignment.",
|
||||
"tracking": "Replicate Python's one-box-to-one-cell greedy assignment: for each box pick the best row (vertical overlap >= 0.3) and the tightest column, then assign to that single (row,column) cell.",
|
||||
"reason": "FillCellTextFromBoxes (table_cells.go) matched each box against EVERY cross-product cell where inter/boxArea >= 0.3 and injected the box text into ALL of them (many-to-many threshold filter). Python's construct_table assigns each box to exactly ONE cell via greedy best row (find_overlapped_with_threshold, inter/boxArea >= 0.3) intersected with tightest column (find_horizontally_tightest_fit). A box straddling two cell boundaries was therefore DUPLICATED into both cells in Go, but appears once in Python. This was an implementation divergence in the box→cell assignment step (NOT an architecture difference — both sides build the grid from TSR rows×columns via cross-product; see deepdoc_table_builder.go GroupCells vs Python construct_table). Exposed by the now-fixed TestFillCellTextFromBoxes_BoxOverlappingTwoCells_SingleAssignment."
|
||||
},
|
||||
{
|
||||
"id": "table-cell-fill-column-algorithm",
|
||||
"tag": "go_bug",
|
||||
"kind": "match_primitive",
|
||||
"applies_to": ["*"],
|
||||
"fields": ["FillCellTextFromBoxes", "BoxMatchesCell"],
|
||||
"owner_fix_side": "go",
|
||||
"status": "resolved",
|
||||
"resolution": "FillCellTextFromBoxes selects the target cell via best row (vertical-overlap ratio >= 0.3 on the full-width row strip) intersected with tightest column (find_horizontally_tightest_fit, no threshold). The single 2-D cell-intersection 0.3 test is gone. Regression: TestFillCellTextFromBoxes_PythonAssignsGoRejects_2DThreshold_FIXED.",
|
||||
"tracking": "Replace the single 2-D cell-overlap test with Python's two-step assignment: best row by vertical overlap (inter/boxArea >= 0.3, since rows span full width) AND tightest column by horizontal edge/center distance (find_horizontally_tightest_fit, no threshold), then assign to that (row,column) cell.",
|
||||
"reason": "Go tested the 0.3 threshold against the 2-D cell intersection (inter/boxArea on the cell rectangle). Python tests 0.3 against the 1-D VERTICAL overlap with the row (rows span the full table width, so overlapped_area(box,row)/boxArea ≈ vertical fraction) and chooses the column by find_horizontally_tightest_fit (vertical overlap required + minimal horizontal edge/center distance, NO threshold). Consequently a box Python assigns to (row,column) was REJECTED by Go when its 2-D cell intersection was < 30% of box area even though its vertical row overlap was >= 30% (e.g. a tall narrow box: row overlap 30%, column tightest, but 2-D = 30% * horizontal_fraction < 30%). Implementation divergence only (same grid construction on both sides). Exposed by the now-fixed TestFillCellTextFromBoxes_PythonAssignsGoRejects_2DThreshold_FIXED."
|
||||
},
|
||||
{
|
||||
"id": "table-cell-fill-no-best-match-tiebreak",
|
||||
"tag": "go_bug",
|
||||
"kind": "selection",
|
||||
"applies_to": ["*"],
|
||||
"fields": ["FillCellTextFromBoxes", "BoxMatchesCell"],
|
||||
"owner_fix_side": "go",
|
||||
"status": "resolved",
|
||||
"resolution": "FillCellTextFromBoxes assigns each box to the single tightest column, so a box overlapping several cells lands in only the tightest one (matching Python, which keeps the single best via its ov/_ov ordering). Regression: TestFillCellTextFromBoxes_EqualBoxRatioSingleAssignment.",
|
||||
"tracking": "When several cells qualify for a box at equal box-area ratio, keep only the single best (max inter/boxArea, tie-broken by inter/cellArea, mirroring Python's ov/_ov ordering) instead of filling all.",
|
||||
"reason": "Python's find_overlapped_with_threshold orders candidates by (ov=inter/boxArea, _ov=inter/cellArea) and returns the single best, using cell-area ratio as a tie-break. Go's FillCellTextFromBoxes filled EVERY qualifying cell and never considered inter/cellArea. So a box overlapping several cells at the same box-area ratio was duplicated into all of them in Go, whereas Python keeps only the one it best fills (highest cell-area ratio). Implementation divergence only (same grid construction on both sides). Exposed by the now-fixed TestFillCellTextFromBoxes_EqualBoxRatioSingleAssignment."
|
||||
}
|
||||
]
|
||||
}
|
||||
Reference in New Issue
Block a user