mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-08-20 23:35:06 +08:00
bd01c78d2b
## Problem In the Go PDF pipeline, `processTablesWithReplacements` (`internal/deepdoc/parser/pdf/table/table_post.go`) marked **every** box overlapping a DLA table region for removal, then `insertTableBoxes` re-inserted an HTML box only when `buildTableHTMLs` produced one. When DLA over-labels a **text** box as a table but TSR produces **no cells** (so `buildTableHTMLs` skips the table and `htmls[ti]` is unset), the box was removed with nothing re-inserted — the original text was **silently dropped**.
927 lines
31 KiB
Go
927 lines
31 KiB
Go
package table
|
||
|
||
import (
|
||
"strings"
|
||
"testing"
|
||
|
||
pdf "ragflow/internal/deepdoc/parser/pdf/type"
|
||
)
|
||
|
||
// ============================================
|
||
// Part 1: findTableAnchors tests
|
||
// ============================================
|
||
|
||
func TestFindTableAnchors_SingleTable(t *testing.T) {
|
||
boxes := []pdf.TextBox{
|
||
{Text: "before", LayoutType: pdf.LayoutTypeText, PageNumber: 0, X0: 10, X1: 100, Top: 10, Bottom: 50},
|
||
{Text: "table1", LayoutType: pdf.LayoutTypeTable, PageNumber: 0, X0: 10, X1: 400, Top: 60, Bottom: 200},
|
||
{Text: "after", LayoutType: pdf.LayoutTypeText, PageNumber: 0, X0: 10, X1: 100, Top: 210, Bottom: 250},
|
||
}
|
||
tables := []pdf.TableItem{
|
||
{
|
||
Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 10, Right: 400, Top: 60, Bottom: 200}},
|
||
RegionLeft: 10, RegionRight: 400, RegionTop: 60, RegionBottom: 200,
|
||
Cells: []pdf.TSRCell{{Text: "cell"}},
|
||
},
|
||
}
|
||
|
||
anchors := findTableAnchors(boxes, tables)
|
||
if len(anchors) != 1 {
|
||
t.Errorf("expected 1 anchor, got %d", len(anchors))
|
||
}
|
||
if anchors[0].pos != 1 {
|
||
t.Errorf("expected anchor at pos 1, got %d", anchors[0].pos)
|
||
}
|
||
}
|
||
|
||
func TestFindTableAnchors_NoBoxes(t *testing.T) {
|
||
boxes := []pdf.TextBox{}
|
||
tables := []pdf.TableItem{
|
||
{
|
||
Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 10, Right: 400, Top: 60, Bottom: 200}},
|
||
Cells: []pdf.TSRCell{{Text: "cell"}},
|
||
},
|
||
}
|
||
|
||
anchors := findTableAnchors(boxes, tables)
|
||
if len(anchors) != 0 {
|
||
t.Errorf("expected 0 anchors, got %d", len(anchors))
|
||
}
|
||
}
|
||
|
||
func TestFindTableAnchors_MultipleTables(t *testing.T) {
|
||
boxes := []pdf.TextBox{
|
||
{Text: "text1", LayoutType: pdf.LayoutTypeText, PageNumber: 0, X0: 10, X1: 100, Top: 10, Bottom: 30},
|
||
{Text: "table1", LayoutType: pdf.LayoutTypeTable, PageNumber: 0, X0: 10, X1: 400, Top: 40, Bottom: 100},
|
||
{Text: "text2", LayoutType: pdf.LayoutTypeText, PageNumber: 0, X0: 10, X1: 100, Top: 110, Bottom: 140},
|
||
{Text: "table2", LayoutType: pdf.LayoutTypeTable, PageNumber: 0, X0: 10, X1: 400, Top: 150, Bottom: 210},
|
||
}
|
||
tables := []pdf.TableItem{
|
||
{
|
||
Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 10, Right: 400, Top: 40, Bottom: 100}},
|
||
RegionLeft: 10, RegionRight: 400, RegionTop: 40, RegionBottom: 100,
|
||
Cells: []pdf.TSRCell{{Text: "cell1"}},
|
||
},
|
||
{
|
||
Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 10, Right: 400, Top: 150, Bottom: 210}},
|
||
RegionLeft: 10, RegionRight: 400, RegionTop: 150, RegionBottom: 210,
|
||
Cells: []pdf.TSRCell{{Text: "cell2"}},
|
||
},
|
||
}
|
||
|
||
anchors := findTableAnchors(boxes, tables)
|
||
if len(anchors) != 2 {
|
||
t.Errorf("expected 2 anchors, got %d", len(anchors))
|
||
}
|
||
}
|
||
|
||
func TestFindTableAnchors_AnchorAboveTable(t *testing.T) {
|
||
boxes := []pdf.TextBox{
|
||
{Text: "above", LayoutType: pdf.LayoutTypeText, PageNumber: 0, X0: 10, X1: 100, Top: 10, Bottom: 30},
|
||
{Text: "table", LayoutType: pdf.LayoutTypeTable, PageNumber: 0, X0: 10, X1: 400, Top: 40, Bottom: 100},
|
||
}
|
||
tables := []pdf.TableItem{
|
||
{
|
||
Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 10, Right: 400, Top: 40, Bottom: 100}},
|
||
RegionLeft: 10, RegionRight: 400, RegionTop: 40, RegionBottom: 100,
|
||
Cells: []pdf.TSRCell{{Text: "cell"}},
|
||
},
|
||
}
|
||
|
||
anchors := findTableAnchors(boxes, tables)
|
||
if len(anchors) != 1 {
|
||
t.Errorf("expected 1 anchor, got %d", len(anchors))
|
||
}
|
||
if anchors[0].pos != 1 {
|
||
t.Errorf("expected anchor at pos 1 (insert after above text), got %d", anchors[0].pos)
|
||
}
|
||
}
|
||
|
||
func TestFindTableAnchors_DifferentPage(t *testing.T) {
|
||
boxes := []pdf.TextBox{
|
||
{Text: "page0", LayoutType: pdf.LayoutTypeText, PageNumber: 0, X0: 10, X1: 100, Top: 10, Bottom: 30},
|
||
{Text: "page1", LayoutType: pdf.LayoutTypeText, PageNumber: 1, X0: 10, X1: 100, Top: 10, Bottom: 30},
|
||
}
|
||
tables := []pdf.TableItem{
|
||
{
|
||
Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 10, Right: 400, Top: 40, Bottom: 100}},
|
||
RegionLeft: 10, RegionRight: 400, RegionTop: 40, RegionBottom: 100,
|
||
Cells: []pdf.TSRCell{{Text: "cell"}},
|
||
},
|
||
}
|
||
|
||
anchors := findTableAnchors(boxes, tables)
|
||
if len(anchors) != 1 {
|
||
t.Errorf("expected 1 anchor, got %d", len(anchors))
|
||
}
|
||
// The text box is above the table, so pos is incremented to 1
|
||
if anchors[0].pos != 1 {
|
||
t.Errorf("expected anchor at pos 1, got %d", anchors[0].pos)
|
||
}
|
||
}
|
||
|
||
// ============================================
|
||
// Part 2: buildTableHTMLs tests
|
||
// ============================================
|
||
|
||
func TestBuildTableHTMLs_SingleTable(t *testing.T) {
|
||
boxes := []pdf.TextBox{}
|
||
tables := []pdf.TableItem{
|
||
{
|
||
Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 10, Right: 400, Top: 60, Bottom: 200}},
|
||
Scale: 1.0,
|
||
Cells: []pdf.TSRCell{
|
||
{X0: 0, Y0: 0, X1: 100, Y1: 50, Text: "A"},
|
||
{X0: 100, Y0: 0, X1: 200, Y1: 50, Text: "B"},
|
||
},
|
||
},
|
||
}
|
||
|
||
htmls := buildTableHTMLs(boxes, tables)
|
||
if len(htmls) != 1 {
|
||
t.Errorf("expected 1 HTML entry, got %d", len(htmls))
|
||
}
|
||
if htmls[0] == "" {
|
||
t.Error("expected non-empty HTML")
|
||
}
|
||
}
|
||
|
||
func TestBuildTableHTMLs_NoCells(t *testing.T) {
|
||
boxes := []pdf.TextBox{}
|
||
tables := []pdf.TableItem{
|
||
{
|
||
Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 10, Right: 400, Top: 60, Bottom: 200}},
|
||
Cells: []pdf.TSRCell{},
|
||
},
|
||
}
|
||
|
||
htmls := buildTableHTMLs(boxes, tables)
|
||
if len(htmls) != 0 {
|
||
t.Errorf("expected 0 HTML entries for no cells, got %d", len(htmls))
|
||
}
|
||
}
|
||
|
||
func TestBuildTableHTMLs_WithTableBoxes(t *testing.T) {
|
||
boxes := []pdf.TextBox{
|
||
{Text: "cell text", LayoutType: pdf.LayoutTypeTable, PageNumber: 0, X0: 10, X1: 100, Top: 60, Bottom: 100},
|
||
}
|
||
tables := []pdf.TableItem{
|
||
{
|
||
Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 10, Right: 400, Top: 60, Bottom: 200}},
|
||
Scale: 1.0,
|
||
Cells: []pdf.TSRCell{
|
||
{X0: 0, Y0: 0, X1: 100, Y1: 50, Text: "A"},
|
||
},
|
||
},
|
||
}
|
||
|
||
htmls := buildTableHTMLs(boxes, tables)
|
||
if len(htmls) != 1 {
|
||
t.Errorf("expected 1 HTML entry, got %d", len(htmls))
|
||
}
|
||
}
|
||
|
||
// ============================================
|
||
// Part 3: insertTableBoxes tests
|
||
// ============================================
|
||
|
||
func TestInsertTableBoxes_Basic(t *testing.T) {
|
||
boxes := []pdf.TextBox{
|
||
{Text: "before", PageNumber: 0},
|
||
{Text: "to replace", LayoutType: pdf.LayoutTypeTable, PageNumber: 0},
|
||
{Text: "after", PageNumber: 0},
|
||
}
|
||
removeSet := map[int]bool{1: true}
|
||
tables := []pdf.TableItem{
|
||
{
|
||
Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 10, Right: 400, Top: 60, Bottom: 200}},
|
||
RegionLeft: 10, RegionRight: 400, RegionTop: 60, RegionBottom: 200,
|
||
},
|
||
}
|
||
anchors := []struct{ ti, pos int }{{ti: 0, pos: 1}}
|
||
htmls := map[int]string{0: "<table>test</table>"}
|
||
|
||
result := insertTableBoxes(boxes, tables, removeSet, anchors, htmls)
|
||
if len(result) != 3 {
|
||
t.Errorf("expected 3 boxes (before + table + after), got %d", len(result))
|
||
}
|
||
if result[1].Text != "<table>test</table>" {
|
||
t.Errorf("expected table HTML at position 1, got %q", result[1].Text)
|
||
}
|
||
}
|
||
|
||
func TestInsertTableBoxes_NoRemove(t *testing.T) {
|
||
boxes := []pdf.TextBox{
|
||
{Text: "before", PageNumber: 0},
|
||
{Text: "after", PageNumber: 0},
|
||
}
|
||
removeSet := map[int]bool{}
|
||
tables := []pdf.TableItem{
|
||
{
|
||
Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 10, Right: 400, Top: 60, Bottom: 200}},
|
||
RegionLeft: 10, RegionRight: 400, RegionTop: 60, RegionBottom: 200,
|
||
},
|
||
}
|
||
anchors := []struct{ ti, pos int }{{ti: 0, pos: 1}}
|
||
htmls := map[int]string{0: "<table>test</table>"}
|
||
|
||
result := insertTableBoxes(boxes, tables, removeSet, anchors, htmls)
|
||
if len(result) != 3 {
|
||
t.Errorf("expected 3 boxes, got %d", len(result))
|
||
}
|
||
}
|
||
|
||
func TestInsertTableBoxes_AtEnd(t *testing.T) {
|
||
boxes := []pdf.TextBox{
|
||
{Text: "first", PageNumber: 0},
|
||
{Text: "second", PageNumber: 0},
|
||
}
|
||
removeSet := map[int]bool{}
|
||
tables := []pdf.TableItem{
|
||
{
|
||
Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 10, Right: 400, Top: 60, Bottom: 200}},
|
||
RegionLeft: 10, RegionRight: 400, RegionTop: 60, RegionBottom: 200,
|
||
},
|
||
}
|
||
anchors := []struct{ ti, pos int }{{ti: 0, pos: 2}}
|
||
htmls := map[int]string{0: "<table>end</table>"}
|
||
|
||
result := insertTableBoxes(boxes, tables, removeSet, anchors, htmls)
|
||
if len(result) != 3 {
|
||
t.Errorf("expected 3 boxes, got %d", len(result))
|
||
}
|
||
if result[2].Text != "<table>end</table>" {
|
||
t.Errorf("expected table at end, got %q", result[2].Text)
|
||
}
|
||
}
|
||
|
||
func TestInsertTableBoxes_MultipleAnchors(t *testing.T) {
|
||
boxes := []pdf.TextBox{
|
||
{Text: "1", PageNumber: 0},
|
||
{Text: "2", PageNumber: 0},
|
||
{Text: "3", PageNumber: 0},
|
||
}
|
||
removeSet := map[int]bool{}
|
||
tables := []pdf.TableItem{
|
||
{
|
||
Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 10, Right: 400, Top: 60, Bottom: 200}},
|
||
RegionLeft: 10, RegionRight: 400, RegionTop: 60, RegionBottom: 200,
|
||
},
|
||
{
|
||
Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 10, Right: 400, Top: 210, Bottom: 350}},
|
||
RegionLeft: 10, RegionRight: 400, RegionTop: 210, RegionBottom: 350,
|
||
},
|
||
}
|
||
anchors := []struct{ ti, pos int }{{ti: 0, pos: 1}, {ti: 1, pos: 3}}
|
||
htmls := map[int]string{0: "<table>A</table>", 1: "<table>B</table>"}
|
||
|
||
result := insertTableBoxes(boxes, tables, removeSet, anchors, htmls)
|
||
if len(result) != 5 {
|
||
t.Errorf("expected 5 boxes, got %d", len(result))
|
||
}
|
||
if result[1].Text != "<table>A</table>" {
|
||
t.Errorf("expected table A at pos 1")
|
||
}
|
||
if result[4].Text != "<table>B</table>" {
|
||
t.Errorf("expected table B at pos 4")
|
||
}
|
||
}
|
||
|
||
func TestInsertTableBoxes_EmptyHTML(t *testing.T) {
|
||
boxes := []pdf.TextBox{{Text: "text", PageNumber: 0}}
|
||
removeSet := map[int]bool{}
|
||
tables := []pdf.TableItem{
|
||
{
|
||
Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 10, Right: 400, Top: 60, Bottom: 200}},
|
||
RegionLeft: 10, RegionRight: 400, RegionTop: 60, RegionBottom: 200,
|
||
},
|
||
}
|
||
anchors := []struct{ ti, pos int }{{ti: 0, pos: 1}}
|
||
htmls := map[int]string{0: ""}
|
||
|
||
result := insertTableBoxes(boxes, tables, removeSet, anchors, htmls)
|
||
if len(result) != 1 {
|
||
t.Errorf("expected 1 box (no empty HTML inserted), got %d", len(result))
|
||
}
|
||
}
|
||
|
||
// ============================================
|
||
// Part 4: Integration tests - verify refactored behavior remains consistent
|
||
// ============================================
|
||
|
||
func TestExtractTableAndReplace_Integration(t *testing.T) {
|
||
boxes := []pdf.TextBox{
|
||
{Text: "intro", LayoutType: pdf.LayoutTypeText, PageNumber: 0, X0: 10, X1: 100, Top: 10, Bottom: 30},
|
||
{Text: "table box", LayoutType: pdf.LayoutTypeTable, PageNumber: 0, X0: 10, X1: 400, Top: 40, Bottom: 150},
|
||
{Text: "outro", LayoutType: pdf.LayoutTypeText, PageNumber: 0, X0: 10, X1: 100, Top: 160, Bottom: 190},
|
||
}
|
||
tables := []pdf.TableItem{
|
||
{
|
||
Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 10, Right: 400, Top: 40, Bottom: 150}},
|
||
RegionLeft: 10, RegionRight: 400, RegionTop: 40, RegionBottom: 150,
|
||
Scale: 1.0,
|
||
Cells: []pdf.TSRCell{
|
||
{X0: 0, Y0: 0, X1: 100, Y1: 50, Text: "A"},
|
||
{X0: 100, Y0: 0, X1: 200, Y1: 50, Text: "B"},
|
||
},
|
||
},
|
||
}
|
||
|
||
result := ExtractTableAndReplace(boxes, tables)
|
||
if len(result) != 3 {
|
||
t.Errorf("expected 3 boxes, got %d", len(result))
|
||
}
|
||
}
|
||
|
||
func TestExtractTableAndReplace_NoTables(t *testing.T) {
|
||
boxes := []pdf.TextBox{
|
||
{Text: "text1", PageNumber: 0},
|
||
{Text: "text2", PageNumber: 0},
|
||
}
|
||
tables := []pdf.TableItem{}
|
||
|
||
result := ExtractTableAndReplace(boxes, tables)
|
||
if len(result) != 2 {
|
||
t.Errorf("expected 2 boxes unchanged, got %d", len(result))
|
||
}
|
||
}
|
||
|
||
func TestExtractTableAndReplace_DataSourceBox(t *testing.T) {
|
||
boxes := []pdf.TextBox{
|
||
{Text: "数据来源: somewhere", LayoutType: pdf.LayoutTypeTable, PageNumber: 0},
|
||
{Text: "normal text", LayoutType: pdf.LayoutTypeText, PageNumber: 0},
|
||
}
|
||
tables := []pdf.TableItem{}
|
||
|
||
result := ExtractTableAndReplace(boxes, tables)
|
||
if len(result) != 1 {
|
||
t.Errorf("expected 1 box (data source removed), got %d", len(result))
|
||
}
|
||
if result[0].Text != "normal text" {
|
||
t.Errorf("expected normal text to remain, got %q", result[0].Text)
|
||
}
|
||
}
|
||
|
||
func TestExtractTableAndReplace_ZeroBoxesWithTables(t *testing.T) {
|
||
boxes := []pdf.TextBox{}
|
||
tables := []pdf.TableItem{
|
||
{
|
||
Scale: 1.0,
|
||
Cells: []pdf.TSRCell{
|
||
{X0: 0, Y0: 0, X1: 100, Y1: 50, Text: "A"},
|
||
},
|
||
},
|
||
}
|
||
|
||
result := ExtractTableAndReplace(boxes, tables)
|
||
if len(result) != 1 {
|
||
t.Errorf("expected 1 table box for zero input boxes, got %d", len(result))
|
||
}
|
||
}
|
||
|
||
// ============================================
|
||
// Part 5: FilterBoxesByRemoveSet unit tests
|
||
// ============================================
|
||
|
||
func TestFilterBoxesByRemoveSet_EmptyRemoveSet(t *testing.T) {
|
||
boxes := []pdf.TextBox{{Text: "a"}, {Text: "b"}, {Text: "c"}}
|
||
removeSet := map[int]bool{}
|
||
|
||
result := FilterBoxesByRemoveSet(boxes, removeSet)
|
||
if len(result) != 3 {
|
||
t.Errorf("expected all boxes to remain, got %d", len(result))
|
||
}
|
||
}
|
||
|
||
func TestFilterBoxesByRemoveSet_RemoveSome(t *testing.T) {
|
||
boxes := []pdf.TextBox{{Text: "keep0"}, {Text: "remove1"}, {Text: "keep2"}, {Text: "remove3"}}
|
||
removeSet := map[int]bool{1: true, 3: true}
|
||
|
||
result := FilterBoxesByRemoveSet(boxes, removeSet)
|
||
if len(result) != 2 {
|
||
t.Errorf("expected 2 boxes, got %d", len(result))
|
||
}
|
||
if result[0].Text != "keep0" || result[1].Text != "keep2" {
|
||
t.Errorf("unexpected filtered result: %+v", result)
|
||
}
|
||
}
|
||
|
||
func TestFilterBoxesByRemoveSet_RemoveAll(t *testing.T) {
|
||
boxes := []pdf.TextBox{{Text: "a"}, {Text: "b"}}
|
||
removeSet := map[int]bool{0: true, 1: true}
|
||
|
||
result := FilterBoxesByRemoveSet(boxes, removeSet)
|
||
if len(result) != 0 {
|
||
t.Errorf("expected empty result, got %d", len(result))
|
||
}
|
||
}
|
||
|
||
func TestFilterBoxesByRemoveSet_EmptyInput(t *testing.T) {
|
||
var boxes []pdf.TextBox
|
||
removeSet := map[int]bool{0: true}
|
||
|
||
result := FilterBoxesByRemoveSet(boxes, removeSet)
|
||
if len(result) != 0 {
|
||
t.Errorf("expected empty result for empty input, got %d", len(result))
|
||
}
|
||
}
|
||
|
||
func TestFilterBoxesByRemoveSet_Preallocation(t *testing.T) {
|
||
// Verify that capacity pre-allocation is reasonable
|
||
boxes := make([]pdf.TextBox, 100)
|
||
removeSet := map[int]bool{}
|
||
for i := 0; i < 30; i++ {
|
||
removeSet[i] = true // Mark 30 entries for removal
|
||
}
|
||
|
||
result := FilterBoxesByRemoveSet(boxes, removeSet)
|
||
if len(result) != 70 {
|
||
t.Errorf("expected 70 boxes, got %d", len(result))
|
||
}
|
||
// Verify capacity is at least 70
|
||
if cap(result) < 70 {
|
||
t.Errorf("expected capacity >= 70, got %d", cap(result))
|
||
}
|
||
}
|
||
|
||
// ============================================
|
||
// Part 6: createTableBoxFromItem unit tests
|
||
// ============================================
|
||
|
||
func TestCreateTableBoxFromItem_Basic(t *testing.T) {
|
||
table := &pdf.TableItem{
|
||
RegionLeft: 10,
|
||
RegionRight: 400,
|
||
RegionTop: 60,
|
||
RegionBottom: 200,
|
||
Positions: []pdf.Position{{
|
||
PageNumbers: []int{1},
|
||
}},
|
||
}
|
||
|
||
box := createTableBoxFromItem(table, "<table>test</table>")
|
||
if box.Text != "<table>test</table>" {
|
||
t.Errorf("expected HTML text, got %q", box.Text)
|
||
}
|
||
if box.LayoutType != pdf.LayoutTypeTable {
|
||
t.Errorf("expected table layout, got %v", box.LayoutType)
|
||
}
|
||
if box.PageNumber != 1 {
|
||
t.Errorf("expected page 1, got %d", box.PageNumber)
|
||
}
|
||
if box.X0 != 10 || box.X1 != 400 || box.Top != 60 || box.Bottom != 200 {
|
||
t.Errorf("expected correct coordinates, got (%.0f,%.0f,%.0f,%.0f)", box.X0, box.X1, box.Top, box.Bottom)
|
||
}
|
||
}
|
||
|
||
func TestCreateTableBoxFromItem_FallbackToPosition(t *testing.T) {
|
||
// When Region fields are empty, use the Position coordinates
|
||
table := &pdf.TableItem{
|
||
Positions: []pdf.Position{{
|
||
PageNumbers: []int{2},
|
||
Left: 20,
|
||
Right: 300,
|
||
Top: 50,
|
||
Bottom: 150,
|
||
}},
|
||
}
|
||
|
||
box := createTableBoxFromItem(table, "<table>fallback</table>")
|
||
if box.X0 != 20 || box.X1 != 300 || box.Top != 50 || box.Bottom != 150 {
|
||
t.Errorf("expected fallback coordinates, got (%.0f,%.0f,%.0f,%.0f)", box.X0, box.X1, box.Top, box.Bottom)
|
||
}
|
||
}
|
||
|
||
func TestCreateTableBoxFromItem_EmptyPositions(t *testing.T) {
|
||
// Also works when Positions is empty
|
||
table := &pdf.TableItem{
|
||
RegionLeft: 10,
|
||
RegionRight: 100,
|
||
RegionTop: 10,
|
||
RegionBottom: 100,
|
||
}
|
||
|
||
box := createTableBoxFromItem(table, "<table>empty-pos</table>")
|
||
if box.PageNumber != 0 {
|
||
t.Errorf("expected page 0, got %d", box.PageNumber)
|
||
}
|
||
}
|
||
|
||
// ============================================
|
||
// Part 7: handleImageOnlyPDFs unit tests
|
||
// ============================================
|
||
|
||
func TestHandleImageOnlyPDFs_EmptyTables(t *testing.T) {
|
||
result := handleImageOnlyPDFs([]pdf.TableItem{})
|
||
if len(result) != 0 {
|
||
t.Errorf("expected empty result, got %d boxes", len(result))
|
||
}
|
||
}
|
||
|
||
func TestHandleImageOnlyPDFs_EmptyCells(t *testing.T) {
|
||
tables := []pdf.TableItem{
|
||
{Cells: []pdf.TSRCell{}}, // Table with no cells
|
||
}
|
||
result := handleImageOnlyPDFs(tables)
|
||
if len(result) != 0 {
|
||
t.Errorf("expected no boxes for empty cells, got %d", len(result))
|
||
}
|
||
}
|
||
|
||
func TestHandleImageOnlyPDFs_SingleTable(t *testing.T) {
|
||
tables := []pdf.TableItem{
|
||
{
|
||
Scale: 1.0,
|
||
CropOffX: 0,
|
||
CropOffY: 0,
|
||
RegionLeft: 10,
|
||
RegionRight: 200,
|
||
RegionTop: 20,
|
||
RegionBottom: 100,
|
||
Positions: []pdf.Position{{PageNumbers: []int{0}}},
|
||
Cells: []pdf.TSRCell{
|
||
{X0: 0, Y0: 0, X1: 100, Y1: 50, Text: "cell1"},
|
||
},
|
||
},
|
||
}
|
||
result := handleImageOnlyPDFs(tables)
|
||
if len(result) != 1 {
|
||
t.Errorf("expected 1 box, got %d", len(result))
|
||
}
|
||
if result[0].LayoutType != pdf.LayoutTypeTable {
|
||
t.Error("expected table layout type")
|
||
}
|
||
}
|
||
|
||
func TestHandleImageOnlyPDFs_MultipleTables(t *testing.T) {
|
||
tables := []pdf.TableItem{
|
||
{
|
||
Scale: 1.0,
|
||
RegionLeft: 10, RegionRight: 200,
|
||
RegionTop: 20, RegionBottom: 100,
|
||
Positions: []pdf.Position{{PageNumbers: []int{0}}},
|
||
Cells: []pdf.TSRCell{{Text: "table1"}},
|
||
},
|
||
{
|
||
// Table with no cells, should be skipped
|
||
Cells: []pdf.TSRCell{},
|
||
},
|
||
{
|
||
Scale: 1.0,
|
||
RegionLeft: 10, RegionRight: 200,
|
||
RegionTop: 120, RegionBottom: 200,
|
||
Positions: []pdf.Position{{PageNumbers: []int{1}}},
|
||
Cells: []pdf.TSRCell{{Text: "table2"}},
|
||
},
|
||
}
|
||
result := handleImageOnlyPDFs(tables)
|
||
if len(result) != 2 {
|
||
t.Errorf("expected 2 boxes, got %d", len(result))
|
||
}
|
||
}
|
||
|
||
// ============================================
|
||
// Phase 2: buildAndSortAnchors and processTablesWithReplacements
|
||
// ============================================
|
||
|
||
func TestBuildAndSortAnchors(t *testing.T) {
|
||
boxes := []pdf.TextBox{
|
||
{Text: "text1", PageNumber: 0, Top: 10},
|
||
{Text: "table1", LayoutType: pdf.LayoutTypeTable, PageNumber: 0, Top: 50},
|
||
{Text: "text2", PageNumber: 0, Top: 100},
|
||
}
|
||
tables := []pdf.TableItem{
|
||
{
|
||
Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 10, Right: 400, Top: 50, Bottom: 80}},
|
||
RegionLeft: 10, RegionRight: 400, RegionTop: 50, RegionBottom: 80,
|
||
Cells: []pdf.TSRCell{{Text: "cell1"}},
|
||
},
|
||
}
|
||
_, replacements := buildReplacements(boxes, tables)
|
||
|
||
anchors := findTableAnchorsWithReplacements(boxes, tables, replacements)
|
||
result := buildAndSortAnchors(anchors)
|
||
|
||
if len(result) != 1 {
|
||
t.Errorf("expected 1 anchor, got %d", len(result))
|
||
}
|
||
}
|
||
|
||
// ============================================
|
||
// Part 8: ConsolidateFigures subfunction unit tests
|
||
// ============================================
|
||
|
||
func TestMarkDataSourceBoxesForRemoval(t *testing.T) {
|
||
boxes := []pdf.TextBox{
|
||
{Text: "数据来源: test", LayoutType: pdf.LayoutTypeFigure, PageNumber: 0},
|
||
{Text: "资料来源：abc", LayoutType: pdf.LayoutTypeFigure, PageNumber: 0},
|
||
{Text: "图表来源 def", LayoutType: pdf.LayoutTypeFigure, PageNumber: 0},
|
||
{Text: "正常图片内容", LayoutType: pdf.LayoutTypeFigure, PageNumber: 0},
|
||
{Text: "数据来源: 不应该移除", LayoutType: pdf.LayoutTypeText, PageNumber: 0}, // Not a figure type
|
||
}
|
||
|
||
removeSet := markDataSourceBoxesForRemoval(boxes)
|
||
if len(removeSet) != 3 {
|
||
t.Errorf("expected 3 boxes marked for removal, got %d", len(removeSet))
|
||
}
|
||
if !removeSet[0] || !removeSet[1] || !removeSet[2] {
|
||
t.Error("expected boxes 0, 1, 2 to be marked for removal")
|
||
}
|
||
if removeSet[3] || removeSet[4] {
|
||
t.Error("expected boxes 3 and 4 NOT to be marked")
|
||
}
|
||
}
|
||
|
||
func TestGroupFigureBoxes(t *testing.T) {
|
||
boxes := []pdf.TextBox{
|
||
{Text: "fig1-part1", LayoutType: pdf.LayoutTypeFigure, PageNumber: 0, LayoutNo: "fig-0"},
|
||
{Text: "fig1-part2", LayoutType: pdf.LayoutTypeFigure, PageNumber: 0, LayoutNo: "fig-0"},
|
||
{Text: "fig2", LayoutType: pdf.LayoutTypeFigure, PageNumber: 0, LayoutNo: "fig-1"},
|
||
{Text: "fig3", LayoutType: pdf.LayoutTypeFigure, PageNumber: 1, LayoutNo: "fig-0"}, // Different page
|
||
{Text: "text", LayoutType: pdf.LayoutTypeText, PageNumber: 0},
|
||
}
|
||
removeSet := map[int]bool{}
|
||
|
||
groups := groupFigureBoxes(boxes, removeSet)
|
||
if len(groups) != 3 {
|
||
t.Errorf("expected 3 groups, got %d", len(groups))
|
||
}
|
||
|
||
// Verify the group's contents
|
||
key1 := figKey{page: 0, ln: "fig-0"}
|
||
if len(groups[key1]) != 2 {
|
||
t.Errorf("expected 2 boxes in fig-0 group, got %d", len(groups[key1]))
|
||
}
|
||
key2 := figKey{page: 0, ln: "fig-1"}
|
||
if len(groups[key2]) != 1 {
|
||
t.Errorf("expected 1 box in fig-1 group, got %d", len(groups[key2]))
|
||
}
|
||
key3 := figKey{page: 1, ln: "fig-0"}
|
||
if len(groups[key3]) != 1 {
|
||
t.Errorf("expected 1 box in page 1 fig-0 group, got %d", len(groups[key3]))
|
||
}
|
||
}
|
||
|
||
func TestMergeFigureGroups(t *testing.T) {
|
||
boxes := []pdf.TextBox{
|
||
{Text: "part1", LayoutType: pdf.LayoutTypeFigure, PageNumber: 0, LayoutNo: "fig-0",
|
||
X0: 10, X1: 100, Top: 10, Bottom: 50},
|
||
{Text: "part2", LayoutType: pdf.LayoutTypeFigure, PageNumber: 0, LayoutNo: "fig-0",
|
||
X0: 50, X1: 150, Top: 30, Bottom: 80},
|
||
{Text: "single", LayoutType: pdf.LayoutTypeFigure, PageNumber: 0, LayoutNo: "fig-1",
|
||
X0: 200, X1: 300, Top: 10, Bottom: 50},
|
||
}
|
||
removeSet := make(map[int]bool)
|
||
groups := map[figKey][]int{
|
||
{page: 0, ln: "fig-0"}: {0, 1},
|
||
{page: 0, ln: "fig-1"}: {2},
|
||
}
|
||
|
||
mergeFigureGroups(boxes, groups, removeSet)
|
||
|
||
// Verify the merged result
|
||
if boxes[0].Text != "part1\npart2" {
|
||
t.Errorf("expected merged text, got %q", boxes[0].Text)
|
||
}
|
||
if boxes[0].X0 != 10 || boxes[0].X1 != 150 || boxes[0].Top != 10 || boxes[0].Bottom != 80 {
|
||
t.Error("expected merged bounding box")
|
||
}
|
||
if !removeSet[1] {
|
||
t.Error("expected box 1 to be marked for removal")
|
||
}
|
||
if removeSet[2] {
|
||
t.Error("expected single box NOT to be marked for removal")
|
||
}
|
||
}
|
||
|
||
func TestConsolidateFigures_Integration(t *testing.T) {
|
||
boxes := []pdf.TextBox{
|
||
{Text: "数据来源: test", LayoutType: pdf.LayoutTypeFigure, PageNumber: 0, LayoutNo: "fig-0"},
|
||
{Text: "fig1-part1", LayoutType: pdf.LayoutTypeFigure, PageNumber: 0, LayoutNo: "fig-0",
|
||
X0: 10, X1: 100, Top: 10, Bottom: 50},
|
||
{Text: "fig1-part2", LayoutType: pdf.LayoutTypeFigure, PageNumber: 0, LayoutNo: "fig-0",
|
||
X0: 50, X1: 150, Top: 30, Bottom: 80},
|
||
{Text: "normal text", LayoutType: pdf.LayoutTypeText, PageNumber: 0},
|
||
}
|
||
|
||
result := ConsolidateFigures(boxes)
|
||
|
||
// Verify result
|
||
if len(result) != 2 { // Merged figure + normal text
|
||
t.Errorf("expected 2 boxes, got %d", len(result))
|
||
}
|
||
|
||
// Check that figures are correctly merged
|
||
var figureFound bool
|
||
for _, b := range result {
|
||
if b.LayoutType == pdf.LayoutTypeFigure {
|
||
figureFound = true
|
||
if b.Text != "fig1-part1\nfig1-part2" {
|
||
t.Errorf("expected merged figure text, got %q", b.Text)
|
||
}
|
||
}
|
||
}
|
||
if !figureFound {
|
||
t.Error("expected figure box in result")
|
||
}
|
||
}
|
||
|
||
func TestConsolidateFigures_NoFigures(t *testing.T) {
|
||
boxes := []pdf.TextBox{
|
||
{Text: "text1", LayoutType: pdf.LayoutTypeText, PageNumber: 0},
|
||
{Text: "text2", LayoutType: pdf.LayoutTypeText, PageNumber: 0},
|
||
}
|
||
|
||
result := ConsolidateFigures(boxes)
|
||
if len(result) != 2 {
|
||
t.Errorf("expected 2 boxes unchanged, got %d", len(result))
|
||
}
|
||
}
|
||
|
||
func TestConsolidateFigures_OnlyDataSource(t *testing.T) {
|
||
boxes := []pdf.TextBox{
|
||
{Text: "数据来源: test", LayoutType: pdf.LayoutTypeFigure, PageNumber: 0},
|
||
}
|
||
|
||
result := ConsolidateFigures(boxes)
|
||
if len(result) != 0 {
|
||
t.Errorf("expected 0 boxes (data source removed), got %d", len(result))
|
||
}
|
||
}
|
||
|
||
func TestExtractTableAndReplace_MergeTablesAcrossPages(t *testing.T) {
|
||
// Regression test: two tables on consecutive pages with overlapping X
|
||
// should be merged by MergeTablesAcrossPages, and buildReplacementsAfterMerge
|
||
// must correctly index into the merged slice (not the original pre-merge slice).
|
||
boxes := []pdf.TextBox{
|
||
{Text: "intro", LayoutType: pdf.LayoutTypeText, PageNumber: 0, X0: 10, X1: 100, Top: 10, Bottom: 30},
|
||
{Text: "table1", LayoutType: pdf.LayoutTypeTable, PageNumber: 0, X0: 10, X1: 400, Top: 40, Bottom: 150},
|
||
{Text: "middle", LayoutType: pdf.LayoutTypeText, PageNumber: 0, X0: 10, X1: 100, Top: 160, Bottom: 190},
|
||
{Text: "table2", LayoutType: pdf.LayoutTypeTable, PageNumber: 1, X0: 10, X1: 400, Top: 10, Bottom: 120},
|
||
{Text: "outro", LayoutType: pdf.LayoutTypeText, PageNumber: 1, X0: 10, X1: 100, Top: 130, Bottom: 160},
|
||
}
|
||
tables := []pdf.TableItem{
|
||
{
|
||
Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 10, Right: 400, Top: 40, Bottom: 150}},
|
||
RegionLeft: 10, RegionRight: 400, RegionTop: 40, RegionBottom: 150,
|
||
Scale: 1.0,
|
||
Cells: []pdf.TSRCell{
|
||
{X0: 0, Y0: 0, X1: 100, Y1: 30, Text: "Page0_A"},
|
||
{X0: 100, Y0: 0, X1: 200, Y1: 30, Text: "Page0_B"},
|
||
},
|
||
},
|
||
{
|
||
Positions: []pdf.Position{{PageNumbers: []int{1}, Left: 10, Right: 400, Top: 10, Bottom: 120}},
|
||
RegionLeft: 10, RegionRight: 400, RegionTop: 10, RegionBottom: 120,
|
||
Scale: 1.0,
|
||
Cells: []pdf.TSRCell{
|
||
{X0: 0, Y0: 50, X1: 100, Y1: 80, Text: "Page1_C"},
|
||
{X0: 100, Y0: 50, X1: 200, Y1: 80, Text: "Page1_D"},
|
||
},
|
||
},
|
||
}
|
||
|
||
result := ExtractTableAndReplace(boxes, tables)
|
||
if len(result) == 0 {
|
||
t.Fatal("expected non-empty result")
|
||
}
|
||
// After merge: 2 table boxes replaced by 1 merged HTML box.
|
||
// Original 5 boxes → 4 expected (intro, merged_table, middle, outro).
|
||
if len(result) != 4 {
|
||
t.Errorf("expected 4 boxes after merge+replace, got %d", len(result))
|
||
}
|
||
// The merged HTML box should contain cells from both pages.
|
||
htmlBox := result[1]
|
||
if !strings.Contains(htmlBox.Text, "Page0") || !strings.Contains(htmlBox.Text, "Page1") {
|
||
t.Errorf("merged HTML should contain cells from both pages, got: %s", htmlBox.Text[:min(100, len(htmlBox.Text))])
|
||
}
|
||
// Verify the original text boxes are preserved in the right order.
|
||
if result[0].Text != "intro" || result[2].Text != "middle" || result[3].Text != "outro" {
|
||
t.Error("non-table boxes should be preserved in original order")
|
||
}
|
||
}
|
||
|
||
// TestProcessTablesWithReplacements_KeepsTextWhenNoCells pins the contract that
|
||
// a text box mislabeled LayoutTypeTable by DLA is kept as prose when TSR yields
|
||
// no cells (HTML missing), instead of being silently dropped.
|
||
func TestProcessTablesWithReplacements_KeepsTextWhenNoCells(t *testing.T) {
|
||
const body = "REFERENCE [1] body text that must not vanish"
|
||
boxes := []pdf.TextBox{
|
||
{Text: "para one", PageNumber: 1, LayoutType: pdf.LayoutTypeText, X0: 0, X1: 100, Top: 0, Bottom: 10},
|
||
{Text: body, PageNumber: 1, LayoutType: pdf.LayoutTypeTable, X0: 0, X1: 100, Top: 12, Bottom: 22},
|
||
{Text: "para three", PageNumber: 1, LayoutType: pdf.LayoutTypeText, X0: 0, X1: 100, Top: 24, Bottom: 34},
|
||
}
|
||
// DLA table region but no TSR cells → buildTableHTMLs skips, htmls[0] unset.
|
||
tables := []pdf.TableItem{{}}
|
||
removeSet := map[int]bool{}
|
||
replacements := []replacement{{tableIdx: 0, boxIdx: 1}}
|
||
|
||
out := processTablesWithReplacements(boxes, tables, removeSet, replacements)
|
||
|
||
expectedTexts := []string{"para one", body, "para three"}
|
||
if len(out) != len(expectedTexts) {
|
||
t.Fatalf("over-labeled 'table' box was dropped: want %d boxes, got %d: %+v", len(expectedTexts), len(out), out)
|
||
}
|
||
for i, want := range expectedTexts {
|
||
if out[i].Text != want {
|
||
t.Errorf("output box %d: want text %q, got %q (%+v)", i, want, out[i].Text, out)
|
||
}
|
||
}
|
||
}
|
||
|
||
// TestProcessTablesWithReplacements_NoDupWhenCoveredByEmptyAndRealTable pins the
|
||
// boundary where a single box is covered by TWO replacements — one whose table
|
||
// has no cells (empty HTML) and one whose table has real cells (non-empty HTML)
|
||
// — so the box must NOT be kept while the real table's HTML is also inserted
|
||
// (that would duplicate the content). The box must be removed and only the real
|
||
// table's HTML inserted.
|
||
func TestProcessTablesWithReplacements_NoDupWhenCoveredByEmptyAndRealTable(t *testing.T) {
|
||
const body = "SHARED BODY spanning two table regions"
|
||
boxes := []pdf.TextBox{
|
||
{Text: "para one", PageNumber: 0, LayoutType: pdf.LayoutTypeText, X0: 0, X1: 100, Top: 0, Bottom: 10},
|
||
{Text: body, PageNumber: 0, LayoutType: pdf.LayoutTypeTable, X0: 0, X1: 400, Top: 12, Bottom: 100},
|
||
{Text: "para three", PageNumber: 0, LayoutType: pdf.LayoutTypeText, X0: 0, X1: 100, Top: 110, Bottom: 120},
|
||
}
|
||
// Both tables occupy the same region as box 1.
|
||
region := pdf.Position{PageNumbers: []int{0}, Left: 0, Right: 400, Top: 12, Bottom: 100}
|
||
tables := []pdf.TableItem{
|
||
{
|
||
// DLA region but no TSR cells → buildTableHTMLs skips, htmls[0] unset.
|
||
Positions: []pdf.Position{region},
|
||
RegionLeft: 0, RegionRight: 400, RegionTop: 12, RegionBottom: 100,
|
||
},
|
||
{
|
||
// Real table with cells → htmls[1] non-empty.
|
||
Positions: []pdf.Position{region},
|
||
RegionLeft: 0, RegionRight: 400, RegionTop: 12, RegionBottom: 100,
|
||
Scale: 1.0,
|
||
Cells: []pdf.TSRCell{{Text: "cell1"}},
|
||
},
|
||
}
|
||
removeSet := map[int]bool{}
|
||
// box 1 is the replacement target for BOTH tables.
|
||
replacements := []replacement{{tableIdx: 0, boxIdx: 1}, {tableIdx: 1, boxIdx: 1}}
|
||
|
||
out := processTablesWithReplacements(boxes, tables, removeSet, replacements)
|
||
|
||
// The box must be removed (replaced by the real table's HTML), not kept
|
||
// alongside it — otherwise the content is duplicated.
|
||
for _, b := range out {
|
||
if b.Text == body {
|
||
t.Fatalf("box covered by a real table was kept, duplicating content: %+v", out)
|
||
}
|
||
}
|
||
// Exactly one table HTML box (table 1) should be present.
|
||
var htmlCount int
|
||
for _, b := range out {
|
||
if b.LayoutType == pdf.LayoutTypeTable && strings.Contains(b.Text, "cell1") {
|
||
htmlCount++
|
||
}
|
||
}
|
||
if htmlCount != 1 {
|
||
t.Errorf("expected exactly one table HTML box, got %d: %+v", htmlCount, out)
|
||
}
|
||
if len(out) != 3 {
|
||
t.Errorf("expected 3 boxes (2 text + 1 HTML), got %d: %+v", len(out), out)
|
||
}
|
||
}
|
||
|
||
// TestProcessTablesWithReplacements_KeepsTextWhenCellsButEmptyHTML guards the
|
||
// defensive case where a table has cells but ConstructTable returns an empty
|
||
// string (e.g. degenerate/orphaned cells). The box must be kept as prose rather
|
||
// than silently dropped. This is reachable when the target box is NOT collected
|
||
// as a table box (so the Y/X fallback in ConstructTable cannot recover text),
|
||
// e.g. a DLA-overlabeled text box whose layout type was not upgraded to Table.
|
||
func TestProcessTablesWithReplacements_KeepsTextWhenCellsButEmptyHTML(t *testing.T) {
|
||
const body = "PROSE that must survive a degenerate table"
|
||
boxes := []pdf.TextBox{
|
||
{Text: "para one", PageNumber: 0, LayoutType: pdf.LayoutTypeText, X0: 0, X1: 100, Top: 0, Bottom: 10},
|
||
{Text: body, PageNumber: 0, LayoutType: pdf.LayoutTypeText, X0: 0, X1: 400, Top: 12, Bottom: 100},
|
||
{Text: "para three", PageNumber: 0, LayoutType: pdf.LayoutTypeText, X0: 0, X1: 100, Top: 110, Bottom: 120},
|
||
}
|
||
tables := []pdf.TableItem{
|
||
{
|
||
// Cells present but without text, and the target box is LayoutTypeText
|
||
// (not collected as a table box) → ConstructTable yields "" → htmls[0]="".
|
||
Positions: []pdf.Position{{PageNumbers: []int{0}, Left: 0, Right: 400, Top: 12, Bottom: 100}},
|
||
RegionLeft: 0, RegionRight: 400, RegionTop: 12, RegionBottom: 100,
|
||
Scale: 1.0,
|
||
Cells: []pdf.TSRCell{{X0: 0, Y0: 0, X1: 100, Y1: 30}},
|
||
},
|
||
}
|
||
removeSet := map[int]bool{}
|
||
replacements := []replacement{{tableIdx: 0, boxIdx: 1}}
|
||
|
||
out := processTablesWithReplacements(boxes, tables, removeSet, replacements)
|
||
|
||
if len(out) != 3 {
|
||
t.Fatalf("degenerate-table box was dropped: want 3 boxes, got %d: %+v", len(out), out)
|
||
}
|
||
for _, b := range out {
|
||
if b.Text == body {
|
||
return // original text preserved
|
||
}
|
||
}
|
||
t.Errorf("degenerate-table box text missing from output: %+v", out)
|
||
}
|