fix(parser): align Go spreadsheet HTML with Python and harden header detection (#18304)

Brings the Go `XLSX`/`XLS`/`CSV` parsers in line with the Python deepdoc spreadsheet HTML contract and improves header detection beyond it.
This commit is contained in:
Jack
2026-08-17 15:17:14 +08:00
committed by GitHub
parent 8d20cbd0b3
commit 0517aebb80
7 changed files with 1080 additions and 145 deletions

View File

@@ -34,18 +34,10 @@ import (
"context"
"encoding/csv"
"fmt"
"html"
"regexp"
"strings"
)
// Go port of Python's ILLEGAL_CHARACTERS_RE from
// deepdoc/parser/excel_parser.py.
// Pattern: [\000-\010]|[\013-\014]|[\016-\037]
// Matches all control chars except TAB (\x09), LF (\x0A), CR (\x0D).
var csvIllegalCharsRe = regexp.MustCompile(`[\x00-\x08]|\x0B|\x0C|[\x0E-\x1F]`)
const csvDefaultChunkRows = 256
const csvDefaultChunkRows = defaultTableChunkRows
const csvSheetName = "Data"
// CSVParser reads RFC-4180 CSV data and emits HTML <table> payloads.
@@ -157,7 +149,7 @@ func (p *CSVParser) ParseWithResult(ctx context.Context, filename string, data [
}
// Clean illegal control characters from all cells.
records = cleanCSVRecords(records)
records = cleanIllegalControlChars(records)
chunkRows := p.ChunkRows
if chunkRows <= 0 {
@@ -171,84 +163,6 @@ func (p *CSVParser) ParseWithResult(ctx context.Context, filename string, data [
"size": len(data),
"encoding": "utf-8",
},
HTML: recordsToHTMLTableChunks(records, chunkRows),
HTML: recordsToHTMLTableChunks(records, chunkRows, csvSheetName),
}
}
// cleanCSVRecords replaces illegal control characters in all cells
// with a single space, matching Python's ILLEGAL_CHARACTERS_RE.
func cleanCSVRecords(records [][]string) [][]string {
out := make([][]string, len(records))
for i, row := range records {
out[i] = make([]string, len(row))
for j, cell := range row {
out[i][j] = csvIllegalCharsRe.ReplaceAllString(cell, " ")
}
}
return out
}
// recordsToHTMLTableChunks renders CSV records as one or more
// self-contained HTML <table> chunks. The first row is always the
// header (<th>). Data rows are split into chunks of chunkRows,
// each chunk being a complete <table> with <caption> and a repeated
// header row. Chunks are joined with newlines.
//
// Mirrors Python's RAGFlowExcelParser.html() chunking:
//
// chunks = (n_data_rows + chunk_rows - 1) // chunk_rows
func recordsToHTMLTableChunks(records [][]string, chunkRows int) string {
if len(records) == 0 {
return "<table><caption>" + csvSheetName + "</caption></table>"
}
// Build the header row once — repeated in every chunk.
headerHTML := buildCSVHeaderRow(records[0])
dataRows := records[1:]
nData := len(dataRows)
if nData == 0 {
// Only a header row exists.
return "<table><caption>" + csvSheetName + "</caption>\n" + headerHTML + "\n</table>"
}
nChunks := (nData + chunkRows - 1) / chunkRows
var b strings.Builder
for ci := 0; ci < nChunks; ci++ {
start := ci * chunkRows
end := start + chunkRows
if end > nData {
end = nData
}
b.WriteString("<table><caption>")
b.WriteString(csvSheetName)
b.WriteString("</caption>\n")
b.WriteString(headerHTML)
for _, row := range dataRows[start:end] {
b.WriteString("<tr>")
for _, cell := range row {
b.WriteString("<td>")
b.WriteString(html.EscapeString(strings.TrimSpace(cell)))
b.WriteString("</td>")
}
b.WriteString("</tr>\n")
}
b.WriteString("</table>\n")
}
return b.String()
}
// buildCSVHeaderRow renders the first row as an HTML <th> header row.
func buildCSVHeaderRow(row []string) string {
var b strings.Builder
b.WriteString("<tr>")
for _, cell := range row {
b.WriteString("<th>")
b.WriteString(html.EscapeString(strings.TrimSpace(cell)))
b.WriteString("</th>")
}
b.WriteString("</tr>\n")
return b.String()
}

View File

@@ -1,9 +0,0 @@
//go:build cgo
package parser
import "html"
func htmlEscape(s string) string {
return html.EscapeString(s)
}

View File

@@ -2,11 +2,5 @@
package parser
import "html"
// OfficeOxide is the lib_type identifier for office_oxide backend.
const OfficeOxide = "office_oxide"
func htmlEscape(s string) string {
return html.EscapeString(s)
}

View File

@@ -0,0 +1,548 @@
//
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language of permissions and
// limitations under the License.
//
package parser
import (
"html"
"regexp"
"strconv"
"strings"
"github.com/xuri/excelize/v2"
)
// tableIllegalCharsRe replaces illegal control characters (everything except
// TAB/LF/CR) with a single space. The pattern matches all C0 control chars
// except TAB (0x09), LF (0x0A) and CR (0x0D).
var tableIllegalCharsRe = regexp.MustCompile(`[\x00-\x08]|\x0B|\x0C|[\x0E-\x1F]`)
// numericCellRe recognises number-like cell text (integers, decimals, comma
// thousands separators, optional leading sign/$ and trailing %). It is a
// deliberately loose heuristic used only for header-vs-data contrast, not for
// value parsing.
var numericCellRe = regexp.MustCompile(`^[\$\+\-]?[\d,]+(\.\d+)?%?$`)
const defaultTableChunkRows = 256
// maxMergeExtentCols caps how far a merged range may widen the header row.
// excelize's GetRows truncates each row at its last valued cell, so a merged
// slave beyond that point is absent and cannot inherit its master's text; we
// pad the header row to the furthest merged column to recover it. A pathological
// far merge (e.g. A1:XFD1) must not be allowed to allocate one cell per column
// for every row, so the width is capped here.
const maxMergeExtentCols = 1024
// cleanIllegalControlChars replaces illegal control characters in all cells
// with a single space.
func cleanIllegalControlChars(records [][]string) [][]string {
out := make([][]string, len(records))
for i, row := range records {
out[i] = make([]string, len(row))
for j, cell := range row {
out[i][j] = tableIllegalCharsRe.ReplaceAllString(cell, " ")
}
}
return out
}
// buildHeaderRow renders a row as an HTML <th> header row.
func buildHeaderRow(row []string) string {
var b strings.Builder
b.WriteString("<tr>")
for _, cell := range row {
b.WriteString("<th>")
b.WriteString(html.EscapeString(strings.TrimSpace(cell)))
b.WriteString("</th>")
}
b.WriteString("</tr>\n")
return b.String()
}
// recordsToHTMLTableChunks renders records as one or more self-contained HTML
// <table> chunks. The first row is always the header (<th>). Data rows are
// split into chunks of chunkRows, each chunk being a complete <table> with
// <caption> and a repeated header row. Chunks are joined with newlines.
//
// The tag schema is <table><caption>{caption}</caption><tr><th>…</th></tr>
// <tr><td>…</td></tr>…</table>. Rows are intentionally NOT wrapped in
// <thead>/<tbody>, so every <table> is one atomic chunk that downstream
// chunkers can consume independently.
func recordsToHTMLTableChunks(records [][]string, chunkRows int, caption string) string {
if len(records) == 0 {
return "<table><caption>" + html.EscapeString(caption) + "</caption></table>"
}
// Build the header row once — repeated in every chunk.
headerHTML := buildHeaderRow(records[0])
dataRows := records[1:]
nData := len(dataRows)
if nData == 0 {
// Only a header row exists.
return "<table><caption>" + html.EscapeString(caption) + "</caption>\n" + headerHTML + "</table>"
}
if chunkRows <= 0 {
chunkRows = defaultTableChunkRows
}
nChunks := (nData + chunkRows - 1) / chunkRows
var b strings.Builder
for ci := 0; ci < nChunks; ci++ {
start := ci * chunkRows
end := start + chunkRows
if end > nData {
end = nData
}
b.WriteString("<table><caption>")
b.WriteString(html.EscapeString(caption))
b.WriteString("</caption>\n")
b.WriteString(headerHTML)
for _, row := range dataRows[start:end] {
b.WriteString("<tr>")
for _, cell := range row {
b.WriteString("<td>")
b.WriteString(html.EscapeString(strings.TrimSpace(cell)))
b.WriteString("</td>")
}
b.WriteString("</tr>\n")
}
b.WriteString("</table>\n")
}
return b.String()
}
// ──────────────────────────────────────────────────────────── axis helpers
// axisToRC parses an "A1"-style cell reference (optionally with a leading "$")
// into 1-based (row, col). A malformed reference yields (0, 0).
func axisToRC(axis string) (row, col int) {
axis = strings.TrimSpace(axis)
axis = strings.TrimPrefix(axis, "$")
i := 0
for i < len(axis) && (axis[i] < '0' || axis[i] > '9') {
i++
}
if i == 0 || i == len(axis) {
return 0, 0
}
letter, num := axis[:i], axis[i:]
r, err := strconv.Atoi(num)
if err != nil {
return 0, 0
}
c := 0
for _, ch := range strings.ToUpper(letter) {
if ch < 'A' || ch > 'Z' {
return 0, 0
}
c = c*26 + int(ch-'A'+1)
}
return r, c
}
// rangeTopRow returns the top (first) row of an A1:B10-style range reference.
func rangeTopRow(ref string) int {
ref = strings.SplitN(ref, ":", 2)[0]
r, _ := axisToRC(ref)
return r
}
// cellAxis builds an "A1"-style reference for 1-based (row, col).
func cellAxis(row, col int) string {
// Build the column letters least-significant digit first, then reverse.
var digits []byte
c := col
for c > 0 {
c-- // 1-based → 0-based for this digit
digits = append(digits, byte('A'+c%26))
c /= 26
}
for i, j := 0, len(digits)-1; i < j; i, j = i+1, j-1 {
digits[i], digits[j] = digits[j], digits[i]
}
return string(digits) + strconv.Itoa(row)
}
// ──────────────────────────────────────────────────────────── merge inheritance
// mergeRange is an excelize merged-cell rectangle, 1-based inclusive.
type mergeRange struct {
sr, sc, er, ec int
}
// mergeRanges returns the merged-cell rectangles of a sheet.
func mergeRanges(f *excelize.File, sheet string) []mergeRange {
var out []mergeRange
cells, err := f.GetMergeCells(sheet)
if err != nil {
return out
}
for _, mc := range cells {
sr, sc := axisToRC(mc.GetStartAxis())
er, ec := axisToRC(mc.GetEndAxis())
if sr == 0 || sc == 0 || er == 0 || ec == 0 {
continue
}
out = append(out, mergeRange{sr, sc, er, ec})
}
return out
}
// mergeMaxCol returns the furthest merged column across all ranges, or 0 if
// there are none.
func mergeMaxCol(ranges []mergeRange) int {
m := 0
for _, r := range ranges {
if r.ec > m {
m = r.ec
}
}
return m
}
// mergeMasterForRow materialises the slave→master map for a single row only.
// A large merged block (e.g. A1:Z100) would otherwise expand to every one of
// its cells; we only ever need the header row's slaves, so expanding per row
// keeps the map O(cols) instead of O(rows×cols).
func mergeMasterForRow(ranges []mergeRange, row int) map[[2]int][2]int {
mm := map[[2]int][2]int{}
for _, r := range ranges {
if row < r.sr || row > r.er {
continue
}
for c := r.sc; c <= r.ec; c++ {
mm[[2]int{row, c}] = [2]int{r.sr, r.sc}
}
}
return mm
}
// inheritMergedHeader fills empty cells in the header row with the value of
// their merge master (typically a wide horizontally-merged title cell). This
// keeps a wide merged title from rendering as a row of blank <th> cells.
func inheritMergedHeader(records [][]string, headerRowIdx int, mm map[[2]int][2]int) {
if headerRowIdx < 1 || headerRowIdx > len(records) {
return
}
row := records[headerRowIdx-1]
for c := 0; c < len(row); c++ {
if strings.TrimSpace(row[c]) != "" {
continue
}
master, ok := mm[[2]int{headerRowIdx, c + 1}]
if !ok || (master[0] == headerRowIdx && master[1] == c+1) {
continue
}
if master[0] < 1 || master[0] > len(records) || master[1] < 1 || master[1] > len(records[master[0]-1]) {
continue
}
val := records[master[0]-1][master[1]-1]
if strings.TrimSpace(val) != "" {
row[c] = val
}
}
}
// mergeExtentCol returns the furthest merged column across all ranges, capped
// at maxMergeExtentCols so a pathological far merge cannot exhaust parser
// memory when the header row is padded to inherit merged text.
func mergeExtentCol(ranges []mergeRange) int {
m := mergeMaxCol(ranges)
if m > maxMergeExtentCols {
return maxMergeExtentCols
}
return m
}
// padRowToWidth grows a single row to at least maxCol, padding with empty
// strings. Only the header row is padded (see renderSheetTables): merged-master
// text is inherited into the header alone, so data rows must not be widened —
// widening them would emit a sea of empty <td> cells for every far merge in the
// sheet and is the memory blow-up flagged in review.
func padRowToWidth(row *[]string, maxCol int) {
if maxCol <= len(*row) {
return
}
padded := make([]string, maxCol)
copy(padded, *row)
*row = padded
}
// ──────────────────────────────────────────────────────────── header detection
// isNumericCell reports whether a cell value reads as a number-like string.
func isNumericCell(s string) bool {
s = strings.TrimSpace(s)
return s != "" && numericCellRe.MatchString(s)
}
// cellIsStyled reports whether a cell is bold or carries a fill, using
// excelize's style lookup.
func cellIsStyled(f *excelize.File, sheet string, row, col int) bool {
idx, err := f.GetCellStyle(sheet, cellAxis(row, col))
if err != nil {
return false
}
st, err := f.GetStyle(idx)
if err != nil || st == nil {
return false
}
if st.Font != nil && st.Font.Bold {
return true
}
if st.Fill.Type != "" || len(st.Fill.Color) > 0 {
return true
}
return false
}
// detectHeaderRow returns the 1-based row that should be treated as the column
// header of a sheet, defaulting to 1. It only diverges from row 1 when there is
// high confidence that row 1 is not the header:
//
// 1. ListObject first: if the sheet defines Excel tables (ListObjects) whose
// top row is > 1, that row is the header. This is the cheapest, most
// accurate signal and never fires for the common case (table starts at row 1).
// 2. Lightweight detection (no ListObject override): scan the top few rows for
// an anchor — a row whose cells are mostly bold/filled, or a row holding
// text labels over numeric data columns below (contrast ≥ 2 columns). A
// candidate that looks like a data row (majority numeric) is skipped. The
// override only applies when the anchor is not already row 1, so the common
// header-on-row-1 sheet is left unchanged.
func detectHeaderRow(f *excelize.File, sheet string, records [][]string) int {
n := len(records)
if n == 0 {
return 1
}
// 1) ListObject first.
if tables, err := f.GetTables(sheet); err == nil && len(tables) > 0 {
minTop := 0
for _, t := range tables {
tr := rangeTopRow(t.Range)
if tr < 1 {
continue
}
if minTop == 0 || tr < minTop {
minTop = tr
}
}
if minTop > 1 {
return minTop
}
}
// 2) Lightweight detection over the top window (bounded to 4 candidate rows,
// and we must leave at least one data row below the candidate).
maxScan := 4
if maxScan > n-1 {
maxScan = n - 1
}
if maxScan < 1 {
return 1
}
for k := 0; k < maxScan; k++ {
row := records[k]
nc := len(row)
if nc < 2 {
continue // title / section stub — keep looking below it
}
// Data-majority reject: a row that is mostly numbers is data, not a
// header. Keep scanning downward for the real header.
nonEmpty, numeric := 0, 0
for _, v := range row {
if v == "" {
continue
}
nonEmpty++
if isNumericCell(v) {
numeric++
}
}
if nonEmpty > 0 && numeric*2 >= nonEmpty {
continue
}
// Styled signal.
styled := 0
for c := 0; c < nc && c < 64; c++ {
if cellIsStyled(f, sheet, k+1, c+1) {
styled++
}
}
// Contrast signal: text label over a numeric column below.
contrast := 0
if k+1 < n {
below := records[k+1]
for c := 0; c < nc && c < 64; c++ {
v := row[c]
if v == "" || isNumericCell(v) {
continue
}
if c < len(below) && isNumericCell(below[c]) {
contrast++
if contrast >= 2 {
break
}
}
}
}
if styled*2 >= nc || contrast >= 2 {
if k == 0 {
return 1 // row 1 is already the header
}
below := records[k+1]
// Accept the candidate as the header when the row directly below
// is a genuine data row (it contains at least one text cell), or
// when that row is purely numeric but the candidate is not a
// subtotal label. The second clause lets a styled text header
// sitting above numeric-only data win; a bold "Total"/"Summary"
// subtotal looks identical locally, but its label matches a
// subtotal keyword so it is still refused and row 1 is kept.
if rowHasTextCell(below) || (isPurelyNumeric(below) && !isSubtotalRow(row)) {
return k + 1
}
}
}
return 1
}
// rowHasTextCell reports whether a row contains at least one non-empty,
// non-numeric (i.e. text) cell. It is used as a body-signature brake for
// header detection.
func rowHasTextCell(row []string) bool {
for _, v := range row {
if v != "" && !isNumericCell(v) {
return true
}
}
return false
}
// isPurelyNumeric reports whether every non-empty cell of a row reads as a
// number-like string. It is used to recognise a numeric continuation row below
// a candidate header.
func isPurelyNumeric(row []string) bool {
nonEmpty := 0
for _, v := range row {
if strings.TrimSpace(v) == "" {
continue
}
nonEmpty++
if !isNumericCell(v) {
return false
}
}
return nonEmpty > 0
}
// subtotalWordRe matches labels that mark a totals/subtotals row. A candidate
// header sitting above a purely-numeric row is refused when any of its cells
// matches, so bold "Total"/"Summary" subtotals are not promoted to the header.
var subtotalWordRe = regexp.MustCompile(`^(total|totals|sum|summary|subtotal|subtotals|grand total|合计|总计|小计|汇总|总额)$`)
// isSubtotalRow reports whether any non-empty cell of a row is a subtotal label
// (case-insensitive, tolerating a trailing colon).
func isSubtotalRow(row []string) bool {
for _, v := range row {
v = strings.ToLower(strings.TrimSpace(v))
v = strings.TrimRight(v, ":")
if v != "" && subtotalWordRe.MatchString(v) {
return true
}
}
return false
}
// decodeChunkRows reads the "chunk_rows" setup knob, returning the default when
// it is absent or non-positive.
func decodeChunkRows(setup map[string]any) int {
if setup == nil {
return defaultTableChunkRows
}
v, ok := setup["chunk_rows"]
if !ok {
return defaultTableChunkRows
}
switch n := v.(type) {
case float64:
rows := int(n)
if rows <= 0 {
return defaultTableChunkRows
}
return rows
case int:
if n <= 0 {
return defaultTableChunkRows
}
return n
case int64:
rows := int(n)
if rows <= 0 {
return defaultTableChunkRows
}
return rows
}
return defaultTableChunkRows
}
// renderSheetTables renders a single workbook sheet into one or more
// self-contained <table> chunks using the shared spreadsheet-HTML contract:
// detect the header row, inherit merged-master text into the header, and split
// data into chunkRows-sized atomic tables each repeating the header. An empty
// or unreadable sheet yields an empty string.
func renderSheetTables(f *excelize.File, sheet string, chunkRows int) string {
rows, err := f.GetRows(sheet)
if err != nil || len(rows) == 0 {
return ""
}
rows = cleanIllegalControlChars(rows)
ranges := mergeRanges(f, sheet)
headerRow := detectHeaderRow(f, sheet, rows)
// Inherit merged-master text into the header row. excelize's GetRows
// truncates each row at its last valued cell, so a merged slave beyond that
// point is absent and cannot inherit its master's text. We therefore pad
// ONLY the header row (the only row we inherit into) to the furthest merged
// column — capped by mergeExtentCol so a pathological far merge cannot
// exhaust memory. Padding runs after detection so a wide merge (e.g. A1:Z1
// title) does not dilute the styled-majority signal of a narrow header.
mm := mergeMasterForRow(ranges, headerRow)
if len(mm) > 0 {
padRowToWidth(&rows[headerRow-1], mergeExtentCol(ranges))
}
inheritMergedHeader(rows, headerRow, mm)
// Reorder so the detected header row becomes records[0]; every other row is
// data. For the common case (header on row 1) this is a no-op.
records := make([][]string, 0, len(rows))
records = append(records, rows[headerRow-1])
for i, r := range rows {
if i == headerRow-1 {
continue
}
records = append(records, r)
}
return recordsToHTMLTableChunks(records, chunkRows, sheet)
}

View File

@@ -0,0 +1,506 @@
package parser
import (
"strings"
"testing"
"github.com/xuri/excelize/v2"
)
// newTestXLSX builds an in-memory .xlsx from a cell writer.
func newTestXLSX(t *testing.T, fill func(f *excelize.File)) []byte {
t.Helper()
f := excelize.NewFile()
defer f.Close()
fill(f)
buf, err := f.WriteToBuffer()
if err != nil {
t.Fatalf("WriteToBuffer: %v", err)
}
return buf.Bytes()
}
// The following helpers fail the test immediately when an Excelize fixture
// operation errors, so incomplete workbook data is never handed to the parser.
func mustSetCell(t *testing.T, f *excelize.File, sheet, axis string, val any) {
t.Helper()
if err := f.SetCellValue(sheet, axis, val); err != nil {
t.Fatalf("SetCellValue(%s!%s): %v", sheet, axis, err)
}
}
func mustMergeCell(t *testing.T, f *excelize.File, sheet, topLeft, bottomRight string) {
t.Helper()
if err := f.MergeCell(sheet, topLeft, bottomRight); err != nil {
t.Fatalf("MergeCell(%s:%s-%s): %v", sheet, topLeft, bottomRight, err)
}
}
func mustNewStyle(t *testing.T, f *excelize.File, style *excelize.Style) int {
t.Helper()
idx, err := f.NewStyle(style)
if err != nil {
t.Fatalf("NewStyle: %v", err)
}
return idx
}
func mustSetCellStyle(t *testing.T, f *excelize.File, sheet, topLeft, bottomRight string, idx int) {
t.Helper()
if err := f.SetCellStyle(sheet, topLeft, bottomRight, idx); err != nil {
t.Fatalf("SetCellStyle(%s:%s-%s): %v", sheet, topLeft, bottomRight, err)
}
}
func mustAddTable(t *testing.T, f *excelize.File, sheet string, table *excelize.Table) {
t.Helper()
if err := f.AddTable(sheet, table); err != nil {
t.Fatalf("AddTable(%s): %v", sheet, err)
}
}
// TestRecordsToHTMLTableChunks_Alignment asserts the chunked output uses the
// shared schema: <caption>, first row as <th>, data as <td>, repeated header
// per 256-row chunk, and NO <thead>/<tbody> wrapper.
func TestRecordsToHTMLTableChunks_Alignment(t *testing.T) {
records := [][]string{{"Name", "Age"}, {"Alice", "30"}, {"Bob", "25"}}
out := recordsToHTMLTableChunks(records, 256, "Sheet1")
if !strings.Contains(out, `<caption>Sheet1</caption>`) {
t.Fatalf("want <caption>Sheet1</caption>, got:\n%s", out)
}
if !strings.Contains(out, "<tr><th>Name</th><th>Age</th></tr>") {
t.Fatalf("want header row as <th>, got:\n%s", out)
}
if !strings.Contains(out, "<tr><td>Alice</td><td>30</td></tr>") {
t.Fatalf("want data row as <td>, got:\n%s", out)
}
if strings.Contains(out, "<thead>") || strings.Contains(out, "<tbody>") {
t.Fatalf("must not emit <thead>/<tbody> to stay byte-compatible with Python/CSV, got:\n%s", out)
}
// Exactly one <table> for <=256 data rows.
if n := strings.Count(out, "<table>"); n != 1 {
t.Fatalf("want 1 <table>, got %d:\n%s", n, out)
}
}
// TestRecordsToHTMLTableChunks_Chunking asserts 256-row chunking with a repeated
// header (ceil(n_data / chunk_rows) chunks).
func TestRecordsToHTMLTableChunks_Chunking(t *testing.T) {
const dataRows = 300
records := make([][]string, 0, dataRows+1)
records = append(records, []string{"C1", "C2"})
for i := 0; i < dataRows; i++ {
records = append(records, []string{"x", "y"})
}
out := recordsToHTMLTableChunks(records, 256, "S")
// 300 data rows → ceil(300/256) = 2 chunks, each repeating the header.
if n := strings.Count(out, "<table>"); n != 2 {
t.Fatalf("want 2 <table> chunks, got %d", n)
}
if n := strings.Count(out, "<tr><th>C1</th><th>C2</th></tr>"); n != 2 {
t.Fatalf("want header repeated in both chunks, got %d", n)
}
}
// TestXLSXParser_HeaderAndCaption asserts the XLSX parser emits a <caption> and
// renders the first row as <th>, and that the header text appears only in <th>.
func TestXLSXParser_HeaderAndCaption(t *testing.T) {
data := newTestXLSX(t, func(f *excelize.File) {
mustSetCell(t, f, "Sheet1", "A1", "Product")
mustSetCell(t, f, "Sheet1", "B1", "Price")
mustSetCell(t, f, "Sheet1", "A2", "Widget")
mustSetCell(t, f, "Sheet1", "B2", "9.99")
mustSetCell(t, f, "Sheet1", "A3", "Gadget")
mustSetCell(t, f, "Sheet1", "B3", "19.99")
})
p, _ := NewXLSXParser("")
res := p.ParseWithResult(t.Context(), "t.xlsx", data)
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
html := res.HTML
if !strings.Contains(html, `<caption>Sheet1</caption>`) {
t.Fatalf("want <caption>Sheet1</caption>, got:\n%s", html)
}
if !strings.Contains(html, "<tr><th>Product</th><th>Price</th></tr>") {
t.Fatalf("want header as <th>, got:\n%s", html)
}
// "Product" must only appear inside a <th>, never inside a <td>.
if strings.Contains(html, "<td>Product</td>") {
t.Fatalf("header value leaked into <td>:\n%s", html)
}
}
// TestXLSXParser_MergedHeaderInheritance asserts a horizontally merged header
// cell's slave columns inherit the master text, so a wide merged title does not
// render as a row of blank <th> cells.
func TestXLSXParser_MergedHeaderInheritance(t *testing.T) {
data := newTestXLSX(t, func(f *excelize.File) {
// Row 1 is the header; A1:C1 merged into one wide label "Sales Report".
mustSetCell(t, f, "Sheet1", "A1", "Sales Report")
mustMergeCell(t, f, "Sheet1", "A1", "C1")
// Make the header row bold so detection anchors row 1.
idx := mustNewStyle(t, f, &excelize.Style{Font: &excelize.Font{Bold: true}})
mustSetCellStyle(t, f, "Sheet1", "A1", "C1", idx)
mustSetCell(t, f, "Sheet1", "A2", "North")
mustSetCell(t, f, "Sheet1", "B2", "10")
mustSetCell(t, f, "Sheet1", "C2", "20")
mustSetCell(t, f, "Sheet1", "A3", "South")
mustSetCell(t, f, "Sheet1", "B3", "30")
mustSetCell(t, f, "Sheet1", "C3", "40")
})
p, _ := NewXLSXParser("")
res := p.ParseWithResult(t.Context(), "t.xlsx", data)
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
html := res.HTML
// The merged master text must have propagated into all three <th> slots.
if !strings.Contains(html, "<tr><th>Sales Report</th><th>Sales Report</th><th>Sales Report</th></tr>") {
t.Fatalf("merged master text not inherited into header <th>, got:\n%s", html)
}
}
// TestDetectHeaderRow_ListObject asserts a ListObject whose top row is > 1 is
// detected as the header row.
func TestDetectHeaderRow_ListObject(t *testing.T) {
data := newTestXLSX(t, func(f *excelize.File) {
mustSetCell(t, f, "Sheet1", "A1", "Title")
mustSetCell(t, f, "Sheet1", "A2", "This is a banner")
mustSetCell(t, f, "Sheet1", "A3", "Name")
mustSetCell(t, f, "Sheet1", "B3", "Age")
mustSetCell(t, f, "Sheet1", "A4", "Alice")
mustSetCell(t, f, "Sheet1", "B4", "30")
mustAddTable(t, f, "Sheet1", &excelize.Table{Range: "A3:B4", Name: "Table1"})
})
p, _ := NewXLSXParser("")
res := p.ParseWithResult(t.Context(), "t.xlsx", data)
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
if !strings.Contains(res.HTML, "<tr><th>Name</th><th>Age</th></tr>") {
t.Fatalf("want ListObject header row detected, got:\n%s", res.HTML)
}
if strings.Contains(res.HTML, "<th>Title</th>") {
t.Fatalf("title row must not be the header:\n%s", res.HTML)
}
}
// TestDetectHeaderRow_Lightweight asserts that when row 1 is numeric data and
// row 2 is a styled text label row, the lightweight detector picks row 2.
func TestDetectHeaderRow_Lightweight(t *testing.T) {
data := newTestXLSX(t, func(f *excelize.File) {
mustSetCell(t, f, "Sheet1", "A1", "100")
mustSetCell(t, f, "Sheet1", "B1", "200")
mustSetCell(t, f, "Sheet1", "A2", "Item")
mustSetCell(t, f, "Sheet1", "B2", "Count")
// Make the header row bold so the styled signal fires.
idx := mustNewStyle(t, f, &excelize.Style{Font: &excelize.Font{Bold: true}})
mustSetCellStyle(t, f, "Sheet1", "A2", "B2", idx)
mustSetCell(t, f, "Sheet1", "A3", "Apple")
mustSetCell(t, f, "Sheet1", "B3", "5")
})
p, _ := NewXLSXParser("")
res := p.ParseWithResult(t.Context(), "t.xlsx", data)
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
if !strings.Contains(res.HTML, "<tr><th>Item</th><th>Count</th></tr>") {
t.Fatalf("want row-2 header detected, got:\n%s", res.HTML)
}
if strings.Contains(res.HTML, "<th>100</th>") {
t.Fatalf("numeric row 1 must not be the header:\n%s", res.HTML)
}
}
// TestXLSXParser_CommonCaseNoRegression asserts the dominant case (header on
// row 1, no merges, no ListObject) renders row 1 as <th> unchanged.
func TestXLSXParser_CommonCaseNoRegression(t *testing.T) {
data := newTestXLSX(t, func(f *excelize.File) {
mustSetCell(t, f, "Sheet1", "A1", "col_a")
mustSetCell(t, f, "Sheet1", "B1", "col_b")
mustSetCell(t, f, "Sheet1", "A2", "1")
mustSetCell(t, f, "Sheet1", "B2", "2")
})
p, _ := NewXLSXParser("")
res := p.ParseWithResult(t.Context(), "t.xlsx", data)
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
if !strings.Contains(res.HTML, "<tr><th>col_a</th><th>col_b</th></tr>") {
t.Fatalf("common-case header must render as <th>:\n%s", res.HTML)
}
}
// TestDetectHeaderRow_BoldSubtotalNotHeader asserts that a bold "Total"-style
// subtotal row sitting directly under a numeric row 1 is NOT promoted to the
// header. The subtotal-label check blocks the override so the numeric row 1
// stays the header.
func TestDetectHeaderRow_BoldSubtotalNotHeader(t *testing.T) {
data := newTestXLSX(t, func(f *excelize.File) {
// Row 1: numeric year header (Python default header row).
mustSetCell(t, f, "Sheet1", "A1", "2023")
mustSetCell(t, f, "Sheet1", "B1", "2024")
// Row 2: bold "Total/Summary" subtotal — must NOT become the header.
idx := mustNewStyle(t, f, &excelize.Style{Font: &excelize.Font{Bold: true}})
mustSetCell(t, f, "Sheet1", "A2", "Total")
mustSetCell(t, f, "Sheet1", "B2", "Summary")
mustSetCellStyle(t, f, "Sheet1", "A2", "B2", idx)
// Row 3: purely numeric continuation under the subtotal.
mustSetCell(t, f, "Sheet1", "A3", "100")
mustSetCell(t, f, "Sheet1", "B3", "200")
})
p, _ := NewXLSXParser("")
res := p.ParseWithResult(t.Context(), "t.xlsx", data)
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
if !strings.Contains(res.HTML, "<tr><th>2023</th><th>2024</th></tr>") {
t.Fatalf("numeric row 1 must remain the header:\n%s", res.HTML)
}
if strings.Contains(res.HTML, "<th>Total</th>") {
t.Fatalf("bold subtotal row must not be promoted to header:\n%s", res.HTML)
}
}
// TestDetectHeaderRow_StyledHeaderPastFarMerge asserts that a bold header on a
// narrow row (row 3) is still detected even when an earlier wide merge
// (A1:Z1 title) would otherwise pad every row to 26 columns. The padding step
// must run AFTER header detection so the styled-majority signal is not diluted.
func TestDetectHeaderRow_StyledHeaderPastFarMerge(t *testing.T) {
data := newTestXLSX(t, func(f *excelize.File) {
// Row 1: a wide merged title cell A1:Z1.
mustSetCell(t, f, "Sheet1", "A1", "Sales Report")
mustMergeCell(t, f, "Sheet1", "A1", "Z1")
titleIdx := mustNewStyle(t, f, &excelize.Style{Font: &excelize.Font{Bold: true}})
mustSetCellStyle(t, f, "Sheet1", "A1", "A1", titleIdx)
// Row 3: the real, narrow, bold header.
hdrIdx := mustNewStyle(t, f, &excelize.Style{Font: &excelize.Font{Bold: true}})
mustSetCell(t, f, "Sheet1", "A3", "Name")
mustSetCell(t, f, "Sheet1", "B3", "Desc")
mustSetCellStyle(t, f, "Sheet1", "A3", "B3", hdrIdx)
// Row 4: data (text in col A, so the body-brake passes for row 3).
mustSetCell(t, f, "Sheet1", "A4", "Alice")
mustSetCell(t, f, "Sheet1", "B4", "x")
})
p, _ := NewXLSXParser("")
res := p.ParseWithResult(t.Context(), "t.xlsx", data)
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
if !strings.Contains(res.HTML, "<th>Name</th><th>Desc</th>") {
t.Fatalf("narrow bold header past far merge must be detected:\n%s", res.HTML)
}
if strings.Contains(res.HTML, "<th>Sales Report</th>") {
t.Fatalf("wide merged title must not be the header:\n%s", res.HTML)
}
}
// TestDetectHeaderRow_StyledTextHeaderOverNumeric asserts that a styled text
// header sitting directly above a purely-numeric data row is still detected as
// the header (not refused by the body-brake), while a bold "Total"/"Summary"
// subtotal over numeric data is not.
func TestDetectHeaderRow_StyledTextHeaderOverNumeric(t *testing.T) {
data := newTestXLSX(t, func(f *excelize.File) {
// Row 1: a single-cell title stub (fewer than 2 columns → skipped).
mustSetCell(t, f, "Sheet1", "A1", "Report Title")
// Row 2: the real, bold text header over numeric-only data below.
idx := mustNewStyle(t, f, &excelize.Style{Font: &excelize.Font{Bold: true}})
mustSetCell(t, f, "Sheet1", "A2", "Product")
mustSetCell(t, f, "Sheet1", "B2", "Units")
mustSetCellStyle(t, f, "Sheet1", "A2", "B2", idx)
// Row 3: purely numeric continuation.
mustSetCell(t, f, "Sheet1", "A3", "5")
mustSetCell(t, f, "Sheet1", "B3", "10")
})
p, _ := NewXLSXParser("")
res := p.ParseWithResult(t.Context(), "t.xlsx", data)
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
if !strings.Contains(res.HTML, "<tr><th>Product</th><th>Units</th></tr>") {
t.Fatalf("styled text header over numeric data must be detected:\n%s", res.HTML)
}
if strings.Contains(res.HTML, "<th>Report Title</th>") {
t.Fatalf("title stub must not be the header:\n%s", res.HTML)
}
}
// TestInheritMergedHeader unit-tests the merge inheritance helper directly.
func TestInheritMergedHeader(t *testing.T) {
records := [][]string{
{"MASTER", "", "Q2"}, // header row: B1 blank, merged from master A1
{"a", "b", "c"},
}
mm := map[[2]int][2]int{
{1, 1}: {1, 1},
{1, 2}: {1, 1}, // B1's master is A1
{1, 3}: {1, 3},
}
inheritMergedHeader(records, 1, mm)
if records[0][1] != "MASTER" {
t.Fatalf("expected B1 to inherit A1 master text, got %q", records[0][1])
}
if records[0][2] != "Q2" {
t.Fatalf("expected non-merged C1 to keep its value, got %q", records[0][2])
}
}
// TestPadRowToWidth asserts the single-row padder widens with empty strings,
// preserves existing cells, and never shrinks an already-wide row.
func TestPadRowToWidth(t *testing.T) {
row := []string{"a", "b"}
padRowToWidth(&row, 5)
if len(row) != 5 {
t.Fatalf("want len 5, got %d", len(row))
}
if row[0] != "a" || row[1] != "b" {
t.Fatalf("original cells lost: %v", row)
}
for i := 2; i < 5; i++ {
if row[i] != "" {
t.Fatalf("pad cell %d want empty, got %q", i, row[i])
}
}
// Never shrinks an already-wide row.
padRowToWidth(&row, 2)
if len(row) != 5 {
t.Fatalf("must not shrink: want 5, got %d", len(row))
}
}
// TestMergeExtentCol asserts the furthest merged column is reported as-is within
// the cap and clamped to maxMergeExtentCols beyond it (the memory guard).
func TestMergeExtentCol(t *testing.T) {
if got := mergeExtentCol(nil); got != 0 {
t.Fatalf("nil ranges: want 0, got %d", got)
}
if got := mergeExtentCol([]mergeRange{{1, 1, 3, 10}}); got != 10 {
t.Fatalf("within cap: want 10, got %d", got)
}
if got := mergeExtentCol([]mergeRange{{1, 1, 1, 5000}}); got != maxMergeExtentCols {
t.Fatalf("beyond cap: want %d, got %d", maxMergeExtentCols, got)
}
}
// TestDecodeChunkRows exercises the chunk_rows setup knob across all the types
// JSON/yaml decoding can produce, plus the defaulting paths.
func TestDecodeChunkRows(t *testing.T) {
if got := decodeChunkRows(nil); got != defaultTableChunkRows {
t.Fatalf("nil setup: want default %d, got %d", defaultTableChunkRows, got)
}
if got := decodeChunkRows(map[string]any{}); got != defaultTableChunkRows {
t.Fatalf("empty setup: want default, got %d", got)
}
if got := decodeChunkRows(map[string]any{"chunk_rows": 100.0}); got != 100 {
t.Fatalf("float64 100: want 100, got %d", got)
}
if got := decodeChunkRows(map[string]any{"chunk_rows": 0.0}); got != defaultTableChunkRows {
t.Fatalf("float64 0: want default, got %d", got)
}
if got := decodeChunkRows(map[string]any{"chunk_rows": -5.0}); got != defaultTableChunkRows {
t.Fatalf("float64 -5: want default, got %d", got)
}
if got := decodeChunkRows(map[string]any{"chunk_rows": 256}); got != 256 {
t.Fatalf("int 256: want 256, got %d", got)
}
if got := decodeChunkRows(map[string]any{"chunk_rows": int64(512)}); got != 512 {
t.Fatalf("int64 512: want 512, got %d", got)
}
if got := decodeChunkRows(map[string]any{"chunk_rows": "256"}); got != defaultTableChunkRows {
t.Fatalf("string (unsupported type): want default, got %d", got)
}
}
// TestRenderSheetTables_FarMergeDoesNotBloatDataRows guards the memory blow-up
// from review: a far merge on a non-header row must not widen the header or any
// data row to the merge's full span. The header stays its natural width and no
// data row carries empty <td> cells.
func TestRenderSheetTables_FarMergeDoesNotBloatDataRows(t *testing.T) {
data := newTestXLSX(t, func(f *excelize.File) {
// Row 1: a wide merged title A1:Z1 (NOT the header).
mustSetCell(t, f, "Sheet1", "A1", "Report Title")
mustMergeCell(t, f, "Sheet1", "A1", "Z1")
// Row 2: the real header, made bold so detection anchors it.
hdrIdx := mustNewStyle(t, f, &excelize.Style{Font: &excelize.Font{Bold: true}})
mustSetCell(t, f, "Sheet1", "A2", "Name")
mustSetCell(t, f, "Sheet1", "B2", "Price")
mustSetCellStyle(t, f, "Sheet1", "A2", "B2", hdrIdx)
// Rows 3-4: data.
mustSetCell(t, f, "Sheet1", "A3", "Alice")
mustSetCell(t, f, "Sheet1", "B3", "9.99")
mustSetCell(t, f, "Sheet1", "A4", "Bob")
mustSetCell(t, f, "Sheet1", "B4", "19.99")
})
p, _ := NewXLSXParser("")
res := p.ParseWithResult(t.Context(), "t.xlsx", data)
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
html := res.HTML
if !strings.Contains(html, "<tr><th>Name</th><th>Price</th></tr>") {
t.Fatalf("want row-2 header detected, got:\n%s", html)
}
if strings.Contains(html, "<th>Title</th>") {
t.Fatalf("wide merged title must not be the header:\n%s", html)
}
// The far merge is on row 1, not the header, so nothing is widened to 26
// columns: the header keeps 2 columns (no empty <th>) and data rows carry
// no empty <td>.
if strings.Contains(html, "<th></th>") {
t.Fatalf("header was bloated by the far merge:\n%s", html)
}
if strings.Count(html, "<td></td>") != 0 {
t.Fatalf("data rows were bloated by the far merge:\n%s", html)
}
}
// TestRenderSheetTables_MultiSheet asserts each sheet becomes its own <table>
// chunk, each carrying its own <caption>.
func TestRenderSheetTables_MultiSheet(t *testing.T) {
data := newTestXLSX(t, func(f *excelize.File) {
mustSetCell(t, f, "Sheet1", "A1", "Name")
mustSetCell(t, f, "Sheet1", "B1", "Age")
mustSetCell(t, f, "Sheet1", "A2", "Alice")
mustSetCell(t, f, "Sheet1", "B2", "30")
if _, err := f.NewSheet("HR"); err != nil {
t.Fatalf("NewSheet: %v", err)
}
mustSetCell(t, f, "HR", "A1", "Dept")
mustSetCell(t, f, "HR", "B1", "Head")
mustSetCell(t, f, "HR", "A2", "Eng")
mustSetCell(t, f, "HR", "B2", "Bob")
})
p, _ := NewXLSXParser("")
res := p.ParseWithResult(t.Context(), "t.xlsx", data)
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
html := res.HTML
if !strings.Contains(html, "<caption>Sheet1</caption>") {
t.Fatalf("want Sheet1 caption, got:\n%s", html)
}
if !strings.Contains(html, "<caption>HR</caption>") {
t.Fatalf("want HR caption, got:\n%s", html)
}
// Two small sheets → two <table> chunks.
if n := strings.Count(html, "<table>"); n != 2 {
t.Fatalf("want 2 <table> chunks, got %d:\n%s", n, html)
}
}
// TestRenderSheetTables_EmptySheet asserts an empty/unreadable sheet yields an
// empty string rather than an empty <table> wrapper.
func TestRenderSheetTables_EmptySheet(t *testing.T) {
// excelize.NewFile yields a single empty "Sheet1".
data := newTestXLSX(t, func(f *excelize.File) {})
p, _ := NewXLSXParser("")
res := p.ParseWithResult(t.Context(), "t.xlsx", data)
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
if res.HTML != "" {
t.Fatalf("empty sheet must yield empty HTML, got:\n%s", res.HTML)
}
}

View File

@@ -29,6 +29,7 @@ type XLSParser struct {
libType string
ParseMethod string
OutputFormat string
ChunkRows int
TCADPAPIServer string
TCADPAPIKey string
TCADPTableResultType string
@@ -41,6 +42,7 @@ func NewXLSParser(libType string) (*XLSParser, error) {
}
return &XLSParser{
libType: libType,
ChunkRows: defaultTableChunkRows,
TCADPTableResultType: "1",
TCADPMarkdownImageResponseType: "1",
}, nil
@@ -60,6 +62,7 @@ func (p *XLSParser) ConfigureFromSetup(setup map[string]any) {
if v, ok := setup["output_format"].(string); ok && v != "" {
p.OutputFormat = v
}
p.ChunkRows = decodeChunkRows(setup)
if v, ok := setup["tcadp_apiserver"].(string); ok && v != "" {
p.TCADPAPIServer = v
}
@@ -98,26 +101,16 @@ func (p *XLSParser) ParseWithResult(ctx context.Context, filename string, data [
}
defer f.Close()
var html strings.Builder
html.WriteString("<html><body>")
for _, sheet := range f.GetSheetList() {
html.WriteString("<h3>")
html.WriteString(sheet)
html.WriteString("</h3>")
rows, _ := f.GetRows(sheet)
html.WriteString("<table>")
for _, row := range rows {
html.WriteString("<tr>")
for _, cell := range row {
html.WriteString("<td>")
html.WriteString(htmlEscape(cell))
html.WriteString("</td>")
}
html.WriteString("</tr>")
}
html.WriteString("</table>")
sheets := f.GetSheetList()
chunkRows := p.ChunkRows
if chunkRows <= 0 {
chunkRows = defaultTableChunkRows
}
var html strings.Builder
for _, sheet := range sheets {
html.WriteString(renderSheetTables(f, sheet, chunkRows))
}
html.WriteString("</body></html>")
return ParseResult{
OutputFormat: "html",

View File

@@ -29,6 +29,7 @@ type XLSXParser struct {
libType string
ParseMethod string
OutputFormat string
ChunkRows int
TCADPAPIServer string
TCADPAPIKey string
TCADPTableResultType string
@@ -41,6 +42,7 @@ func NewXLSXParser(libType string) (*XLSXParser, error) {
}
return &XLSXParser{
libType: libType,
ChunkRows: defaultTableChunkRows,
TCADPTableResultType: "1",
TCADPMarkdownImageResponseType: "1",
}, nil
@@ -60,6 +62,7 @@ func (p *XLSXParser) ConfigureFromSetup(setup map[string]any) {
if v, ok := setup["output_format"].(string); ok && v != "" {
p.OutputFormat = v
}
p.ChunkRows = decodeChunkRows(setup)
if v, ok := setup["tcadp_apiserver"].(string); ok && v != "" {
p.TCADPAPIServer = v
}
@@ -117,29 +120,15 @@ func (p *XLSXParser) ParseWithResult(ctx context.Context, filename string, data
defer f.Close()
sheets := f.GetSheetList()
var html strings.Builder
html.WriteString("<html><body>")
for _, sheet := range sheets {
html.WriteString("<h3>")
html.WriteString(sheet)
html.WriteString("</h3>")
rows, err := f.GetRows(sheet)
if err != nil {
continue
}
html.WriteString("<table>")
for _, row := range rows {
html.WriteString("<tr>")
for _, cell := range row {
html.WriteString("<td>")
html.WriteString(htmlEscape(cell))
html.WriteString("</td>")
}
html.WriteString("</tr>")
}
html.WriteString("</table>")
chunkRows := p.ChunkRows
if chunkRows <= 0 {
chunkRows = defaultTableChunkRows
}
var html strings.Builder
for _, sheet := range sheets {
html.WriteString(renderSheetTables(f, sheet, chunkRows))
}
html.WriteString("</body></html>")
return ParseResult{
OutputFormat: "html",