mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-08-18 22:38:25 +08:00
fix(parser): align Go spreadsheet HTML with Python and harden header detection (#18304)
Brings the Go `XLSX`/`XLS`/`CSV` parsers in line with the Python deepdoc spreadsheet HTML contract and improves header detection beyond it.
This commit is contained in:
@@ -34,18 +34,10 @@ import (
|
||||
"context"
|
||||
"encoding/csv"
|
||||
"fmt"
|
||||
"html"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Go port of Python's ILLEGAL_CHARACTERS_RE from
|
||||
// deepdoc/parser/excel_parser.py.
|
||||
// Pattern: [\000-\010]|[\013-\014]|[\016-\037]
|
||||
// Matches all control chars except TAB (\x09), LF (\x0A), CR (\x0D).
|
||||
var csvIllegalCharsRe = regexp.MustCompile(`[\x00-\x08]|\x0B|\x0C|[\x0E-\x1F]`)
|
||||
|
||||
const csvDefaultChunkRows = 256
|
||||
const csvDefaultChunkRows = defaultTableChunkRows
|
||||
const csvSheetName = "Data"
|
||||
|
||||
// CSVParser reads RFC-4180 CSV data and emits HTML <table> payloads.
|
||||
@@ -157,7 +149,7 @@ func (p *CSVParser) ParseWithResult(ctx context.Context, filename string, data [
|
||||
}
|
||||
|
||||
// Clean illegal control characters from all cells.
|
||||
records = cleanCSVRecords(records)
|
||||
records = cleanIllegalControlChars(records)
|
||||
|
||||
chunkRows := p.ChunkRows
|
||||
if chunkRows <= 0 {
|
||||
@@ -171,84 +163,6 @@ func (p *CSVParser) ParseWithResult(ctx context.Context, filename string, data [
|
||||
"size": len(data),
|
||||
"encoding": "utf-8",
|
||||
},
|
||||
HTML: recordsToHTMLTableChunks(records, chunkRows),
|
||||
HTML: recordsToHTMLTableChunks(records, chunkRows, csvSheetName),
|
||||
}
|
||||
}
|
||||
|
||||
// cleanCSVRecords replaces illegal control characters in all cells
|
||||
// with a single space, matching Python's ILLEGAL_CHARACTERS_RE.
|
||||
func cleanCSVRecords(records [][]string) [][]string {
|
||||
out := make([][]string, len(records))
|
||||
for i, row := range records {
|
||||
out[i] = make([]string, len(row))
|
||||
for j, cell := range row {
|
||||
out[i][j] = csvIllegalCharsRe.ReplaceAllString(cell, " ")
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// recordsToHTMLTableChunks renders CSV records as one or more
|
||||
// self-contained HTML <table> chunks. The first row is always the
|
||||
// header (<th>). Data rows are split into chunks of chunkRows,
|
||||
// each chunk being a complete <table> with <caption> and a repeated
|
||||
// header row. Chunks are joined with newlines.
|
||||
//
|
||||
// Mirrors Python's RAGFlowExcelParser.html() chunking:
|
||||
//
|
||||
// chunks = (n_data_rows + chunk_rows - 1) // chunk_rows
|
||||
func recordsToHTMLTableChunks(records [][]string, chunkRows int) string {
|
||||
if len(records) == 0 {
|
||||
return "<table><caption>" + csvSheetName + "</caption></table>"
|
||||
}
|
||||
|
||||
// Build the header row once — repeated in every chunk.
|
||||
headerHTML := buildCSVHeaderRow(records[0])
|
||||
dataRows := records[1:]
|
||||
nData := len(dataRows)
|
||||
|
||||
if nData == 0 {
|
||||
// Only a header row exists.
|
||||
return "<table><caption>" + csvSheetName + "</caption>\n" + headerHTML + "\n</table>"
|
||||
}
|
||||
|
||||
nChunks := (nData + chunkRows - 1) / chunkRows
|
||||
var b strings.Builder
|
||||
for ci := 0; ci < nChunks; ci++ {
|
||||
start := ci * chunkRows
|
||||
end := start + chunkRows
|
||||
if end > nData {
|
||||
end = nData
|
||||
}
|
||||
|
||||
b.WriteString("<table><caption>")
|
||||
b.WriteString(csvSheetName)
|
||||
b.WriteString("</caption>\n")
|
||||
b.WriteString(headerHTML)
|
||||
|
||||
for _, row := range dataRows[start:end] {
|
||||
b.WriteString("<tr>")
|
||||
for _, cell := range row {
|
||||
b.WriteString("<td>")
|
||||
b.WriteString(html.EscapeString(strings.TrimSpace(cell)))
|
||||
b.WriteString("</td>")
|
||||
}
|
||||
b.WriteString("</tr>\n")
|
||||
}
|
||||
b.WriteString("</table>\n")
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// buildCSVHeaderRow renders the first row as an HTML <th> header row.
|
||||
func buildCSVHeaderRow(row []string) string {
|
||||
var b strings.Builder
|
||||
b.WriteString("<tr>")
|
||||
for _, cell := range row {
|
||||
b.WriteString("<th>")
|
||||
b.WriteString(html.EscapeString(strings.TrimSpace(cell)))
|
||||
b.WriteString("</th>")
|
||||
}
|
||||
b.WriteString("</tr>\n")
|
||||
return b.String()
|
||||
}
|
||||
|
||||
@@ -1,9 +0,0 @@
|
||||
//go:build cgo
|
||||
|
||||
package parser
|
||||
|
||||
import "html"
|
||||
|
||||
func htmlEscape(s string) string {
|
||||
return html.EscapeString(s)
|
||||
}
|
||||
@@ -2,11 +2,5 @@
|
||||
|
||||
package parser
|
||||
|
||||
import "html"
|
||||
|
||||
// OfficeOxide is the lib_type identifier for office_oxide backend.
|
||||
const OfficeOxide = "office_oxide"
|
||||
|
||||
func htmlEscape(s string) string {
|
||||
return html.EscapeString(s)
|
||||
}
|
||||
|
||||
548
internal/parser/parser/office_table_render.go
Normal file
548
internal/parser/parser/office_table_render.go
Normal file
@@ -0,0 +1,548 @@
|
||||
//
|
||||
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language of permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
package parser
|
||||
|
||||
import (
|
||||
"html"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/xuri/excelize/v2"
|
||||
)
|
||||
|
||||
// tableIllegalCharsRe replaces illegal control characters (everything except
|
||||
// TAB/LF/CR) with a single space. The pattern matches all C0 control chars
|
||||
// except TAB (0x09), LF (0x0A) and CR (0x0D).
|
||||
var tableIllegalCharsRe = regexp.MustCompile(`[\x00-\x08]|\x0B|\x0C|[\x0E-\x1F]`)
|
||||
|
||||
// numericCellRe recognises number-like cell text (integers, decimals, comma
|
||||
// thousands separators, optional leading sign/$ and trailing %). It is a
|
||||
// deliberately loose heuristic used only for header-vs-data contrast, not for
|
||||
// value parsing.
|
||||
var numericCellRe = regexp.MustCompile(`^[\$\+\-]?[\d,]+(\.\d+)?%?$`)
|
||||
|
||||
const defaultTableChunkRows = 256
|
||||
|
||||
// maxMergeExtentCols caps how far a merged range may widen the header row.
|
||||
// excelize's GetRows truncates each row at its last valued cell, so a merged
|
||||
// slave beyond that point is absent and cannot inherit its master's text; we
|
||||
// pad the header row to the furthest merged column to recover it. A pathological
|
||||
// far merge (e.g. A1:XFD1) must not be allowed to allocate one cell per column
|
||||
// for every row, so the width is capped here.
|
||||
const maxMergeExtentCols = 1024
|
||||
|
||||
// cleanIllegalControlChars replaces illegal control characters in all cells
|
||||
// with a single space.
|
||||
func cleanIllegalControlChars(records [][]string) [][]string {
|
||||
out := make([][]string, len(records))
|
||||
for i, row := range records {
|
||||
out[i] = make([]string, len(row))
|
||||
for j, cell := range row {
|
||||
out[i][j] = tableIllegalCharsRe.ReplaceAllString(cell, " ")
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// buildHeaderRow renders a row as an HTML <th> header row.
|
||||
func buildHeaderRow(row []string) string {
|
||||
var b strings.Builder
|
||||
b.WriteString("<tr>")
|
||||
for _, cell := range row {
|
||||
b.WriteString("<th>")
|
||||
b.WriteString(html.EscapeString(strings.TrimSpace(cell)))
|
||||
b.WriteString("</th>")
|
||||
}
|
||||
b.WriteString("</tr>\n")
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// recordsToHTMLTableChunks renders records as one or more self-contained HTML
|
||||
// <table> chunks. The first row is always the header (<th>). Data rows are
|
||||
// split into chunks of chunkRows, each chunk being a complete <table> with
|
||||
// <caption> and a repeated header row. Chunks are joined with newlines.
|
||||
//
|
||||
// The tag schema is <table><caption>{caption}</caption><tr><th>…</th></tr>
|
||||
// <tr><td>…</td></tr>…</table>. Rows are intentionally NOT wrapped in
|
||||
// <thead>/<tbody>, so every <table> is one atomic chunk that downstream
|
||||
// chunkers can consume independently.
|
||||
func recordsToHTMLTableChunks(records [][]string, chunkRows int, caption string) string {
|
||||
if len(records) == 0 {
|
||||
return "<table><caption>" + html.EscapeString(caption) + "</caption></table>"
|
||||
}
|
||||
|
||||
// Build the header row once — repeated in every chunk.
|
||||
headerHTML := buildHeaderRow(records[0])
|
||||
dataRows := records[1:]
|
||||
nData := len(dataRows)
|
||||
|
||||
if nData == 0 {
|
||||
// Only a header row exists.
|
||||
return "<table><caption>" + html.EscapeString(caption) + "</caption>\n" + headerHTML + "</table>"
|
||||
}
|
||||
|
||||
if chunkRows <= 0 {
|
||||
chunkRows = defaultTableChunkRows
|
||||
}
|
||||
|
||||
nChunks := (nData + chunkRows - 1) / chunkRows
|
||||
var b strings.Builder
|
||||
for ci := 0; ci < nChunks; ci++ {
|
||||
start := ci * chunkRows
|
||||
end := start + chunkRows
|
||||
if end > nData {
|
||||
end = nData
|
||||
}
|
||||
|
||||
b.WriteString("<table><caption>")
|
||||
b.WriteString(html.EscapeString(caption))
|
||||
b.WriteString("</caption>\n")
|
||||
b.WriteString(headerHTML)
|
||||
|
||||
for _, row := range dataRows[start:end] {
|
||||
b.WriteString("<tr>")
|
||||
for _, cell := range row {
|
||||
b.WriteString("<td>")
|
||||
b.WriteString(html.EscapeString(strings.TrimSpace(cell)))
|
||||
b.WriteString("</td>")
|
||||
}
|
||||
b.WriteString("</tr>\n")
|
||||
}
|
||||
b.WriteString("</table>\n")
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// ──────────────────────────────────────────────────────────── axis helpers
|
||||
|
||||
// axisToRC parses an "A1"-style cell reference (optionally with a leading "$")
|
||||
// into 1-based (row, col). A malformed reference yields (0, 0).
|
||||
func axisToRC(axis string) (row, col int) {
|
||||
axis = strings.TrimSpace(axis)
|
||||
axis = strings.TrimPrefix(axis, "$")
|
||||
i := 0
|
||||
for i < len(axis) && (axis[i] < '0' || axis[i] > '9') {
|
||||
i++
|
||||
}
|
||||
if i == 0 || i == len(axis) {
|
||||
return 0, 0
|
||||
}
|
||||
letter, num := axis[:i], axis[i:]
|
||||
r, err := strconv.Atoi(num)
|
||||
if err != nil {
|
||||
return 0, 0
|
||||
}
|
||||
c := 0
|
||||
for _, ch := range strings.ToUpper(letter) {
|
||||
if ch < 'A' || ch > 'Z' {
|
||||
return 0, 0
|
||||
}
|
||||
c = c*26 + int(ch-'A'+1)
|
||||
}
|
||||
return r, c
|
||||
}
|
||||
|
||||
// rangeTopRow returns the top (first) row of an A1:B10-style range reference.
|
||||
func rangeTopRow(ref string) int {
|
||||
ref = strings.SplitN(ref, ":", 2)[0]
|
||||
r, _ := axisToRC(ref)
|
||||
return r
|
||||
}
|
||||
|
||||
// cellAxis builds an "A1"-style reference for 1-based (row, col).
|
||||
func cellAxis(row, col int) string {
|
||||
// Build the column letters least-significant digit first, then reverse.
|
||||
var digits []byte
|
||||
c := col
|
||||
for c > 0 {
|
||||
c-- // 1-based → 0-based for this digit
|
||||
digits = append(digits, byte('A'+c%26))
|
||||
c /= 26
|
||||
}
|
||||
for i, j := 0, len(digits)-1; i < j; i, j = i+1, j-1 {
|
||||
digits[i], digits[j] = digits[j], digits[i]
|
||||
}
|
||||
return string(digits) + strconv.Itoa(row)
|
||||
}
|
||||
|
||||
// ──────────────────────────────────────────────────────────── merge inheritance
|
||||
|
||||
// mergeRange is an excelize merged-cell rectangle, 1-based inclusive.
|
||||
type mergeRange struct {
|
||||
sr, sc, er, ec int
|
||||
}
|
||||
|
||||
// mergeRanges returns the merged-cell rectangles of a sheet.
|
||||
func mergeRanges(f *excelize.File, sheet string) []mergeRange {
|
||||
var out []mergeRange
|
||||
cells, err := f.GetMergeCells(sheet)
|
||||
if err != nil {
|
||||
return out
|
||||
}
|
||||
for _, mc := range cells {
|
||||
sr, sc := axisToRC(mc.GetStartAxis())
|
||||
er, ec := axisToRC(mc.GetEndAxis())
|
||||
if sr == 0 || sc == 0 || er == 0 || ec == 0 {
|
||||
continue
|
||||
}
|
||||
out = append(out, mergeRange{sr, sc, er, ec})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// mergeMaxCol returns the furthest merged column across all ranges, or 0 if
|
||||
// there are none.
|
||||
func mergeMaxCol(ranges []mergeRange) int {
|
||||
m := 0
|
||||
for _, r := range ranges {
|
||||
if r.ec > m {
|
||||
m = r.ec
|
||||
}
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// mergeMasterForRow materialises the slave→master map for a single row only.
|
||||
// A large merged block (e.g. A1:Z100) would otherwise expand to every one of
|
||||
// its cells; we only ever need the header row's slaves, so expanding per row
|
||||
// keeps the map O(cols) instead of O(rows×cols).
|
||||
func mergeMasterForRow(ranges []mergeRange, row int) map[[2]int][2]int {
|
||||
mm := map[[2]int][2]int{}
|
||||
for _, r := range ranges {
|
||||
if row < r.sr || row > r.er {
|
||||
continue
|
||||
}
|
||||
for c := r.sc; c <= r.ec; c++ {
|
||||
mm[[2]int{row, c}] = [2]int{r.sr, r.sc}
|
||||
}
|
||||
}
|
||||
return mm
|
||||
}
|
||||
|
||||
// inheritMergedHeader fills empty cells in the header row with the value of
|
||||
// their merge master (typically a wide horizontally-merged title cell). This
|
||||
// keeps a wide merged title from rendering as a row of blank <th> cells.
|
||||
func inheritMergedHeader(records [][]string, headerRowIdx int, mm map[[2]int][2]int) {
|
||||
if headerRowIdx < 1 || headerRowIdx > len(records) {
|
||||
return
|
||||
}
|
||||
row := records[headerRowIdx-1]
|
||||
for c := 0; c < len(row); c++ {
|
||||
if strings.TrimSpace(row[c]) != "" {
|
||||
continue
|
||||
}
|
||||
master, ok := mm[[2]int{headerRowIdx, c + 1}]
|
||||
if !ok || (master[0] == headerRowIdx && master[1] == c+1) {
|
||||
continue
|
||||
}
|
||||
if master[0] < 1 || master[0] > len(records) || master[1] < 1 || master[1] > len(records[master[0]-1]) {
|
||||
continue
|
||||
}
|
||||
val := records[master[0]-1][master[1]-1]
|
||||
if strings.TrimSpace(val) != "" {
|
||||
row[c] = val
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// mergeExtentCol returns the furthest merged column across all ranges, capped
|
||||
// at maxMergeExtentCols so a pathological far merge cannot exhaust parser
|
||||
// memory when the header row is padded to inherit merged text.
|
||||
func mergeExtentCol(ranges []mergeRange) int {
|
||||
m := mergeMaxCol(ranges)
|
||||
if m > maxMergeExtentCols {
|
||||
return maxMergeExtentCols
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// padRowToWidth grows a single row to at least maxCol, padding with empty
|
||||
// strings. Only the header row is padded (see renderSheetTables): merged-master
|
||||
// text is inherited into the header alone, so data rows must not be widened —
|
||||
// widening them would emit a sea of empty <td> cells for every far merge in the
|
||||
// sheet and is the memory blow-up flagged in review.
|
||||
func padRowToWidth(row *[]string, maxCol int) {
|
||||
if maxCol <= len(*row) {
|
||||
return
|
||||
}
|
||||
padded := make([]string, maxCol)
|
||||
copy(padded, *row)
|
||||
*row = padded
|
||||
}
|
||||
|
||||
// ──────────────────────────────────────────────────────────── header detection
|
||||
|
||||
// isNumericCell reports whether a cell value reads as a number-like string.
|
||||
func isNumericCell(s string) bool {
|
||||
s = strings.TrimSpace(s)
|
||||
return s != "" && numericCellRe.MatchString(s)
|
||||
}
|
||||
|
||||
// cellIsStyled reports whether a cell is bold or carries a fill, using
|
||||
// excelize's style lookup.
|
||||
func cellIsStyled(f *excelize.File, sheet string, row, col int) bool {
|
||||
idx, err := f.GetCellStyle(sheet, cellAxis(row, col))
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
st, err := f.GetStyle(idx)
|
||||
if err != nil || st == nil {
|
||||
return false
|
||||
}
|
||||
if st.Font != nil && st.Font.Bold {
|
||||
return true
|
||||
}
|
||||
if st.Fill.Type != "" || len(st.Fill.Color) > 0 {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// detectHeaderRow returns the 1-based row that should be treated as the column
|
||||
// header of a sheet, defaulting to 1. It only diverges from row 1 when there is
|
||||
// high confidence that row 1 is not the header:
|
||||
//
|
||||
// 1. ListObject first: if the sheet defines Excel tables (ListObjects) whose
|
||||
// top row is > 1, that row is the header. This is the cheapest, most
|
||||
// accurate signal and never fires for the common case (table starts at row 1).
|
||||
// 2. Lightweight detection (no ListObject override): scan the top few rows for
|
||||
// an anchor — a row whose cells are mostly bold/filled, or a row holding
|
||||
// text labels over numeric data columns below (contrast ≥ 2 columns). A
|
||||
// candidate that looks like a data row (majority numeric) is skipped. The
|
||||
// override only applies when the anchor is not already row 1, so the common
|
||||
// header-on-row-1 sheet is left unchanged.
|
||||
func detectHeaderRow(f *excelize.File, sheet string, records [][]string) int {
|
||||
n := len(records)
|
||||
if n == 0 {
|
||||
return 1
|
||||
}
|
||||
|
||||
// 1) ListObject first.
|
||||
if tables, err := f.GetTables(sheet); err == nil && len(tables) > 0 {
|
||||
minTop := 0
|
||||
for _, t := range tables {
|
||||
tr := rangeTopRow(t.Range)
|
||||
if tr < 1 {
|
||||
continue
|
||||
}
|
||||
if minTop == 0 || tr < minTop {
|
||||
minTop = tr
|
||||
}
|
||||
}
|
||||
if minTop > 1 {
|
||||
return minTop
|
||||
}
|
||||
}
|
||||
|
||||
// 2) Lightweight detection over the top window (bounded to 4 candidate rows,
|
||||
// and we must leave at least one data row below the candidate).
|
||||
maxScan := 4
|
||||
if maxScan > n-1 {
|
||||
maxScan = n - 1
|
||||
}
|
||||
if maxScan < 1 {
|
||||
return 1
|
||||
}
|
||||
for k := 0; k < maxScan; k++ {
|
||||
row := records[k]
|
||||
nc := len(row)
|
||||
if nc < 2 {
|
||||
continue // title / section stub — keep looking below it
|
||||
}
|
||||
|
||||
// Data-majority reject: a row that is mostly numbers is data, not a
|
||||
// header. Keep scanning downward for the real header.
|
||||
nonEmpty, numeric := 0, 0
|
||||
for _, v := range row {
|
||||
if v == "" {
|
||||
continue
|
||||
}
|
||||
nonEmpty++
|
||||
if isNumericCell(v) {
|
||||
numeric++
|
||||
}
|
||||
}
|
||||
if nonEmpty > 0 && numeric*2 >= nonEmpty {
|
||||
continue
|
||||
}
|
||||
|
||||
// Styled signal.
|
||||
styled := 0
|
||||
for c := 0; c < nc && c < 64; c++ {
|
||||
if cellIsStyled(f, sheet, k+1, c+1) {
|
||||
styled++
|
||||
}
|
||||
}
|
||||
|
||||
// Contrast signal: text label over a numeric column below.
|
||||
contrast := 0
|
||||
if k+1 < n {
|
||||
below := records[k+1]
|
||||
for c := 0; c < nc && c < 64; c++ {
|
||||
v := row[c]
|
||||
if v == "" || isNumericCell(v) {
|
||||
continue
|
||||
}
|
||||
if c < len(below) && isNumericCell(below[c]) {
|
||||
contrast++
|
||||
if contrast >= 2 {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if styled*2 >= nc || contrast >= 2 {
|
||||
if k == 0 {
|
||||
return 1 // row 1 is already the header
|
||||
}
|
||||
below := records[k+1]
|
||||
// Accept the candidate as the header when the row directly below
|
||||
// is a genuine data row (it contains at least one text cell), or
|
||||
// when that row is purely numeric but the candidate is not a
|
||||
// subtotal label. The second clause lets a styled text header
|
||||
// sitting above numeric-only data win; a bold "Total"/"Summary"
|
||||
// subtotal looks identical locally, but its label matches a
|
||||
// subtotal keyword so it is still refused and row 1 is kept.
|
||||
if rowHasTextCell(below) || (isPurelyNumeric(below) && !isSubtotalRow(row)) {
|
||||
return k + 1
|
||||
}
|
||||
}
|
||||
}
|
||||
return 1
|
||||
}
|
||||
|
||||
// rowHasTextCell reports whether a row contains at least one non-empty,
|
||||
// non-numeric (i.e. text) cell. It is used as a body-signature brake for
|
||||
// header detection.
|
||||
func rowHasTextCell(row []string) bool {
|
||||
for _, v := range row {
|
||||
if v != "" && !isNumericCell(v) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// isPurelyNumeric reports whether every non-empty cell of a row reads as a
|
||||
// number-like string. It is used to recognise a numeric continuation row below
|
||||
// a candidate header.
|
||||
func isPurelyNumeric(row []string) bool {
|
||||
nonEmpty := 0
|
||||
for _, v := range row {
|
||||
if strings.TrimSpace(v) == "" {
|
||||
continue
|
||||
}
|
||||
nonEmpty++
|
||||
if !isNumericCell(v) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return nonEmpty > 0
|
||||
}
|
||||
|
||||
// subtotalWordRe matches labels that mark a totals/subtotals row. A candidate
|
||||
// header sitting above a purely-numeric row is refused when any of its cells
|
||||
// matches, so bold "Total"/"Summary" subtotals are not promoted to the header.
|
||||
var subtotalWordRe = regexp.MustCompile(`^(total|totals|sum|summary|subtotal|subtotals|grand total|合计|总计|小计|汇总|总额)$`)
|
||||
|
||||
// isSubtotalRow reports whether any non-empty cell of a row is a subtotal label
|
||||
// (case-insensitive, tolerating a trailing colon).
|
||||
func isSubtotalRow(row []string) bool {
|
||||
for _, v := range row {
|
||||
v = strings.ToLower(strings.TrimSpace(v))
|
||||
v = strings.TrimRight(v, "::")
|
||||
if v != "" && subtotalWordRe.MatchString(v) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// decodeChunkRows reads the "chunk_rows" setup knob, returning the default when
|
||||
// it is absent or non-positive.
|
||||
func decodeChunkRows(setup map[string]any) int {
|
||||
if setup == nil {
|
||||
return defaultTableChunkRows
|
||||
}
|
||||
v, ok := setup["chunk_rows"]
|
||||
if !ok {
|
||||
return defaultTableChunkRows
|
||||
}
|
||||
switch n := v.(type) {
|
||||
case float64:
|
||||
rows := int(n)
|
||||
if rows <= 0 {
|
||||
return defaultTableChunkRows
|
||||
}
|
||||
return rows
|
||||
case int:
|
||||
if n <= 0 {
|
||||
return defaultTableChunkRows
|
||||
}
|
||||
return n
|
||||
case int64:
|
||||
rows := int(n)
|
||||
if rows <= 0 {
|
||||
return defaultTableChunkRows
|
||||
}
|
||||
return rows
|
||||
}
|
||||
return defaultTableChunkRows
|
||||
}
|
||||
|
||||
// renderSheetTables renders a single workbook sheet into one or more
|
||||
// self-contained <table> chunks using the shared spreadsheet-HTML contract:
|
||||
// detect the header row, inherit merged-master text into the header, and split
|
||||
// data into chunkRows-sized atomic tables each repeating the header. An empty
|
||||
// or unreadable sheet yields an empty string.
|
||||
func renderSheetTables(f *excelize.File, sheet string, chunkRows int) string {
|
||||
rows, err := f.GetRows(sheet)
|
||||
if err != nil || len(rows) == 0 {
|
||||
return ""
|
||||
}
|
||||
rows = cleanIllegalControlChars(rows)
|
||||
|
||||
ranges := mergeRanges(f, sheet)
|
||||
headerRow := detectHeaderRow(f, sheet, rows)
|
||||
|
||||
// Inherit merged-master text into the header row. excelize's GetRows
|
||||
// truncates each row at its last valued cell, so a merged slave beyond that
|
||||
// point is absent and cannot inherit its master's text. We therefore pad
|
||||
// ONLY the header row (the only row we inherit into) to the furthest merged
|
||||
// column — capped by mergeExtentCol so a pathological far merge cannot
|
||||
// exhaust memory. Padding runs after detection so a wide merge (e.g. A1:Z1
|
||||
// title) does not dilute the styled-majority signal of a narrow header.
|
||||
mm := mergeMasterForRow(ranges, headerRow)
|
||||
if len(mm) > 0 {
|
||||
padRowToWidth(&rows[headerRow-1], mergeExtentCol(ranges))
|
||||
}
|
||||
inheritMergedHeader(rows, headerRow, mm)
|
||||
|
||||
// Reorder so the detected header row becomes records[0]; every other row is
|
||||
// data. For the common case (header on row 1) this is a no-op.
|
||||
records := make([][]string, 0, len(rows))
|
||||
records = append(records, rows[headerRow-1])
|
||||
for i, r := range rows {
|
||||
if i == headerRow-1 {
|
||||
continue
|
||||
}
|
||||
records = append(records, r)
|
||||
}
|
||||
return recordsToHTMLTableChunks(records, chunkRows, sheet)
|
||||
}
|
||||
506
internal/parser/parser/office_table_render_test.go
Normal file
506
internal/parser/parser/office_table_render_test.go
Normal file
@@ -0,0 +1,506 @@
|
||||
package parser
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/xuri/excelize/v2"
|
||||
)
|
||||
|
||||
// newTestXLSX builds an in-memory .xlsx from a cell writer.
|
||||
func newTestXLSX(t *testing.T, fill func(f *excelize.File)) []byte {
|
||||
t.Helper()
|
||||
f := excelize.NewFile()
|
||||
defer f.Close()
|
||||
fill(f)
|
||||
buf, err := f.WriteToBuffer()
|
||||
if err != nil {
|
||||
t.Fatalf("WriteToBuffer: %v", err)
|
||||
}
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
// The following helpers fail the test immediately when an Excelize fixture
|
||||
// operation errors, so incomplete workbook data is never handed to the parser.
|
||||
func mustSetCell(t *testing.T, f *excelize.File, sheet, axis string, val any) {
|
||||
t.Helper()
|
||||
if err := f.SetCellValue(sheet, axis, val); err != nil {
|
||||
t.Fatalf("SetCellValue(%s!%s): %v", sheet, axis, err)
|
||||
}
|
||||
}
|
||||
|
||||
func mustMergeCell(t *testing.T, f *excelize.File, sheet, topLeft, bottomRight string) {
|
||||
t.Helper()
|
||||
if err := f.MergeCell(sheet, topLeft, bottomRight); err != nil {
|
||||
t.Fatalf("MergeCell(%s:%s-%s): %v", sheet, topLeft, bottomRight, err)
|
||||
}
|
||||
}
|
||||
|
||||
func mustNewStyle(t *testing.T, f *excelize.File, style *excelize.Style) int {
|
||||
t.Helper()
|
||||
idx, err := f.NewStyle(style)
|
||||
if err != nil {
|
||||
t.Fatalf("NewStyle: %v", err)
|
||||
}
|
||||
return idx
|
||||
}
|
||||
|
||||
func mustSetCellStyle(t *testing.T, f *excelize.File, sheet, topLeft, bottomRight string, idx int) {
|
||||
t.Helper()
|
||||
if err := f.SetCellStyle(sheet, topLeft, bottomRight, idx); err != nil {
|
||||
t.Fatalf("SetCellStyle(%s:%s-%s): %v", sheet, topLeft, bottomRight, err)
|
||||
}
|
||||
}
|
||||
|
||||
func mustAddTable(t *testing.T, f *excelize.File, sheet string, table *excelize.Table) {
|
||||
t.Helper()
|
||||
if err := f.AddTable(sheet, table); err != nil {
|
||||
t.Fatalf("AddTable(%s): %v", sheet, err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRecordsToHTMLTableChunks_Alignment asserts the chunked output uses the
|
||||
// shared schema: <caption>, first row as <th>, data as <td>, repeated header
|
||||
// per 256-row chunk, and NO <thead>/<tbody> wrapper.
|
||||
func TestRecordsToHTMLTableChunks_Alignment(t *testing.T) {
|
||||
records := [][]string{{"Name", "Age"}, {"Alice", "30"}, {"Bob", "25"}}
|
||||
out := recordsToHTMLTableChunks(records, 256, "Sheet1")
|
||||
|
||||
if !strings.Contains(out, `<caption>Sheet1</caption>`) {
|
||||
t.Fatalf("want <caption>Sheet1</caption>, got:\n%s", out)
|
||||
}
|
||||
if !strings.Contains(out, "<tr><th>Name</th><th>Age</th></tr>") {
|
||||
t.Fatalf("want header row as <th>, got:\n%s", out)
|
||||
}
|
||||
if !strings.Contains(out, "<tr><td>Alice</td><td>30</td></tr>") {
|
||||
t.Fatalf("want data row as <td>, got:\n%s", out)
|
||||
}
|
||||
if strings.Contains(out, "<thead>") || strings.Contains(out, "<tbody>") {
|
||||
t.Fatalf("must not emit <thead>/<tbody> to stay byte-compatible with Python/CSV, got:\n%s", out)
|
||||
}
|
||||
// Exactly one <table> for <=256 data rows.
|
||||
if n := strings.Count(out, "<table>"); n != 1 {
|
||||
t.Fatalf("want 1 <table>, got %d:\n%s", n, out)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRecordsToHTMLTableChunks_Chunking asserts 256-row chunking with a repeated
|
||||
// header (ceil(n_data / chunk_rows) chunks).
|
||||
func TestRecordsToHTMLTableChunks_Chunking(t *testing.T) {
|
||||
const dataRows = 300
|
||||
records := make([][]string, 0, dataRows+1)
|
||||
records = append(records, []string{"C1", "C2"})
|
||||
for i := 0; i < dataRows; i++ {
|
||||
records = append(records, []string{"x", "y"})
|
||||
}
|
||||
out := recordsToHTMLTableChunks(records, 256, "S")
|
||||
// 300 data rows → ceil(300/256) = 2 chunks, each repeating the header.
|
||||
if n := strings.Count(out, "<table>"); n != 2 {
|
||||
t.Fatalf("want 2 <table> chunks, got %d", n)
|
||||
}
|
||||
if n := strings.Count(out, "<tr><th>C1</th><th>C2</th></tr>"); n != 2 {
|
||||
t.Fatalf("want header repeated in both chunks, got %d", n)
|
||||
}
|
||||
}
|
||||
|
||||
// TestXLSXParser_HeaderAndCaption asserts the XLSX parser emits a <caption> and
|
||||
// renders the first row as <th>, and that the header text appears only in <th>.
|
||||
func TestXLSXParser_HeaderAndCaption(t *testing.T) {
|
||||
data := newTestXLSX(t, func(f *excelize.File) {
|
||||
mustSetCell(t, f, "Sheet1", "A1", "Product")
|
||||
mustSetCell(t, f, "Sheet1", "B1", "Price")
|
||||
mustSetCell(t, f, "Sheet1", "A2", "Widget")
|
||||
mustSetCell(t, f, "Sheet1", "B2", "9.99")
|
||||
mustSetCell(t, f, "Sheet1", "A3", "Gadget")
|
||||
mustSetCell(t, f, "Sheet1", "B3", "19.99")
|
||||
})
|
||||
p, _ := NewXLSXParser("")
|
||||
res := p.ParseWithResult(t.Context(), "t.xlsx", data)
|
||||
if res.Err != nil {
|
||||
t.Fatalf("ParseWithResult: %v", res.Err)
|
||||
}
|
||||
html := res.HTML
|
||||
if !strings.Contains(html, `<caption>Sheet1</caption>`) {
|
||||
t.Fatalf("want <caption>Sheet1</caption>, got:\n%s", html)
|
||||
}
|
||||
if !strings.Contains(html, "<tr><th>Product</th><th>Price</th></tr>") {
|
||||
t.Fatalf("want header as <th>, got:\n%s", html)
|
||||
}
|
||||
// "Product" must only appear inside a <th>, never inside a <td>.
|
||||
if strings.Contains(html, "<td>Product</td>") {
|
||||
t.Fatalf("header value leaked into <td>:\n%s", html)
|
||||
}
|
||||
}
|
||||
|
||||
// TestXLSXParser_MergedHeaderInheritance asserts a horizontally merged header
|
||||
// cell's slave columns inherit the master text, so a wide merged title does not
|
||||
// render as a row of blank <th> cells.
|
||||
func TestXLSXParser_MergedHeaderInheritance(t *testing.T) {
|
||||
data := newTestXLSX(t, func(f *excelize.File) {
|
||||
// Row 1 is the header; A1:C1 merged into one wide label "Sales Report".
|
||||
mustSetCell(t, f, "Sheet1", "A1", "Sales Report")
|
||||
mustMergeCell(t, f, "Sheet1", "A1", "C1")
|
||||
// Make the header row bold so detection anchors row 1.
|
||||
idx := mustNewStyle(t, f, &excelize.Style{Font: &excelize.Font{Bold: true}})
|
||||
mustSetCellStyle(t, f, "Sheet1", "A1", "C1", idx)
|
||||
mustSetCell(t, f, "Sheet1", "A2", "North")
|
||||
mustSetCell(t, f, "Sheet1", "B2", "10")
|
||||
mustSetCell(t, f, "Sheet1", "C2", "20")
|
||||
mustSetCell(t, f, "Sheet1", "A3", "South")
|
||||
mustSetCell(t, f, "Sheet1", "B3", "30")
|
||||
mustSetCell(t, f, "Sheet1", "C3", "40")
|
||||
})
|
||||
p, _ := NewXLSXParser("")
|
||||
res := p.ParseWithResult(t.Context(), "t.xlsx", data)
|
||||
if res.Err != nil {
|
||||
t.Fatalf("ParseWithResult: %v", res.Err)
|
||||
}
|
||||
html := res.HTML
|
||||
// The merged master text must have propagated into all three <th> slots.
|
||||
if !strings.Contains(html, "<tr><th>Sales Report</th><th>Sales Report</th><th>Sales Report</th></tr>") {
|
||||
t.Fatalf("merged master text not inherited into header <th>, got:\n%s", html)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDetectHeaderRow_ListObject asserts a ListObject whose top row is > 1 is
|
||||
// detected as the header row.
|
||||
func TestDetectHeaderRow_ListObject(t *testing.T) {
|
||||
data := newTestXLSX(t, func(f *excelize.File) {
|
||||
mustSetCell(t, f, "Sheet1", "A1", "Title")
|
||||
mustSetCell(t, f, "Sheet1", "A2", "This is a banner")
|
||||
mustSetCell(t, f, "Sheet1", "A3", "Name")
|
||||
mustSetCell(t, f, "Sheet1", "B3", "Age")
|
||||
mustSetCell(t, f, "Sheet1", "A4", "Alice")
|
||||
mustSetCell(t, f, "Sheet1", "B4", "30")
|
||||
mustAddTable(t, f, "Sheet1", &excelize.Table{Range: "A3:B4", Name: "Table1"})
|
||||
})
|
||||
p, _ := NewXLSXParser("")
|
||||
res := p.ParseWithResult(t.Context(), "t.xlsx", data)
|
||||
if res.Err != nil {
|
||||
t.Fatalf("ParseWithResult: %v", res.Err)
|
||||
}
|
||||
if !strings.Contains(res.HTML, "<tr><th>Name</th><th>Age</th></tr>") {
|
||||
t.Fatalf("want ListObject header row detected, got:\n%s", res.HTML)
|
||||
}
|
||||
if strings.Contains(res.HTML, "<th>Title</th>") {
|
||||
t.Fatalf("title row must not be the header:\n%s", res.HTML)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDetectHeaderRow_Lightweight asserts that when row 1 is numeric data and
|
||||
// row 2 is a styled text label row, the lightweight detector picks row 2.
|
||||
func TestDetectHeaderRow_Lightweight(t *testing.T) {
|
||||
data := newTestXLSX(t, func(f *excelize.File) {
|
||||
mustSetCell(t, f, "Sheet1", "A1", "100")
|
||||
mustSetCell(t, f, "Sheet1", "B1", "200")
|
||||
mustSetCell(t, f, "Sheet1", "A2", "Item")
|
||||
mustSetCell(t, f, "Sheet1", "B2", "Count")
|
||||
// Make the header row bold so the styled signal fires.
|
||||
idx := mustNewStyle(t, f, &excelize.Style{Font: &excelize.Font{Bold: true}})
|
||||
mustSetCellStyle(t, f, "Sheet1", "A2", "B2", idx)
|
||||
mustSetCell(t, f, "Sheet1", "A3", "Apple")
|
||||
mustSetCell(t, f, "Sheet1", "B3", "5")
|
||||
})
|
||||
p, _ := NewXLSXParser("")
|
||||
res := p.ParseWithResult(t.Context(), "t.xlsx", data)
|
||||
if res.Err != nil {
|
||||
t.Fatalf("ParseWithResult: %v", res.Err)
|
||||
}
|
||||
if !strings.Contains(res.HTML, "<tr><th>Item</th><th>Count</th></tr>") {
|
||||
t.Fatalf("want row-2 header detected, got:\n%s", res.HTML)
|
||||
}
|
||||
if strings.Contains(res.HTML, "<th>100</th>") {
|
||||
t.Fatalf("numeric row 1 must not be the header:\n%s", res.HTML)
|
||||
}
|
||||
}
|
||||
|
||||
// TestXLSXParser_CommonCaseNoRegression asserts the dominant case (header on
|
||||
// row 1, no merges, no ListObject) renders row 1 as <th> unchanged.
|
||||
func TestXLSXParser_CommonCaseNoRegression(t *testing.T) {
|
||||
data := newTestXLSX(t, func(f *excelize.File) {
|
||||
mustSetCell(t, f, "Sheet1", "A1", "col_a")
|
||||
mustSetCell(t, f, "Sheet1", "B1", "col_b")
|
||||
mustSetCell(t, f, "Sheet1", "A2", "1")
|
||||
mustSetCell(t, f, "Sheet1", "B2", "2")
|
||||
})
|
||||
p, _ := NewXLSXParser("")
|
||||
res := p.ParseWithResult(t.Context(), "t.xlsx", data)
|
||||
if res.Err != nil {
|
||||
t.Fatalf("ParseWithResult: %v", res.Err)
|
||||
}
|
||||
if !strings.Contains(res.HTML, "<tr><th>col_a</th><th>col_b</th></tr>") {
|
||||
t.Fatalf("common-case header must render as <th>:\n%s", res.HTML)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDetectHeaderRow_BoldSubtotalNotHeader asserts that a bold "Total"-style
|
||||
// subtotal row sitting directly under a numeric row 1 is NOT promoted to the
|
||||
// header. The subtotal-label check blocks the override so the numeric row 1
|
||||
// stays the header.
|
||||
func TestDetectHeaderRow_BoldSubtotalNotHeader(t *testing.T) {
|
||||
data := newTestXLSX(t, func(f *excelize.File) {
|
||||
// Row 1: numeric year header (Python default header row).
|
||||
mustSetCell(t, f, "Sheet1", "A1", "2023")
|
||||
mustSetCell(t, f, "Sheet1", "B1", "2024")
|
||||
// Row 2: bold "Total/Summary" subtotal — must NOT become the header.
|
||||
idx := mustNewStyle(t, f, &excelize.Style{Font: &excelize.Font{Bold: true}})
|
||||
mustSetCell(t, f, "Sheet1", "A2", "Total")
|
||||
mustSetCell(t, f, "Sheet1", "B2", "Summary")
|
||||
mustSetCellStyle(t, f, "Sheet1", "A2", "B2", idx)
|
||||
// Row 3: purely numeric continuation under the subtotal.
|
||||
mustSetCell(t, f, "Sheet1", "A3", "100")
|
||||
mustSetCell(t, f, "Sheet1", "B3", "200")
|
||||
})
|
||||
p, _ := NewXLSXParser("")
|
||||
res := p.ParseWithResult(t.Context(), "t.xlsx", data)
|
||||
if res.Err != nil {
|
||||
t.Fatalf("ParseWithResult: %v", res.Err)
|
||||
}
|
||||
if !strings.Contains(res.HTML, "<tr><th>2023</th><th>2024</th></tr>") {
|
||||
t.Fatalf("numeric row 1 must remain the header:\n%s", res.HTML)
|
||||
}
|
||||
if strings.Contains(res.HTML, "<th>Total</th>") {
|
||||
t.Fatalf("bold subtotal row must not be promoted to header:\n%s", res.HTML)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDetectHeaderRow_StyledHeaderPastFarMerge asserts that a bold header on a
|
||||
// narrow row (row 3) is still detected even when an earlier wide merge
|
||||
// (A1:Z1 title) would otherwise pad every row to 26 columns. The padding step
|
||||
// must run AFTER header detection so the styled-majority signal is not diluted.
|
||||
func TestDetectHeaderRow_StyledHeaderPastFarMerge(t *testing.T) {
|
||||
data := newTestXLSX(t, func(f *excelize.File) {
|
||||
// Row 1: a wide merged title cell A1:Z1.
|
||||
mustSetCell(t, f, "Sheet1", "A1", "Sales Report")
|
||||
mustMergeCell(t, f, "Sheet1", "A1", "Z1")
|
||||
titleIdx := mustNewStyle(t, f, &excelize.Style{Font: &excelize.Font{Bold: true}})
|
||||
mustSetCellStyle(t, f, "Sheet1", "A1", "A1", titleIdx)
|
||||
// Row 3: the real, narrow, bold header.
|
||||
hdrIdx := mustNewStyle(t, f, &excelize.Style{Font: &excelize.Font{Bold: true}})
|
||||
mustSetCell(t, f, "Sheet1", "A3", "Name")
|
||||
mustSetCell(t, f, "Sheet1", "B3", "Desc")
|
||||
mustSetCellStyle(t, f, "Sheet1", "A3", "B3", hdrIdx)
|
||||
// Row 4: data (text in col A, so the body-brake passes for row 3).
|
||||
mustSetCell(t, f, "Sheet1", "A4", "Alice")
|
||||
mustSetCell(t, f, "Sheet1", "B4", "x")
|
||||
})
|
||||
p, _ := NewXLSXParser("")
|
||||
res := p.ParseWithResult(t.Context(), "t.xlsx", data)
|
||||
if res.Err != nil {
|
||||
t.Fatalf("ParseWithResult: %v", res.Err)
|
||||
}
|
||||
if !strings.Contains(res.HTML, "<th>Name</th><th>Desc</th>") {
|
||||
t.Fatalf("narrow bold header past far merge must be detected:\n%s", res.HTML)
|
||||
}
|
||||
if strings.Contains(res.HTML, "<th>Sales Report</th>") {
|
||||
t.Fatalf("wide merged title must not be the header:\n%s", res.HTML)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDetectHeaderRow_StyledTextHeaderOverNumeric asserts that a styled text
|
||||
// header sitting directly above a purely-numeric data row is still detected as
|
||||
// the header (not refused by the body-brake), while a bold "Total"/"Summary"
|
||||
// subtotal over numeric data is not.
|
||||
func TestDetectHeaderRow_StyledTextHeaderOverNumeric(t *testing.T) {
|
||||
data := newTestXLSX(t, func(f *excelize.File) {
|
||||
// Row 1: a single-cell title stub (fewer than 2 columns → skipped).
|
||||
mustSetCell(t, f, "Sheet1", "A1", "Report Title")
|
||||
// Row 2: the real, bold text header over numeric-only data below.
|
||||
idx := mustNewStyle(t, f, &excelize.Style{Font: &excelize.Font{Bold: true}})
|
||||
mustSetCell(t, f, "Sheet1", "A2", "Product")
|
||||
mustSetCell(t, f, "Sheet1", "B2", "Units")
|
||||
mustSetCellStyle(t, f, "Sheet1", "A2", "B2", idx)
|
||||
// Row 3: purely numeric continuation.
|
||||
mustSetCell(t, f, "Sheet1", "A3", "5")
|
||||
mustSetCell(t, f, "Sheet1", "B3", "10")
|
||||
})
|
||||
p, _ := NewXLSXParser("")
|
||||
res := p.ParseWithResult(t.Context(), "t.xlsx", data)
|
||||
if res.Err != nil {
|
||||
t.Fatalf("ParseWithResult: %v", res.Err)
|
||||
}
|
||||
if !strings.Contains(res.HTML, "<tr><th>Product</th><th>Units</th></tr>") {
|
||||
t.Fatalf("styled text header over numeric data must be detected:\n%s", res.HTML)
|
||||
}
|
||||
if strings.Contains(res.HTML, "<th>Report Title</th>") {
|
||||
t.Fatalf("title stub must not be the header:\n%s", res.HTML)
|
||||
}
|
||||
}
|
||||
|
||||
// TestInheritMergedHeader unit-tests the merge inheritance helper directly.
|
||||
func TestInheritMergedHeader(t *testing.T) {
|
||||
records := [][]string{
|
||||
{"MASTER", "", "Q2"}, // header row: B1 blank, merged from master A1
|
||||
{"a", "b", "c"},
|
||||
}
|
||||
mm := map[[2]int][2]int{
|
||||
{1, 1}: {1, 1},
|
||||
{1, 2}: {1, 1}, // B1's master is A1
|
||||
{1, 3}: {1, 3},
|
||||
}
|
||||
inheritMergedHeader(records, 1, mm)
|
||||
if records[0][1] != "MASTER" {
|
||||
t.Fatalf("expected B1 to inherit A1 master text, got %q", records[0][1])
|
||||
}
|
||||
if records[0][2] != "Q2" {
|
||||
t.Fatalf("expected non-merged C1 to keep its value, got %q", records[0][2])
|
||||
}
|
||||
}
|
||||
|
||||
// TestPadRowToWidth asserts the single-row padder widens with empty strings,
|
||||
// preserves existing cells, and never shrinks an already-wide row.
|
||||
func TestPadRowToWidth(t *testing.T) {
|
||||
row := []string{"a", "b"}
|
||||
padRowToWidth(&row, 5)
|
||||
if len(row) != 5 {
|
||||
t.Fatalf("want len 5, got %d", len(row))
|
||||
}
|
||||
if row[0] != "a" || row[1] != "b" {
|
||||
t.Fatalf("original cells lost: %v", row)
|
||||
}
|
||||
for i := 2; i < 5; i++ {
|
||||
if row[i] != "" {
|
||||
t.Fatalf("pad cell %d want empty, got %q", i, row[i])
|
||||
}
|
||||
}
|
||||
// Never shrinks an already-wide row.
|
||||
padRowToWidth(&row, 2)
|
||||
if len(row) != 5 {
|
||||
t.Fatalf("must not shrink: want 5, got %d", len(row))
|
||||
}
|
||||
}
|
||||
|
||||
// TestMergeExtentCol asserts the furthest merged column is reported as-is within
|
||||
// the cap and clamped to maxMergeExtentCols beyond it (the memory guard).
|
||||
func TestMergeExtentCol(t *testing.T) {
|
||||
if got := mergeExtentCol(nil); got != 0 {
|
||||
t.Fatalf("nil ranges: want 0, got %d", got)
|
||||
}
|
||||
if got := mergeExtentCol([]mergeRange{{1, 1, 3, 10}}); got != 10 {
|
||||
t.Fatalf("within cap: want 10, got %d", got)
|
||||
}
|
||||
if got := mergeExtentCol([]mergeRange{{1, 1, 1, 5000}}); got != maxMergeExtentCols {
|
||||
t.Fatalf("beyond cap: want %d, got %d", maxMergeExtentCols, got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDecodeChunkRows exercises the chunk_rows setup knob across all the types
|
||||
// JSON/yaml decoding can produce, plus the defaulting paths.
|
||||
func TestDecodeChunkRows(t *testing.T) {
|
||||
if got := decodeChunkRows(nil); got != defaultTableChunkRows {
|
||||
t.Fatalf("nil setup: want default %d, got %d", defaultTableChunkRows, got)
|
||||
}
|
||||
if got := decodeChunkRows(map[string]any{}); got != defaultTableChunkRows {
|
||||
t.Fatalf("empty setup: want default, got %d", got)
|
||||
}
|
||||
if got := decodeChunkRows(map[string]any{"chunk_rows": 100.0}); got != 100 {
|
||||
t.Fatalf("float64 100: want 100, got %d", got)
|
||||
}
|
||||
if got := decodeChunkRows(map[string]any{"chunk_rows": 0.0}); got != defaultTableChunkRows {
|
||||
t.Fatalf("float64 0: want default, got %d", got)
|
||||
}
|
||||
if got := decodeChunkRows(map[string]any{"chunk_rows": -5.0}); got != defaultTableChunkRows {
|
||||
t.Fatalf("float64 -5: want default, got %d", got)
|
||||
}
|
||||
if got := decodeChunkRows(map[string]any{"chunk_rows": 256}); got != 256 {
|
||||
t.Fatalf("int 256: want 256, got %d", got)
|
||||
}
|
||||
if got := decodeChunkRows(map[string]any{"chunk_rows": int64(512)}); got != 512 {
|
||||
t.Fatalf("int64 512: want 512, got %d", got)
|
||||
}
|
||||
if got := decodeChunkRows(map[string]any{"chunk_rows": "256"}); got != defaultTableChunkRows {
|
||||
t.Fatalf("string (unsupported type): want default, got %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRenderSheetTables_FarMergeDoesNotBloatDataRows guards the memory blow-up
|
||||
// from review: a far merge on a non-header row must not widen the header or any
|
||||
// data row to the merge's full span. The header stays its natural width and no
|
||||
// data row carries empty <td> cells.
|
||||
func TestRenderSheetTables_FarMergeDoesNotBloatDataRows(t *testing.T) {
|
||||
data := newTestXLSX(t, func(f *excelize.File) {
|
||||
// Row 1: a wide merged title A1:Z1 (NOT the header).
|
||||
mustSetCell(t, f, "Sheet1", "A1", "Report Title")
|
||||
mustMergeCell(t, f, "Sheet1", "A1", "Z1")
|
||||
// Row 2: the real header, made bold so detection anchors it.
|
||||
hdrIdx := mustNewStyle(t, f, &excelize.Style{Font: &excelize.Font{Bold: true}})
|
||||
mustSetCell(t, f, "Sheet1", "A2", "Name")
|
||||
mustSetCell(t, f, "Sheet1", "B2", "Price")
|
||||
mustSetCellStyle(t, f, "Sheet1", "A2", "B2", hdrIdx)
|
||||
// Rows 3-4: data.
|
||||
mustSetCell(t, f, "Sheet1", "A3", "Alice")
|
||||
mustSetCell(t, f, "Sheet1", "B3", "9.99")
|
||||
mustSetCell(t, f, "Sheet1", "A4", "Bob")
|
||||
mustSetCell(t, f, "Sheet1", "B4", "19.99")
|
||||
})
|
||||
p, _ := NewXLSXParser("")
|
||||
res := p.ParseWithResult(t.Context(), "t.xlsx", data)
|
||||
if res.Err != nil {
|
||||
t.Fatalf("ParseWithResult: %v", res.Err)
|
||||
}
|
||||
html := res.HTML
|
||||
if !strings.Contains(html, "<tr><th>Name</th><th>Price</th></tr>") {
|
||||
t.Fatalf("want row-2 header detected, got:\n%s", html)
|
||||
}
|
||||
if strings.Contains(html, "<th>Title</th>") {
|
||||
t.Fatalf("wide merged title must not be the header:\n%s", html)
|
||||
}
|
||||
// The far merge is on row 1, not the header, so nothing is widened to 26
|
||||
// columns: the header keeps 2 columns (no empty <th>) and data rows carry
|
||||
// no empty <td>.
|
||||
if strings.Contains(html, "<th></th>") {
|
||||
t.Fatalf("header was bloated by the far merge:\n%s", html)
|
||||
}
|
||||
if strings.Count(html, "<td></td>") != 0 {
|
||||
t.Fatalf("data rows were bloated by the far merge:\n%s", html)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRenderSheetTables_MultiSheet asserts each sheet becomes its own <table>
|
||||
// chunk, each carrying its own <caption>.
|
||||
func TestRenderSheetTables_MultiSheet(t *testing.T) {
|
||||
data := newTestXLSX(t, func(f *excelize.File) {
|
||||
mustSetCell(t, f, "Sheet1", "A1", "Name")
|
||||
mustSetCell(t, f, "Sheet1", "B1", "Age")
|
||||
mustSetCell(t, f, "Sheet1", "A2", "Alice")
|
||||
mustSetCell(t, f, "Sheet1", "B2", "30")
|
||||
if _, err := f.NewSheet("HR"); err != nil {
|
||||
t.Fatalf("NewSheet: %v", err)
|
||||
}
|
||||
mustSetCell(t, f, "HR", "A1", "Dept")
|
||||
mustSetCell(t, f, "HR", "B1", "Head")
|
||||
mustSetCell(t, f, "HR", "A2", "Eng")
|
||||
mustSetCell(t, f, "HR", "B2", "Bob")
|
||||
})
|
||||
p, _ := NewXLSXParser("")
|
||||
res := p.ParseWithResult(t.Context(), "t.xlsx", data)
|
||||
if res.Err != nil {
|
||||
t.Fatalf("ParseWithResult: %v", res.Err)
|
||||
}
|
||||
html := res.HTML
|
||||
if !strings.Contains(html, "<caption>Sheet1</caption>") {
|
||||
t.Fatalf("want Sheet1 caption, got:\n%s", html)
|
||||
}
|
||||
if !strings.Contains(html, "<caption>HR</caption>") {
|
||||
t.Fatalf("want HR caption, got:\n%s", html)
|
||||
}
|
||||
// Two small sheets → two <table> chunks.
|
||||
if n := strings.Count(html, "<table>"); n != 2 {
|
||||
t.Fatalf("want 2 <table> chunks, got %d:\n%s", n, html)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRenderSheetTables_EmptySheet asserts an empty/unreadable sheet yields an
|
||||
// empty string rather than an empty <table> wrapper.
|
||||
func TestRenderSheetTables_EmptySheet(t *testing.T) {
|
||||
// excelize.NewFile yields a single empty "Sheet1".
|
||||
data := newTestXLSX(t, func(f *excelize.File) {})
|
||||
p, _ := NewXLSXParser("")
|
||||
res := p.ParseWithResult(t.Context(), "t.xlsx", data)
|
||||
if res.Err != nil {
|
||||
t.Fatalf("ParseWithResult: %v", res.Err)
|
||||
}
|
||||
if res.HTML != "" {
|
||||
t.Fatalf("empty sheet must yield empty HTML, got:\n%s", res.HTML)
|
||||
}
|
||||
}
|
||||
@@ -29,6 +29,7 @@ type XLSParser struct {
|
||||
libType string
|
||||
ParseMethod string
|
||||
OutputFormat string
|
||||
ChunkRows int
|
||||
TCADPAPIServer string
|
||||
TCADPAPIKey string
|
||||
TCADPTableResultType string
|
||||
@@ -41,6 +42,7 @@ func NewXLSParser(libType string) (*XLSParser, error) {
|
||||
}
|
||||
return &XLSParser{
|
||||
libType: libType,
|
||||
ChunkRows: defaultTableChunkRows,
|
||||
TCADPTableResultType: "1",
|
||||
TCADPMarkdownImageResponseType: "1",
|
||||
}, nil
|
||||
@@ -60,6 +62,7 @@ func (p *XLSParser) ConfigureFromSetup(setup map[string]any) {
|
||||
if v, ok := setup["output_format"].(string); ok && v != "" {
|
||||
p.OutputFormat = v
|
||||
}
|
||||
p.ChunkRows = decodeChunkRows(setup)
|
||||
if v, ok := setup["tcadp_apiserver"].(string); ok && v != "" {
|
||||
p.TCADPAPIServer = v
|
||||
}
|
||||
@@ -98,26 +101,16 @@ func (p *XLSParser) ParseWithResult(ctx context.Context, filename string, data [
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
var html strings.Builder
|
||||
html.WriteString("<html><body>")
|
||||
for _, sheet := range f.GetSheetList() {
|
||||
html.WriteString("<h3>")
|
||||
html.WriteString(sheet)
|
||||
html.WriteString("</h3>")
|
||||
rows, _ := f.GetRows(sheet)
|
||||
html.WriteString("<table>")
|
||||
for _, row := range rows {
|
||||
html.WriteString("<tr>")
|
||||
for _, cell := range row {
|
||||
html.WriteString("<td>")
|
||||
html.WriteString(htmlEscape(cell))
|
||||
html.WriteString("</td>")
|
||||
}
|
||||
html.WriteString("</tr>")
|
||||
}
|
||||
html.WriteString("</table>")
|
||||
sheets := f.GetSheetList()
|
||||
chunkRows := p.ChunkRows
|
||||
if chunkRows <= 0 {
|
||||
chunkRows = defaultTableChunkRows
|
||||
}
|
||||
|
||||
var html strings.Builder
|
||||
for _, sheet := range sheets {
|
||||
html.WriteString(renderSheetTables(f, sheet, chunkRows))
|
||||
}
|
||||
html.WriteString("</body></html>")
|
||||
|
||||
return ParseResult{
|
||||
OutputFormat: "html",
|
||||
|
||||
@@ -29,6 +29,7 @@ type XLSXParser struct {
|
||||
libType string
|
||||
ParseMethod string
|
||||
OutputFormat string
|
||||
ChunkRows int
|
||||
TCADPAPIServer string
|
||||
TCADPAPIKey string
|
||||
TCADPTableResultType string
|
||||
@@ -41,6 +42,7 @@ func NewXLSXParser(libType string) (*XLSXParser, error) {
|
||||
}
|
||||
return &XLSXParser{
|
||||
libType: libType,
|
||||
ChunkRows: defaultTableChunkRows,
|
||||
TCADPTableResultType: "1",
|
||||
TCADPMarkdownImageResponseType: "1",
|
||||
}, nil
|
||||
@@ -60,6 +62,7 @@ func (p *XLSXParser) ConfigureFromSetup(setup map[string]any) {
|
||||
if v, ok := setup["output_format"].(string); ok && v != "" {
|
||||
p.OutputFormat = v
|
||||
}
|
||||
p.ChunkRows = decodeChunkRows(setup)
|
||||
if v, ok := setup["tcadp_apiserver"].(string); ok && v != "" {
|
||||
p.TCADPAPIServer = v
|
||||
}
|
||||
@@ -117,29 +120,15 @@ func (p *XLSXParser) ParseWithResult(ctx context.Context, filename string, data
|
||||
defer f.Close()
|
||||
|
||||
sheets := f.GetSheetList()
|
||||
var html strings.Builder
|
||||
html.WriteString("<html><body>")
|
||||
for _, sheet := range sheets {
|
||||
html.WriteString("<h3>")
|
||||
html.WriteString(sheet)
|
||||
html.WriteString("</h3>")
|
||||
rows, err := f.GetRows(sheet)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
html.WriteString("<table>")
|
||||
for _, row := range rows {
|
||||
html.WriteString("<tr>")
|
||||
for _, cell := range row {
|
||||
html.WriteString("<td>")
|
||||
html.WriteString(htmlEscape(cell))
|
||||
html.WriteString("</td>")
|
||||
}
|
||||
html.WriteString("</tr>")
|
||||
}
|
||||
html.WriteString("</table>")
|
||||
chunkRows := p.ChunkRows
|
||||
if chunkRows <= 0 {
|
||||
chunkRows = defaultTableChunkRows
|
||||
}
|
||||
|
||||
var html strings.Builder
|
||||
for _, sheet := range sheets {
|
||||
html.WriteString(renderSheetTables(f, sheet, chunkRows))
|
||||
}
|
||||
html.WriteString("</body></html>")
|
||||
|
||||
return ParseResult{
|
||||
OutputFormat: "html",
|
||||
|
||||
Reference in New Issue
Block a user