diff --git a/internal/parser/parser/csv_parser.go b/internal/parser/parser/csv_parser.go index c37e5d88f6..8505fb90ba 100644 --- a/internal/parser/parser/csv_parser.go +++ b/internal/parser/parser/csv_parser.go @@ -34,18 +34,10 @@ import ( "context" "encoding/csv" "fmt" - "html" - "regexp" "strings" ) -// Go port of Python's ILLEGAL_CHARACTERS_RE from -// deepdoc/parser/excel_parser.py. -// Pattern: [\000-\010]|[\013-\014]|[\016-\037] -// Matches all control chars except TAB (\x09), LF (\x0A), CR (\x0D). -var csvIllegalCharsRe = regexp.MustCompile(`[\x00-\x08]|\x0B|\x0C|[\x0E-\x1F]`) - -const csvDefaultChunkRows = 256 +const csvDefaultChunkRows = defaultTableChunkRows const csvSheetName = "Data" // CSVParser reads RFC-4180 CSV data and emits HTML payloads. @@ -157,7 +149,7 @@ func (p *CSVParser) ParseWithResult(ctx context.Context, filename string, data [ } // Clean illegal control characters from all cells. - records = cleanCSVRecords(records) + records = cleanIllegalControlChars(records) chunkRows := p.ChunkRows if chunkRows <= 0 { @@ -171,84 +163,6 @@ func (p *CSVParser) ParseWithResult(ctx context.Context, filename string, data [ "size": len(data), "encoding": "utf-8", }, - HTML: recordsToHTMLTableChunks(records, chunkRows), + HTML: recordsToHTMLTableChunks(records, chunkRows, csvSheetName), } } - -// cleanCSVRecords replaces illegal control characters in all cells -// with a single space, matching Python's ILLEGAL_CHARACTERS_RE. -func cleanCSVRecords(records [][]string) [][]string { - out := make([][]string, len(records)) - for i, row := range records { - out[i] = make([]string, len(row)) - for j, cell := range row { - out[i][j] = csvIllegalCharsRe.ReplaceAllString(cell, " ") - } - } - return out -} - -// recordsToHTMLTableChunks renders CSV records as one or more -// self-contained HTML
chunks. The first row is always the -// header (
). Data rows are split into chunks of chunkRows, -// each chunk being a complete with ") - for _, cell := range row { - b.WriteString("") - } - b.WriteString("\n") - return b.String() -} diff --git a/internal/parser/parser/office_parsers_cgo_shared.go b/internal/parser/parser/office_parsers_cgo_shared.go deleted file mode 100644 index d128fee295..0000000000 --- a/internal/parser/parser/office_parsers_cgo_shared.go +++ /dev/null @@ -1,9 +0,0 @@ -//go:build cgo - -package parser - -import "html" - -func htmlEscape(s string) string { - return html.EscapeString(s) -} diff --git a/internal/parser/parser/office_parsers_shared.go b/internal/parser/parser/office_parsers_shared.go index e9411f3af5..3f9e957761 100644 --- a/internal/parser/parser/office_parsers_shared.go +++ b/internal/parser/parser/office_parsers_shared.go @@ -2,11 +2,5 @@ package parser -import "html" - // OfficeOxide is the lib_type identifier for office_oxide backend. const OfficeOxide = "office_oxide" - -func htmlEscape(s string) string { - return html.EscapeString(s) -} diff --git a/internal/parser/parser/office_table_render.go b/internal/parser/parser/office_table_render.go new file mode 100644 index 0000000000..43a844417d --- /dev/null +++ b/internal/parser/parser/office_table_render.go @@ -0,0 +1,548 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language of permissions and +// limitations under the License. +// + +package parser + +import ( + "html" + "regexp" + "strconv" + "strings" + + "github.com/xuri/excelize/v2" +) + +// tableIllegalCharsRe replaces illegal control characters (everything except +// TAB/LF/CR) with a single space. The pattern matches all C0 control chars +// except TAB (0x09), LF (0x0A) and CR (0x0D). +var tableIllegalCharsRe = regexp.MustCompile(`[\x00-\x08]|\x0B|\x0C|[\x0E-\x1F]`) + +// numericCellRe recognises number-like cell text (integers, decimals, comma +// thousands separators, optional leading sign/$ and trailing %). It is a +// deliberately loose heuristic used only for header-vs-data contrast, not for +// value parsing. +var numericCellRe = regexp.MustCompile(`^[\$\+\-]?[\d,]+(\.\d+)?%?$`) + +const defaultTableChunkRows = 256 + +// maxMergeExtentCols caps how far a merged range may widen the header row. +// excelize's GetRows truncates each row at its last valued cell, so a merged +// slave beyond that point is absent and cannot inherit its master's text; we +// pad the header row to the furthest merged column to recover it. A pathological +// far merge (e.g. A1:XFD1) must not be allowed to allocate one cell per column +// for every row, so the width is capped here. +const maxMergeExtentCols = 1024 + +// cleanIllegalControlChars replaces illegal control characters in all cells +// with a single space. +func cleanIllegalControlChars(records [][]string) [][]string { + out := make([][]string, len(records)) + for i, row := range records { + out[i] = make([]string, len(row)) + for j, cell := range row { + out[i][j] = tableIllegalCharsRe.ReplaceAllString(cell, " ") + } + } + return out +} + +// buildHeaderRow renders a row as an HTML ") + for _, cell := range row { + b.WriteString("") + } + b.WriteString("\n") + return b.String() +} + +// recordsToHTMLTableChunks renders records as one or more self-contained HTML +//
and a repeated -// header row. Chunks are joined with newlines. -// -// Mirrors Python's RAGFlowExcelParser.html() chunking: -// -// chunks = (n_data_rows + chunk_rows - 1) // chunk_rows -func recordsToHTMLTableChunks(records [][]string, chunkRows int) string { - if len(records) == 0 { - return "
" + csvSheetName + "
" - } - - // Build the header row once — repeated in every chunk. - headerHTML := buildCSVHeaderRow(records[0]) - dataRows := records[1:] - nData := len(dataRows) - - if nData == 0 { - // Only a header row exists. - return "\n" + headerHTML + "\n
" + csvSheetName + "
" - } - - nChunks := (nData + chunkRows - 1) / chunkRows - var b strings.Builder - for ci := 0; ci < nChunks; ci++ { - start := ci * chunkRows - end := start + chunkRows - if end > nData { - end = nData - } - - b.WriteString("\n") - b.WriteString(headerHTML) - - for _, row := range dataRows[start:end] { - b.WriteString("") - for _, cell := range row { - b.WriteString("") - } - b.WriteString("\n") - } - b.WriteString("
") - b.WriteString(csvSheetName) - b.WriteString("
") - b.WriteString(html.EscapeString(strings.TrimSpace(cell))) - b.WriteString("
\n") - } - return b.String() -} - -// buildCSVHeaderRow renders the first row as an HTML
header row. -func buildCSVHeaderRow(row []string) string { - var b strings.Builder - b.WriteString("
") - b.WriteString(html.EscapeString(strings.TrimSpace(cell))) - b.WriteString("
header row. +func buildHeaderRow(row []string) string { + var b strings.Builder + b.WriteString("
") + b.WriteString(html.EscapeString(strings.TrimSpace(cell))) + b.WriteString("
chunks. The first row is always the header (
). Data rows are +// split into chunks of chunkRows, each chunk being a complete with +// /, so every
and a repeated header row. Chunks are joined with newlines. +// +// The tag schema is +// …
{caption}
…
…
. Rows are intentionally NOT wrapped in +//
is one atomic chunk that downstream +// chunkers can consume independently. +func recordsToHTMLTableChunks(records [][]string, chunkRows int, caption string) string { + if len(records) == 0 { + return "
" + html.EscapeString(caption) + "
" + } + + // Build the header row once — repeated in every chunk. + headerHTML := buildHeaderRow(records[0]) + dataRows := records[1:] + nData := len(dataRows) + + if nData == 0 { + // Only a header row exists. + return "\n" + headerHTML + "
" + html.EscapeString(caption) + "
" + } + + if chunkRows <= 0 { + chunkRows = defaultTableChunkRows + } + + nChunks := (nData + chunkRows - 1) / chunkRows + var b strings.Builder + for ci := 0; ci < nChunks; ci++ { + start := ci * chunkRows + end := start + chunkRows + if end > nData { + end = nData + } + + b.WriteString("\n") + b.WriteString(headerHTML) + + for _, row := range dataRows[start:end] { + b.WriteString("") + for _, cell := range row { + b.WriteString("") + } + b.WriteString("\n") + } + b.WriteString("
") + b.WriteString(html.EscapeString(caption)) + b.WriteString("
") + b.WriteString(html.EscapeString(strings.TrimSpace(cell))) + b.WriteString("
\n") + } + return b.String() +} + +// ──────────────────────────────────────────────────────────── axis helpers + +// axisToRC parses an "A1"-style cell reference (optionally with a leading "$") +// into 1-based (row, col). A malformed reference yields (0, 0). +func axisToRC(axis string) (row, col int) { + axis = strings.TrimSpace(axis) + axis = strings.TrimPrefix(axis, "$") + i := 0 + for i < len(axis) && (axis[i] < '0' || axis[i] > '9') { + i++ + } + if i == 0 || i == len(axis) { + return 0, 0 + } + letter, num := axis[:i], axis[i:] + r, err := strconv.Atoi(num) + if err != nil { + return 0, 0 + } + c := 0 + for _, ch := range strings.ToUpper(letter) { + if ch < 'A' || ch > 'Z' { + return 0, 0 + } + c = c*26 + int(ch-'A'+1) + } + return r, c +} + +// rangeTopRow returns the top (first) row of an A1:B10-style range reference. +func rangeTopRow(ref string) int { + ref = strings.SplitN(ref, ":", 2)[0] + r, _ := axisToRC(ref) + return r +} + +// cellAxis builds an "A1"-style reference for 1-based (row, col). +func cellAxis(row, col int) string { + // Build the column letters least-significant digit first, then reverse. + var digits []byte + c := col + for c > 0 { + c-- // 1-based → 0-based for this digit + digits = append(digits, byte('A'+c%26)) + c /= 26 + } + for i, j := 0, len(digits)-1; i < j; i, j = i+1, j-1 { + digits[i], digits[j] = digits[j], digits[i] + } + return string(digits) + strconv.Itoa(row) +} + +// ──────────────────────────────────────────────────────────── merge inheritance + +// mergeRange is an excelize merged-cell rectangle, 1-based inclusive. +type mergeRange struct { + sr, sc, er, ec int +} + +// mergeRanges returns the merged-cell rectangles of a sheet. +func mergeRanges(f *excelize.File, sheet string) []mergeRange { + var out []mergeRange + cells, err := f.GetMergeCells(sheet) + if err != nil { + return out + } + for _, mc := range cells { + sr, sc := axisToRC(mc.GetStartAxis()) + er, ec := axisToRC(mc.GetEndAxis()) + if sr == 0 || sc == 0 || er == 0 || ec == 0 { + continue + } + out = append(out, mergeRange{sr, sc, er, ec}) + } + return out +} + +// mergeMaxCol returns the furthest merged column across all ranges, or 0 if +// there are none. +func mergeMaxCol(ranges []mergeRange) int { + m := 0 + for _, r := range ranges { + if r.ec > m { + m = r.ec + } + } + return m +} + +// mergeMasterForRow materialises the slave→master map for a single row only. +// A large merged block (e.g. A1:Z100) would otherwise expand to every one of +// its cells; we only ever need the header row's slaves, so expanding per row +// keeps the map O(cols) instead of O(rows×cols). +func mergeMasterForRow(ranges []mergeRange, row int) map[[2]int][2]int { + mm := map[[2]int][2]int{} + for _, r := range ranges { + if row < r.sr || row > r.er { + continue + } + for c := r.sc; c <= r.ec; c++ { + mm[[2]int{row, c}] = [2]int{r.sr, r.sc} + } + } + return mm +} + +// inheritMergedHeader fills empty cells in the header row with the value of +// their merge master (typically a wide horizontally-merged title cell). This +// keeps a wide merged title from rendering as a row of blank 		cells. +func inheritMergedHeader(records [][]string, headerRowIdx int, mm map[[2]int][2]int) { + if headerRowIdx < 1 || headerRowIdx > len(records) { + return + } + row := records[headerRowIdx-1] + for c := 0; c < len(row); c++ { + if strings.TrimSpace(row[c]) != "" { + continue + } + master, ok := mm[[2]int{headerRowIdx, c + 1}] + if !ok || (master[0] == headerRowIdx && master[1] == c+1) { + continue + } + if master[0] < 1 || master[0] > len(records) || master[1] < 1 || master[1] > len(records[master[0]-1]) { + continue + } + val := records[master[0]-1][master[1]-1] + if strings.TrimSpace(val) != "" { + row[c] = val + } + } +} + +// mergeExtentCol returns the furthest merged column across all ranges, capped +// at maxMergeExtentCols so a pathological far merge cannot exhaust parser +// memory when the header row is padded to inherit merged text. +func mergeExtentCol(ranges []mergeRange) int { + m := mergeMaxCol(ranges) + if m > maxMergeExtentCols { + return maxMergeExtentCols + } + return m +} + +// padRowToWidth grows a single row to at least maxCol, padding with empty +// strings. Only the header row is padded (see renderSheetTables): merged-master +// text is inherited into the header alone, so data rows must not be widened — +// widening them would emit a sea of empty cells for every far merge in the +// sheet and is the memory blow-up flagged in review. +func padRowToWidth(row *[]string, maxCol int) { + if maxCol <= len(*row) { + return + } + padded := make([]string, maxCol) + copy(padded, *row) + *row = padded +} + +// ──────────────────────────────────────────────────────────── header detection + +// isNumericCell reports whether a cell value reads as a number-like string. +func isNumericCell(s string) bool { + s = strings.TrimSpace(s) + return s != "" && numericCellRe.MatchString(s) +} + +// cellIsStyled reports whether a cell is bold or carries a fill, using +// excelize's style lookup. +func cellIsStyled(f *excelize.File, sheet string, row, col int) bool { + idx, err := f.GetCellStyle(sheet, cellAxis(row, col)) + if err != nil { + return false + } + st, err := f.GetStyle(idx) + if err != nil || st == nil { + return false + } + if st.Font != nil && st.Font.Bold { + return true + } + if st.Fill.Type != "" || len(st.Fill.Color) > 0 { + return true + } + return false +} + +// detectHeaderRow returns the 1-based row that should be treated as the column +// header of a sheet, defaulting to 1. It only diverges from row 1 when there is +// high confidence that row 1 is not the header: +// +// 1. ListObject first: if the sheet defines Excel tables (ListObjects) whose +// top row is > 1, that row is the header. This is the cheapest, most +// accurate signal and never fires for the common case (table starts at row 1). +// 2. Lightweight detection (no ListObject override): scan the top few rows for +// an anchor — a row whose cells are mostly bold/filled, or a row holding +// text labels over numeric data columns below (contrast ≥ 2 columns). A +// candidate that looks like a data row (majority numeric) is skipped. The +// override only applies when the anchor is not already row 1, so the common +// header-on-row-1 sheet is left unchanged. +func detectHeaderRow(f *excelize.File, sheet string, records [][]string) int { + n := len(records) + if n == 0 { + return 1 + } + + // 1) ListObject first. + if tables, err := f.GetTables(sheet); err == nil && len(tables) > 0 { + minTop := 0 + for _, t := range tables { + tr := rangeTopRow(t.Range) + if tr < 1 { + continue + } + if minTop == 0 || tr < minTop { + minTop = tr + } + } + if minTop > 1 { + return minTop + } + } + + // 2) Lightweight detection over the top window (bounded to 4 candidate rows, + // and we must leave at least one data row below the candidate). + maxScan := 4 + if maxScan > n-1 { + maxScan = n - 1 + } + if maxScan < 1 { + return 1 + } + for k := 0; k < maxScan; k++ { + row := records[k] + nc := len(row) + if nc < 2 { + continue // title / section stub — keep looking below it + } + + // Data-majority reject: a row that is mostly numbers is data, not a + // header. Keep scanning downward for the real header. + nonEmpty, numeric := 0, 0 + for _, v := range row { + if v == "" { + continue + } + nonEmpty++ + if isNumericCell(v) { + numeric++ + } + } + if nonEmpty > 0 && numeric*2 >= nonEmpty { + continue + } + + // Styled signal. + styled := 0 + for c := 0; c < nc && c < 64; c++ { + if cellIsStyled(f, sheet, k+1, c+1) { + styled++ + } + } + + // Contrast signal: text label over a numeric column below. + contrast := 0 + if k+1 < n { + below := records[k+1] + for c := 0; c < nc && c < 64; c++ { + v := row[c] + if v == "" || isNumericCell(v) { + continue + } + if c < len(below) && isNumericCell(below[c]) { + contrast++ + if contrast >= 2 { + break + } + } + } + } + + if styled*2 >= nc || contrast >= 2 { + if k == 0 { + return 1 // row 1 is already the header + } + below := records[k+1] + // Accept the candidate as the header when the row directly below + // is a genuine data row (it contains at least one text cell), or + // when that row is purely numeric but the candidate is not a + // subtotal label. The second clause lets a styled text header + // sitting above numeric-only data win; a bold "Total"/"Summary" + // subtotal looks identical locally, but its label matches a + // subtotal keyword so it is still refused and row 1 is kept. + if rowHasTextCell(below) || (isPurelyNumeric(below) && !isSubtotalRow(row)) { + return k + 1 + } + } + } + return 1 +} + +// rowHasTextCell reports whether a row contains at least one non-empty, +// non-numeric (i.e. text) cell. It is used as a body-signature brake for +// header detection. +func rowHasTextCell(row []string) bool { + for _, v := range row { + if v != "" && !isNumericCell(v) { + return true + } + } + return false +} + +// isPurelyNumeric reports whether every non-empty cell of a row reads as a +// number-like string. It is used to recognise a numeric continuation row below +// a candidate header. +func isPurelyNumeric(row []string) bool { + nonEmpty := 0 + for _, v := range row { + if strings.TrimSpace(v) == "" { + continue + } + nonEmpty++ + if !isNumericCell(v) { + return false + } + } + return nonEmpty > 0 +} + +// subtotalWordRe matches labels that mark a totals/subtotals row. A candidate +// header sitting above a purely-numeric row is refused when any of its cells +// matches, so bold "Total"/"Summary" subtotals are not promoted to the header. +var subtotalWordRe = regexp.MustCompile(`^(total|totals|sum|summary|subtotal|subtotals|grand total|合计|总计|小计|汇总|总额)$`) + +// isSubtotalRow reports whether any non-empty cell of a row is a subtotal label +// (case-insensitive, tolerating a trailing colon). +func isSubtotalRow(row []string) bool { + for _, v := range row { + v = strings.ToLower(strings.TrimSpace(v)) + v = strings.TrimRight(v, ":：") + if v != "" && subtotalWordRe.MatchString(v) { + return true + } + } + return false +} + +// decodeChunkRows reads the "chunk_rows" setup knob, returning the default when +// it is absent or non-positive. +func decodeChunkRows(setup map[string]any) int { + if setup == nil { + return defaultTableChunkRows + } + v, ok := setup["chunk_rows"] + if !ok { + return defaultTableChunkRows + } + switch n := v.(type) { + case float64: + rows := int(n) + if rows <= 0 { + return defaultTableChunkRows + } + return rows + case int: + if n <= 0 { + return defaultTableChunkRows + } + return n + case int64: + rows := int(n) + if rows <= 0 { + return defaultTableChunkRows + } + return rows + } + return defaultTableChunkRows +} + +// renderSheetTables renders a single workbook sheet into one or more +// self-contained chunks using the shared spreadsheet-HTML contract: +// detect the header row, inherit merged-master text into the header, and split +// data into chunkRows-sized atomic tables each repeating the header. An empty +// or unreadable sheet yields an empty string. +func renderSheetTables(f *excelize.File, sheet string, chunkRows int) string { + rows, err := f.GetRows(sheet) + if err != nil || len(rows) == 0 { + return "" + } + rows = cleanIllegalControlChars(rows) + + ranges := mergeRanges(f, sheet) + headerRow := detectHeaderRow(f, sheet, rows) + + // Inherit merged-master text into the header row. excelize's GetRows + // truncates each row at its last valued cell, so a merged slave beyond that + // point is absent and cannot inherit its master's text. We therefore pad + // ONLY the header row (the only row we inherit into) to the furthest merged + // column — capped by mergeExtentCol so a pathological far merge cannot + // exhaust memory. Padding runs after detection so a wide merge (e.g. A1:Z1 + // title) does not dilute the styled-majority signal of a narrow header. + mm := mergeMasterForRow(ranges, headerRow) + if len(mm) > 0 { + padRowToWidth(&rows[headerRow-1], mergeExtentCol(ranges)) + } + inheritMergedHeader(rows, headerRow, mm) + + // Reorder so the detected header row becomes records[0]; every other row is + // data. For the common case (header on row 1) this is a no-op. + records := make([][]string, 0, len(rows)) + records = append(records, rows[headerRow-1]) + for i, r := range rows { + if i == headerRow-1 { + continue + } + records = append(records, r) + } + return recordsToHTMLTableChunks(records, chunkRows, sheet) +} diff --git a/internal/parser/parser/office_table_render_test.go b/internal/parser/parser/office_table_render_test.go new file mode 100644 index 0000000000..1d7453d3b0 --- /dev/null +++ b/internal/parser/parser/office_table_render_test.go @@ -0,0 +1,506 @@ +package parser + +import ( + "strings" + "testing" + + "github.com/xuri/excelize/v2" +) + +// newTestXLSX builds an in-memory .xlsx from a cell writer. +func newTestXLSX(t *testing.T, fill func(f *excelize.File)) []byte { + t.Helper() + f := excelize.NewFile() + defer f.Close() + fill(f) + buf, err := f.WriteToBuffer() + if err != nil { + t.Fatalf("WriteToBuffer: %v", err) + } + return buf.Bytes() +} + +// The following helpers fail the test immediately when an Excelize fixture +// operation errors, so incomplete workbook data is never handed to the parser. +func mustSetCell(t *testing.T, f *excelize.File, sheet, axis string, val any) { + t.Helper() + if err := f.SetCellValue(sheet, axis, val); err != nil { + t.Fatalf("SetCellValue(%s!%s): %v", sheet, axis, err) + } +} + +func mustMergeCell(t *testing.T, f *excelize.File, sheet, topLeft, bottomRight string) { + t.Helper() + if err := f.MergeCell(sheet, topLeft, bottomRight); err != nil { + t.Fatalf("MergeCell(%s:%s-%s): %v", sheet, topLeft, bottomRight, err) + } +} + +func mustNewStyle(t *testing.T, f *excelize.File, style *excelize.Style) int { + t.Helper() + idx, err := f.NewStyle(style) + if err != nil { + t.Fatalf("NewStyle: %v", err) + } + return idx +} + +func mustSetCellStyle(t *testing.T, f *excelize.File, sheet, topLeft, bottomRight string, idx int) { + t.Helper() + if err := f.SetCellStyle(sheet, topLeft, bottomRight, idx); err != nil { + t.Fatalf("SetCellStyle(%s:%s-%s): %v", sheet, topLeft, bottomRight, err) + } +} + +func mustAddTable(t *testing.T, f *excelize.File, sheet string, table *excelize.Table) { + t.Helper() + if err := f.AddTable(sheet, table); err != nil { + t.Fatalf("AddTable(%s): %v", sheet, err) + } +} + +// TestRecordsToHTMLTableChunks_Alignment asserts the chunked output uses the +// shared schema: / wrapper. +func TestRecordsToHTMLTableChunks_Alignment(t *testing.T) { + records := [][]string{{"Name", "Age"}, {"Alice", "30"}, {"Bob", "25"}} + out := recordsToHTMLTableChunks(records, 256, "Sheet1") + + if !strings.Contains(out, ``) { + t.Fatalf("want , got:\n%s", out) + } + if !strings.Contains(out, "") { + t.Fatalf("want header row as ") { + t.Fatalf("want data row as ") || strings.Contains(out, "") { + t.Fatalf("must not emit / to stay byte-compatible with Python/CSV, got:\n%s", out) + } + // Exactly one
, first row as
, data as , repeated header +// per 256-row chunk, and NO
Sheet1Sheet1
NameAge
, got:\n%s", out) + } + if !strings.Contains(out, "
Alice30
, got:\n%s", out) + } + if strings.Contains(out, "
for <=256 data rows. + if n := strings.Count(out, "
"); n != 1 { + t.Fatalf("want 1
, got %d:\n%s", n, out) + } +} + +// TestRecordsToHTMLTableChunks_Chunking asserts 256-row chunking with a repeated +// header (ceil(n_data / chunk_rows) chunks). +func TestRecordsToHTMLTableChunks_Chunking(t *testing.T) { + const dataRows = 300 + records := make([][]string, 0, dataRows+1) + records = append(records, []string{"C1", "C2"}) + for i := 0; i < dataRows; i++ { + records = append(records, []string{"x", "y"}) + } + out := recordsToHTMLTableChunks(records, 256, "S") + // 300 data rows → ceil(300/256) = 2 chunks, each repeating the header. + if n := strings.Count(out, "
"); n != 2 { + t.Fatalf("want 2
chunks, got %d", n) + } + if n := strings.Count(out, ""); n != 2 { + t.Fatalf("want header repeated in both chunks, got %d", n) + } +} + +// TestXLSXParser_HeaderAndCaption asserts the XLSX parser emits a `) { + t.Fatalf("want , got:\n%s", html) + } + if !strings.Contains(html, "") { + t.Fatalf("want header as ") { + t.Fatalf("header value leaked into ") { + t.Fatalf("merged master text not inherited into header ") { + t.Fatalf("want ListObject header row detected, got:\n%s", res.HTML) + } + if strings.Contains(res.HTML, "") { + t.Fatalf("title row must not be the header:\n%s", res.HTML) + } +} + +// TestDetectHeaderRow_Lightweight asserts that when row 1 is numeric data and +// row 2 is a styled text label row, the lightweight detector picks row 2. +func TestDetectHeaderRow_Lightweight(t *testing.T) { + data := newTestXLSX(t, func(f *excelize.File) { + mustSetCell(t, f, "Sheet1", "A1", "100") + mustSetCell(t, f, "Sheet1", "B1", "200") + mustSetCell(t, f, "Sheet1", "A2", "Item") + mustSetCell(t, f, "Sheet1", "B2", "Count") + // Make the header row bold so the styled signal fires. + idx := mustNewStyle(t, f, &excelize.Style{Font: &excelize.Font{Bold: true}}) + mustSetCellStyle(t, f, "Sheet1", "A2", "B2", idx) + mustSetCell(t, f, "Sheet1", "A3", "Apple") + mustSetCell(t, f, "Sheet1", "B3", "5") + }) + p, _ := NewXLSXParser("") + res := p.ParseWithResult(t.Context(), "t.xlsx", data) + if res.Err != nil { + t.Fatalf("ParseWithResult: %v", res.Err) + } + if !strings.Contains(res.HTML, "") { + t.Fatalf("want row-2 header detected, got:\n%s", res.HTML) + } + if strings.Contains(res.HTML, "") { + t.Fatalf("numeric row 1 must not be the header:\n%s", res.HTML) + } +} + +// TestXLSXParser_CommonCaseNoRegression asserts the dominant case (header on +// row 1, no merges, no ListObject) renders row 1 as ") { + t.Fatalf("common-case header must render as ") { + t.Fatalf("numeric row 1 must remain the header:\n%s", res.HTML) + } + if strings.Contains(res.HTML, "") { + t.Fatalf("bold subtotal row must not be promoted to header:\n%s", res.HTML) + } +} + +// TestDetectHeaderRow_StyledHeaderPastFarMerge asserts that a bold header on a +// narrow row (row 3) is still detected even when an earlier wide merge +// (A1:Z1 title) would otherwise pad every row to 26 columns. The padding step +// must run AFTER header detection so the styled-majority signal is not diluted. +func TestDetectHeaderRow_StyledHeaderPastFarMerge(t *testing.T) { + data := newTestXLSX(t, func(f *excelize.File) { + // Row 1: a wide merged title cell A1:Z1. + mustSetCell(t, f, "Sheet1", "A1", "Sales Report") + mustMergeCell(t, f, "Sheet1", "A1", "Z1") + titleIdx := mustNewStyle(t, f, &excelize.Style{Font: &excelize.Font{Bold: true}}) + mustSetCellStyle(t, f, "Sheet1", "A1", "A1", titleIdx) + // Row 3: the real, narrow, bold header. + hdrIdx := mustNewStyle(t, f, &excelize.Style{Font: &excelize.Font{Bold: true}}) + mustSetCell(t, f, "Sheet1", "A3", "Name") + mustSetCell(t, f, "Sheet1", "B3", "Desc") + mustSetCellStyle(t, f, "Sheet1", "A3", "B3", hdrIdx) + // Row 4: data (text in col A, so the body-brake passes for row 3). + mustSetCell(t, f, "Sheet1", "A4", "Alice") + mustSetCell(t, f, "Sheet1", "B4", "x") + }) + p, _ := NewXLSXParser("") + res := p.ParseWithResult(t.Context(), "t.xlsx", data) + if res.Err != nil { + t.Fatalf("ParseWithResult: %v", res.Err) + } + if !strings.Contains(res.HTML, "") { + t.Fatalf("narrow bold header past far merge must be detected:\n%s", res.HTML) + } + if strings.Contains(res.HTML, "") { + t.Fatalf("wide merged title must not be the header:\n%s", res.HTML) + } +} + +// TestDetectHeaderRow_StyledTextHeaderOverNumeric asserts that a styled text +// header sitting directly above a purely-numeric data row is still detected as +// the header (not refused by the body-brake), while a bold "Total"/"Summary" +// subtotal over numeric data is not. +func TestDetectHeaderRow_StyledTextHeaderOverNumeric(t *testing.T) { + data := newTestXLSX(t, func(f *excelize.File) { + // Row 1: a single-cell title stub (fewer than 2 columns → skipped). + mustSetCell(t, f, "Sheet1", "A1", "Report Title") + // Row 2: the real, bold text header over numeric-only data below. + idx := mustNewStyle(t, f, &excelize.Style{Font: &excelize.Font{Bold: true}}) + mustSetCell(t, f, "Sheet1", "A2", "Product") + mustSetCell(t, f, "Sheet1", "B2", "Units") + mustSetCellStyle(t, f, "Sheet1", "A2", "B2", idx) + // Row 3: purely numeric continuation. + mustSetCell(t, f, "Sheet1", "A3", "5") + mustSetCell(t, f, "Sheet1", "B3", "10") + }) + p, _ := NewXLSXParser("") + res := p.ParseWithResult(t.Context(), "t.xlsx", data) + if res.Err != nil { + t.Fatalf("ParseWithResult: %v", res.Err) + } + if !strings.Contains(res.HTML, "") { + t.Fatalf("styled text header over numeric data must be detected:\n%s", res.HTML) + } + if strings.Contains(res.HTML, "") { + t.Fatalf("title stub must not be the header:\n%s", res.HTML) + } +} + +// TestInheritMergedHeader unit-tests the merge inheritance helper directly. +func TestInheritMergedHeader(t *testing.T) { + records := [][]string{ + {"MASTER", "", "Q2"}, // header row: B1 blank, merged from master A1 + {"a", "b", "c"}, + } + mm := map[[2]int][2]int{ + {1, 1}: {1, 1}, + {1, 2}: {1, 1}, // B1's master is A1 + {1, 3}: {1, 3}, + } + inheritMergedHeader(records, 1, mm) + if records[0][1] != "MASTER" { + t.Fatalf("expected B1 to inherit A1 master text, got %q", records[0][1]) + } + if records[0][2] != "Q2" { + t.Fatalf("expected non-merged C1 to keep its value, got %q", records[0][2]) + } +} + +// TestPadRowToWidth asserts the single-row padder widens with empty strings, +// preserves existing cells, and never shrinks an already-wide row. +func TestPadRowToWidth(t *testing.T) { + row := []string{"a", "b"} + padRowToWidth(&row, 5) + if len(row) != 5 { + t.Fatalf("want len 5, got %d", len(row)) + } + if row[0] != "a" || row[1] != "b" { + t.Fatalf("original cells lost: %v", row) + } + for i := 2; i < 5; i++ { + if row[i] != "" { + t.Fatalf("pad cell %d want empty, got %q", i, row[i]) + } + } + // Never shrinks an already-wide row. + padRowToWidth(&row, 2) + if len(row) != 5 { + t.Fatalf("must not shrink: want 5, got %d", len(row)) + } +} + +// TestMergeExtentCol asserts the furthest merged column is reported as-is within +// the cap and clamped to maxMergeExtentCols beyond it (the memory guard). +func TestMergeExtentCol(t *testing.T) { + if got := mergeExtentCol(nil); got != 0 { + t.Fatalf("nil ranges: want 0, got %d", got) + } + if got := mergeExtentCol([]mergeRange{{1, 1, 3, 10}}); got != 10 { + t.Fatalf("within cap: want 10, got %d", got) + } + if got := mergeExtentCol([]mergeRange{{1, 1, 1, 5000}}); got != maxMergeExtentCols { + t.Fatalf("beyond cap: want %d, got %d", maxMergeExtentCols, got) + } +} + +// TestDecodeChunkRows exercises the chunk_rows setup knob across all the types +// JSON/yaml decoding can produce, plus the defaulting paths. +func TestDecodeChunkRows(t *testing.T) { + if got := decodeChunkRows(nil); got != defaultTableChunkRows { + t.Fatalf("nil setup: want default %d, got %d", defaultTableChunkRows, got) + } + if got := decodeChunkRows(map[string]any{}); got != defaultTableChunkRows { + t.Fatalf("empty setup: want default, got %d", got) + } + if got := decodeChunkRows(map[string]any{"chunk_rows": 100.0}); got != 100 { + t.Fatalf("float64 100: want 100, got %d", got) + } + if got := decodeChunkRows(map[string]any{"chunk_rows": 0.0}); got != defaultTableChunkRows { + t.Fatalf("float64 0: want default, got %d", got) + } + if got := decodeChunkRows(map[string]any{"chunk_rows": -5.0}); got != defaultTableChunkRows { + t.Fatalf("float64 -5: want default, got %d", got) + } + if got := decodeChunkRows(map[string]any{"chunk_rows": 256}); got != 256 { + t.Fatalf("int 256: want 256, got %d", got) + } + if got := decodeChunkRows(map[string]any{"chunk_rows": int64(512)}); got != 512 { + t.Fatalf("int64 512: want 512, got %d", got) + } + if got := decodeChunkRows(map[string]any{"chunk_rows": "256"}); got != defaultTableChunkRows { + t.Fatalf("string (unsupported type): want default, got %d", got) + } +} + +// TestRenderSheetTables_FarMergeDoesNotBloatDataRows guards the memory blow-up +// from review: a far merge on a non-header row must not widen the header or any +// data row to the merge's full span. The header stays its natural width and no +// data row carries empty ") { + t.Fatalf("want row-2 header detected, got:\n%s", html) + } + if strings.Contains(html, "") { + t.Fatalf("wide merged title must not be the header:\n%s", html) + } + // The far merge is on row 1, not the header, so nothing is widened to 26 + // columns: the header keeps 2 columns (no empty ") { + t.Fatalf("header was bloated by the far merge:\n%s", html) + } + if strings.Count(html, "") != 0 { + t.Fatalf("data rows were bloated by the far merge:\n%s", html) + } +} + +// TestRenderSheetTables_MultiSheet asserts each sheet becomes its own
C1C2
 and +// renders the first row as
, and that the header text appears only in . +func TestXLSXParser_HeaderAndCaption(t *testing.T) { + data := newTestXLSX(t, func(f *excelize.File) { + mustSetCell(t, f, "Sheet1", "A1", "Product") + mustSetCell(t, f, "Sheet1", "B1", "Price") + mustSetCell(t, f, "Sheet1", "A2", "Widget") + mustSetCell(t, f, "Sheet1", "B2", "9.99") + mustSetCell(t, f, "Sheet1", "A3", "Gadget") + mustSetCell(t, f, "Sheet1", "B3", "19.99") + }) + p, _ := NewXLSXParser("") + res := p.ParseWithResult(t.Context(), "t.xlsx", data) + if res.Err != nil { + t.Fatalf("ParseWithResult: %v", res.Err) + } + html := res.HTML + if !strings.Contains(html, `
Sheet1Sheet1
ProductPrice
, got:\n%s", html) + } + // "Product" must only appear inside a , never inside a . + if strings.Contains(html, "Product:\n%s", html) + } +} + +// TestXLSXParser_MergedHeaderInheritance asserts a horizontally merged header +// cell's slave columns inherit the master text, so a wide merged title does not +// render as a row of blank cells. +func TestXLSXParser_MergedHeaderInheritance(t *testing.T) { + data := newTestXLSX(t, func(f *excelize.File) { + // Row 1 is the header; A1:C1 merged into one wide label "Sales Report". + mustSetCell(t, f, "Sheet1", "A1", "Sales Report") + mustMergeCell(t, f, "Sheet1", "A1", "C1") + // Make the header row bold so detection anchors row 1. + idx := mustNewStyle(t, f, &excelize.Style{Font: &excelize.Font{Bold: true}}) + mustSetCellStyle(t, f, "Sheet1", "A1", "C1", idx) + mustSetCell(t, f, "Sheet1", "A2", "North") + mustSetCell(t, f, "Sheet1", "B2", "10") + mustSetCell(t, f, "Sheet1", "C2", "20") + mustSetCell(t, f, "Sheet1", "A3", "South") + mustSetCell(t, f, "Sheet1", "B3", "30") + mustSetCell(t, f, "Sheet1", "C3", "40") + }) + p, _ := NewXLSXParser("") + res := p.ParseWithResult(t.Context(), "t.xlsx", data) + if res.Err != nil { + t.Fatalf("ParseWithResult: %v", res.Err) + } + html := res.HTML + // The merged master text must have propagated into all three slots. + if !strings.Contains(html, "
Sales ReportSales ReportSales Report
, got:\n%s", html) + } +} + +// TestDetectHeaderRow_ListObject asserts a ListObject whose top row is > 1 is +// detected as the header row. +func TestDetectHeaderRow_ListObject(t *testing.T) { + data := newTestXLSX(t, func(f *excelize.File) { + mustSetCell(t, f, "Sheet1", "A1", "Title") + mustSetCell(t, f, "Sheet1", "A2", "This is a banner") + mustSetCell(t, f, "Sheet1", "A3", "Name") + mustSetCell(t, f, "Sheet1", "B3", "Age") + mustSetCell(t, f, "Sheet1", "A4", "Alice") + mustSetCell(t, f, "Sheet1", "B4", "30") + mustAddTable(t, f, "Sheet1", &excelize.Table{Range: "A3:B4", Name: "Table1"}) + }) + p, _ := NewXLSXParser("") + res := p.ParseWithResult(t.Context(), "t.xlsx", data) + if res.Err != nil { + t.Fatalf("ParseWithResult: %v", res.Err) + } + if !strings.Contains(res.HTML, "
NameAge
Title
ItemCount
100 unchanged. +func TestXLSXParser_CommonCaseNoRegression(t *testing.T) { + data := newTestXLSX(t, func(f *excelize.File) { + mustSetCell(t, f, "Sheet1", "A1", "col_a") + mustSetCell(t, f, "Sheet1", "B1", "col_b") + mustSetCell(t, f, "Sheet1", "A2", "1") + mustSetCell(t, f, "Sheet1", "B2", "2") + }) + p, _ := NewXLSXParser("") + res := p.ParseWithResult(t.Context(), "t.xlsx", data) + if res.Err != nil { + t.Fatalf("ParseWithResult: %v", res.Err) + } + if !strings.Contains(res.HTML, "
col_acol_b
:\n%s", res.HTML) + } +} + +// TestDetectHeaderRow_BoldSubtotalNotHeader asserts that a bold "Total"-style +// subtotal row sitting directly under a numeric row 1 is NOT promoted to the +// header. The subtotal-label check blocks the override so the numeric row 1 +// stays the header. +func TestDetectHeaderRow_BoldSubtotalNotHeader(t *testing.T) { + data := newTestXLSX(t, func(f *excelize.File) { + // Row 1: numeric year header (Python default header row). + mustSetCell(t, f, "Sheet1", "A1", "2023") + mustSetCell(t, f, "Sheet1", "B1", "2024") + // Row 2: bold "Total/Summary" subtotal — must NOT become the header. + idx := mustNewStyle(t, f, &excelize.Style{Font: &excelize.Font{Bold: true}}) + mustSetCell(t, f, "Sheet1", "A2", "Total") + mustSetCell(t, f, "Sheet1", "B2", "Summary") + mustSetCellStyle(t, f, "Sheet1", "A2", "B2", idx) + // Row 3: purely numeric continuation under the subtotal. + mustSetCell(t, f, "Sheet1", "A3", "100") + mustSetCell(t, f, "Sheet1", "B3", "200") + }) + p, _ := NewXLSXParser("") + res := p.ParseWithResult(t.Context(), "t.xlsx", data) + if res.Err != nil { + t.Fatalf("ParseWithResult: %v", res.Err) + } + if !strings.Contains(res.HTML, "
20232024
TotalNameDescSales Report
ProductUnits
Report Title cells. +func TestRenderSheetTables_FarMergeDoesNotBloatDataRows(t *testing.T) { + data := newTestXLSX(t, func(f *excelize.File) { + // Row 1: a wide merged title A1:Z1 (NOT the header). + mustSetCell(t, f, "Sheet1", "A1", "Report Title") + mustMergeCell(t, f, "Sheet1", "A1", "Z1") + // Row 2: the real header, made bold so detection anchors it. + hdrIdx := mustNewStyle(t, f, &excelize.Style{Font: &excelize.Font{Bold: true}}) + mustSetCell(t, f, "Sheet1", "A2", "Name") + mustSetCell(t, f, "Sheet1", "B2", "Price") + mustSetCellStyle(t, f, "Sheet1", "A2", "B2", hdrIdx) + // Rows 3-4: data. + mustSetCell(t, f, "Sheet1", "A3", "Alice") + mustSetCell(t, f, "Sheet1", "B3", "9.99") + mustSetCell(t, f, "Sheet1", "A4", "Bob") + mustSetCell(t, f, "Sheet1", "B4", "19.99") + }) + p, _ := NewXLSXParser("") + res := p.ParseWithResult(t.Context(), "t.xlsx", data) + if res.Err != nil { + t.Fatalf("ParseWithResult: %v", res.Err) + } + html := res.HTML + if !strings.Contains(html, "
NamePrice
Title) and data rows carry + // no empty . + if strings.Contains(html, "
+// chunk, each carrying its own ") { + t.Fatalf("want Sheet1 caption, got:\n%s", html) + } + if !strings.Contains(html, "") { + t.Fatalf("want HR caption, got:\n%s", html) + } + // Two small sheets → two
. +func TestRenderSheetTables_MultiSheet(t *testing.T) { + data := newTestXLSX(t, func(f *excelize.File) { + mustSetCell(t, f, "Sheet1", "A1", "Name") + mustSetCell(t, f, "Sheet1", "B1", "Age") + mustSetCell(t, f, "Sheet1", "A2", "Alice") + mustSetCell(t, f, "Sheet1", "B2", "30") + if _, err := f.NewSheet("HR"); err != nil { + t.Fatalf("NewSheet: %v", err) + } + mustSetCell(t, f, "HR", "A1", "Dept") + mustSetCell(t, f, "HR", "B1", "Head") + mustSetCell(t, f, "HR", "A2", "Eng") + mustSetCell(t, f, "HR", "B2", "Bob") + }) + p, _ := NewXLSXParser("") + res := p.ParseWithResult(t.Context(), "t.xlsx", data) + if res.Err != nil { + t.Fatalf("ParseWithResult: %v", res.Err) + } + html := res.HTML + if !strings.Contains(html, "Sheet1HR
chunks. + if n := strings.Count(html, "
"); n != 2 { + t.Fatalf("want 2
chunks, got %d:\n%s", n, html) + } +} + +// TestRenderSheetTables_EmptySheet asserts an empty/unreadable sheet yields an +// empty string rather than an empty
wrapper. +func TestRenderSheetTables_EmptySheet(t *testing.T) { + // excelize.NewFile yields a single empty "Sheet1". + data := newTestXLSX(t, func(f *excelize.File) {}) + p, _ := NewXLSXParser("") + res := p.ParseWithResult(t.Context(), "t.xlsx", data) + if res.Err != nil { + t.Fatalf("ParseWithResult: %v", res.Err) + } + if res.HTML != "" { + t.Fatalf("empty sheet must yield empty HTML, got:\n%s", res.HTML) + } +} diff --git a/internal/parser/parser/xls_parser.go b/internal/parser/parser/xls_parser.go index 216d85c038..02a34f9bfa 100644 --- a/internal/parser/parser/xls_parser.go +++ b/internal/parser/parser/xls_parser.go @@ -29,6 +29,7 @@ type XLSParser struct { libType string ParseMethod string OutputFormat string + ChunkRows int TCADPAPIServer string TCADPAPIKey string TCADPTableResultType string @@ -41,6 +42,7 @@ func NewXLSParser(libType string) (*XLSParser, error) { } return &XLSParser{ libType: libType, + ChunkRows: defaultTableChunkRows, TCADPTableResultType: "1", TCADPMarkdownImageResponseType: "1", }, nil @@ -60,6 +62,7 @@ func (p *XLSParser) ConfigureFromSetup(setup map[string]any) { if v, ok := setup["output_format"].(string); ok && v != "" { p.OutputFormat = v } + p.ChunkRows = decodeChunkRows(setup) if v, ok := setup["tcadp_apiserver"].(string); ok && v != "" { p.TCADPAPIServer = v } @@ -98,26 +101,16 @@ func (p *XLSParser) ParseWithResult(ctx context.Context, filename string, data [ } defer f.Close() - var html strings.Builder - html.WriteString("") - for _, sheet := range f.GetSheetList() { - html.WriteString("

") - html.WriteString(sheet) - html.WriteString("

") - rows, _ := f.GetRows(sheet) - html.WriteString("
") - for _, row := range rows { - html.WriteString("") - for _, cell := range row { - html.WriteString("") - } - html.WriteString("") - } - html.WriteString("
") - html.WriteString(htmlEscape(cell)) - html.WriteString("
") + sheets := f.GetSheetList() + chunkRows := p.ChunkRows + if chunkRows <= 0 { + chunkRows = defaultTableChunkRows + } + + var html strings.Builder + for _, sheet := range sheets { + html.WriteString(renderSheetTables(f, sheet, chunkRows)) } - html.WriteString("") return ParseResult{ OutputFormat: "html", diff --git a/internal/parser/parser/xlsx_parser.go b/internal/parser/parser/xlsx_parser.go index 5d2478d600..02f698941c 100644 --- a/internal/parser/parser/xlsx_parser.go +++ b/internal/parser/parser/xlsx_parser.go @@ -29,6 +29,7 @@ type XLSXParser struct { libType string ParseMethod string OutputFormat string + ChunkRows int TCADPAPIServer string TCADPAPIKey string TCADPTableResultType string @@ -41,6 +42,7 @@ func NewXLSXParser(libType string) (*XLSXParser, error) { } return &XLSXParser{ libType: libType, + ChunkRows: defaultTableChunkRows, TCADPTableResultType: "1", TCADPMarkdownImageResponseType: "1", }, nil @@ -60,6 +62,7 @@ func (p *XLSXParser) ConfigureFromSetup(setup map[string]any) { if v, ok := setup["output_format"].(string); ok && v != "" { p.OutputFormat = v } + p.ChunkRows = decodeChunkRows(setup) if v, ok := setup["tcadp_apiserver"].(string); ok && v != "" { p.TCADPAPIServer = v } @@ -117,29 +120,15 @@ func (p *XLSXParser) ParseWithResult(ctx context.Context, filename string, data defer f.Close() sheets := f.GetSheetList() - var html strings.Builder - html.WriteString("") - for _, sheet := range sheets { - html.WriteString("

") - html.WriteString(sheet) - html.WriteString("

") - rows, err := f.GetRows(sheet) - if err != nil { - continue - } - html.WriteString("") - for _, row := range rows { - html.WriteString("") - for _, cell := range row { - html.WriteString("") - } - html.WriteString("") - } - html.WriteString("
") - html.WriteString(htmlEscape(cell)) - html.WriteString("
") + chunkRows := p.ChunkRows + if chunkRows <= 0 { + chunkRows = defaultTableChunkRows + } + + var html strings.Builder + for _, sheet := range sheets { + html.WriteString(renderSheetTables(f, sheet, chunkRows)) } - html.WriteString("") return ParseResult{ OutputFormat: "html",