skills/anomalyco__opentui
1
0
Fork 0
mirror of https://github.com/anomalyco/opentui.git synced 2026-09-19 01:26:03 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor(core): clarify text layout units (#1429)

Browse Source
This commit is contained in:
Sebastian
2026-08-25 22:52:07 +02:00
committed by GitHub
parent d2089a81ae
commit 465c3ccad0
21 changed files with 762 additions and 763 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/native/src/bench.zig
+2 -2
View File
@@ -57,7 +57,7 @@ const buffer_draw_text_buffer_bench = @import("bench/buffer-draw-text-buffer_ben
const buffer_color_blending_bench = @import("bench/buffer-color-blending_bench.zig");
const buffer_draw_box_bench = @import("bench/buffer-draw-box_bench.zig");
const utf8_bench = @import("bench/utf8_bench.zig");
const text_chunk_graphemes_bench = @import("bench/text-chunk-graphemes_bench.zig");
const text_chunk_render_clusters_bench = @import("bench/text-chunk-render-clusters_bench.zig");
const editor_view_bench = @import("bench/editor-view_bench.zig");
const terminal_image_bench = @import("bench/terminal-image_bench.zig");
const renderer_image_bench = @import("bench/renderer-image_bench.zig");
@@ -116,7 +116,7 @@ pub fn main(init: std.process.Init) !void {
.{ .name = buffer_color_blending_bench.benchName, .run = buffer_color_blending_bench.run },
.{ .name = buffer_draw_box_bench.benchName, .run = buffer_draw_box_bench.run },
.{ .name = utf8_bench.benchName, .run = utf8_bench.run },
.{ .name = text_chunk_graphemes_bench.benchName, .run = text_chunk_graphemes_bench.run },
.{ .name = text_chunk_render_clusters_bench.benchName, .run = text_chunk_render_clusters_bench.run },
.{ .name = editor_view_bench.benchName, .run = editor_view_bench.run },
.{ .name = terminal_image_bench.benchName, .run = terminal_image_bench.run },
.{ .name = renderer_image_bench.benchName, .run = renderer_image_bench.run },
packages/native/src/bench/text-buffer-coords_bench.zig
+1 -1
View File
@@ -23,7 +23,7 @@ fn createTestBuffer(allocator: std.mem.Allocator, line_count: u32, chars_per_lin
.mem_id = 0,
.byte_start = 0,
.byte_end = chars_per_line,
.width = @intCast(chars_per_line),
.width_cols = @intCast(chars_per_line),
.flags = TextChunk.Flags.ASCII_ONLY,
},
});
packages/native/src/bench/text-chunk-graphemes_bench.zig → packages/native/src/bench/text-chunk-render-clusters_bench.zig
+17 -17
View File
@@ -10,7 +10,7 @@ const BenchResult = bench_utils.BenchResult;
const BenchStats = bench_utils.BenchStats;
const MemStat = bench_utils.MemStat;
pub const benchName = "TextChunk getGraphemes";
pub const benchName = "TextChunk getRenderClusters";
const TextType = enum { ascii, mixed, heavy_unicode };
@@ -76,7 +76,7 @@ fn generateTestText(allocator: std.mem.Allocator, size: usize, text_type: TextTy
return buffer.toOwnedSlice(allocator);
}
fn benchGetGraphemes(
fn benchGetRenderClusters(
io: std.Io,
allocator: std.mem.Allocator,
size: usize,
@@ -104,7 +104,7 @@ fn benchGetGraphemes(
// Width is approximate - clamped to u16 max
const approx_width: u32 = @intCast(@min(text.len, std.math.maxInt(u32)));
var stats: BenchStats = .{};
var grapheme_count: usize = 0;
var render_cluster_count: usize = 0;
var final_mem: usize = 0;
for (0..iterations) |i| {
@@ -117,12 +117,12 @@ fn benchGetGraphemes(
.mem_id = mem_id,
.byte_start = 0,
.byte_end = @intCast(text.len),
.width = approx_width,
.width_cols = approx_width,
.flags = if (is_ascii) TextChunk.Flags.ASCII_ONLY else 0,
};
const timer = bench_utils.BenchTimer.start(io);
const graphemes = try chunk.getGraphemes(
const render_clusters = try chunk.getRenderClusters(
arena_alloc,
&registry,
4, // tab width
@@ -131,12 +131,12 @@ fn benchGetGraphemes(
stats.record(timer.read());
if (i == 0) {
grapheme_count = graphemes.len;
render_cluster_count = render_clusters.len;
}
if (i == iterations - 1 and show_mem) {
// Estimate memory used for grapheme storage
final_mem = graphemes.len * @sizeOf(seg_mod.GraphemeInfo);
// Estimate memory used for sparse render-cluster metadata.
final_mem = render_clusters.len * @sizeOf(seg_mod.RenderClusterInfo);
}
}
@@ -148,13 +148,13 @@ fn benchGetGraphemes(
const name = try std.fmt.allocPrint(
allocator,
"getGraphemes {s} ({d} bytes, {d} graphemes)",
.{ type_str, size, grapheme_count },
"getRenderClusters {s} ({d} bytes, {d} render clusters)",
.{ type_str, size, render_cluster_count },
);
const mem_stats: ?[]const MemStat = if (show_mem) blk: {
const mem_stat_slice = try allocator.alloc(MemStat, 1);
mem_stat_slice[0] = .{ .name = "Graphemes", .bytes = final_mem };
mem_stat_slice[0] = .{ .name = "Render clusters", .bytes = final_mem };
break :blk mem_stat_slice;
} else null;
@@ -189,11 +189,11 @@ fn computeBenchName(allocator: std.mem.Allocator, size: usize, text_type: TextTy
.mem_id = mem_id,
.byte_start = 0,
.byte_end = @intCast(text.len),
.width = approx_width,
.width_cols = approx_width,
.flags = if (is_ascii) TextChunk.Flags.ASCII_ONLY else 0,
};
const graphemes = try chunk.getGraphemes(
const render_clusters = try chunk.getRenderClusters(
temp_alloc,
&registry,
4, // tab width
@@ -208,8 +208,8 @@ fn computeBenchName(allocator: std.mem.Allocator, size: usize, text_type: TextTy
return std.fmt.allocPrint(
allocator,
"getGraphemes {s} ({d} bytes, {d} graphemes)",
.{ type_str, size, graphemes.len },
"getRenderClusters {s} ({d} bytes, {d} render clusters)",
.{ type_str, size, render_clusters.len },
);
}
@@ -234,7 +234,7 @@ pub fn run(
if (bench_filter == null) {
for (text_types) |text_type| {
for (sizes) |size| {
const result = try benchGetGraphemes(
const result = try benchGetRenderClusters(
io,
allocator,
size,
@@ -254,7 +254,7 @@ pub fn run(
continue;
}
var result = try benchGetGraphemes(
var result = try benchGetRenderClusters(
io,
allocator,
size,
packages/native/src/buffer.zig
+104 -104
View File
@@ -1295,12 +1295,12 @@ pub const OptimizedBuffer = struct {
}
}
var grapheme_list: std.ArrayListUnmanaged(utf8.GraphemeInfo) = .empty;
defer grapheme_list.deinit(self.allocator);
var render_cluster_list: std.ArrayListUnmanaged(utf8.RenderClusterInfo) = .empty;
defer render_cluster_list.deinit(self.allocator);
const tab_width: u8 = 2;
try utf8.findGraphemeInfo(self.allocator, text, tab_width, is_ascii_only, self.width_method, &grapheme_list);
const specials = grapheme_list.items;
try utf8.findRenderClusterInfo(self.allocator, text, tab_width, is_ascii_only, self.width_method, &render_cluster_list);
const render_clusters = render_cluster_list.items;
var advance_cells: u32 = 0;
var byte_offset: u32 = 0;
@@ -1311,28 +1311,28 @@ pub const OptimizedBuffer = struct {
const charX = x + advance_cells;
if (charX >= self.width) break;
const at_special = special_idx < specials.len and specials[special_idx].col_offset == col;
const at_special = special_idx < render_clusters.len and render_clusters[special_idx].col_start == col;
var grapheme_bytes: []const u8 = undefined;
var g_width: u32 = undefined;
var cluster_width_cols: u32 = undefined;
if (at_special) {
const g = specials[special_idx];
grapheme_bytes = text[g.byte_offset .. g.byte_offset + g.byte_len];
g_width = g.width;
byte_offset = g.byte_offset + g.byte_len;
const g = render_clusters[special_idx];
grapheme_bytes = text[g.byte_start .. g.byte_start + g.byte_len];
cluster_width_cols = g.width_cols;
byte_offset = g.byte_start + g.byte_len;
special_idx += 1;
} else {
if (byte_offset >= text.len) break;
grapheme_bytes = text[byte_offset .. byte_offset + 1];
g_width = 1;
cluster_width_cols = 1;
byte_offset += 1;
}
const is_tab = grapheme_bytes.len == 1 and grapheme_bytes[0] == '\t';
if (!is_tab and !self.isPointInScissor(@intCast(charX), @intCast(y))) {
advance_cells += g_width;
col += g_width;
advance_cells += cluster_width_cols;
col += cluster_width_cols;
continue;
}
@@ -1345,21 +1345,21 @@ pub const OptimizedBuffer = struct {
bgColor = ansi.rgbColor(0, 0, 0, 255);
}
const cell_width = utf8.getWidthAt(text, if (at_special) specials[special_idx - 1].byte_offset else byte_offset - 1, tab_width, self.width_method);
const cell_width = utf8.getWidthAt(text, if (at_special) render_clusters[special_idx - 1].byte_start else byte_offset - 1, tab_width, self.width_method);
if (cell_width == 0) {
col += g_width;
col += cluster_width_cols;
continue;
}
if (cell_width > 1 and !is_tab) {
if (charX + cell_width > self.width) {
advance_cells += g_width;
col += g_width;
advance_cells += cluster_width_cols;
col += cluster_width_cols;
continue;
}
for (1..cell_width) |span_offset| {
if (!self.isPointInScissor(@intCast(charX + @as(u32, @intCast(span_offset))), @intCast(y))) {
advance_cells += g_width;
col += g_width;
advance_cells += cluster_width_cols;
col += cluster_width_cols;
continue :text_loop;
}
}
@@ -1367,7 +1367,7 @@ pub const OptimizedBuffer = struct {
if (is_tab) {
var tab_col: u32 = 0;
while (tab_col < g_width) : (tab_col += 1) {
while (tab_col < cluster_width_cols) : (tab_col += 1) {
const tab_x = charX + tab_col;
if (tab_x >= self.width) break;
if (!self.isPointInScissor(@intCast(tab_x), @intCast(y))) continue;
@@ -1375,8 +1375,8 @@ pub const OptimizedBuffer = struct {
const cell = makeCell(DEFAULT_SPACE_CHAR, fg, bgColor, attributes);
if (explicit_colors_opaque) self.set(tab_x, y, cell) else self.setTextCell(tab_x, y, cell);
}
advance_cells += g_width;
col += g_width;
advance_cells += cluster_width_cols;
col += cluster_width_cols;
continue;
}
@@ -1392,7 +1392,7 @@ pub const OptimizedBuffer = struct {
if (explicit_colors_opaque) self.set(charX, y, cell) else self.setTextCell(charX, y, cell);
advance_cells += cell_width;
col += g_width;
col += cluster_width_cols;
}
}
@@ -1675,7 +1675,7 @@ pub const OptimizedBuffer = struct {
const total_line_count = text_buffer.lineCount();
const line_info = view.getCachedLineInfo();
var globalCharPos: u32 = if (firstVisibleLine < line_info.line_start_cols.len)
var document_cell_offset: u32 = if (firstVisibleLine < line_info.line_start_cols.len)
line_info.line_start_cols[firstVisibleLine]
else
0;
@@ -1684,8 +1684,8 @@ pub const OptimizedBuffer = struct {
if (currentY >= bufferBottomY) break;
currentX = x;
var column_in_line: u32 = 0;
globalCharPos = vline.col_offset;
var rendered_col_in_vline: u32 = 0;
document_cell_offset = vline.document_cell_offset;
// When viewport is set, virtual_lines is a slice starting from viewport.y
// But getVirtualLineSpans expects absolute indices, so we need to use the absolute index
@@ -1694,7 +1694,7 @@ pub const OptimizedBuffer = struct {
const vline_idx = viewport_offset + firstVisibleLine + slice_idx;
const vline_span_info = view.getVirtualLineSpans(vline_idx);
const spans = vline_span_info.spans;
const col_offset = vline_span_info.col_offset;
const col_offset = vline_span_info.source_col_start;
var span_idx: usize = 0;
var lineFg = text_defaults.fg orelse ansi.rgbColor(255, 255, 255, 255);
var lineBg = text_defaults.bg orelse ansi.rgbColor(0, 0, 0, 0);
@@ -1732,11 +1732,11 @@ pub const OptimizedBuffer = struct {
for (vline.chunks.items) |vchunk| {
const chunk = vchunk.chunk;
const chunk_bytes = chunk.getBytes(text_buffer.memRegistry());
const specials = chunk.getGraphemes(text_buffer.getAllocator(), text_buffer.memRegistry(), text_buffer.tabWidth(), text_buffer.widthMethod()) catch continue;
const line_col_offset = vline.col_offset;
const render_clusters = chunk.getRenderClusters(text_buffer.getAllocator(), text_buffer.memRegistry(), text_buffer.tabWidth(), text_buffer.widthMethod()) catch continue;
const line_col_offset = vline.document_cell_offset;
if (currentX >= @as(i32, @intCast(self.width))) {
globalCharPos += vchunk.width_cols;
document_cell_offset += vchunk.width_cols;
currentX += @intCast(vchunk.width_cols);
continue;
}
@@ -1746,115 +1746,115 @@ pub const OptimizedBuffer = struct {
var byte_offset = vchunk.byte_start_in_chunk;
const byte_end = vchunk.byte_start_in_chunk + vchunk.byte_len;
while (special_idx < specials.len and specials[special_idx].byte_offset + specials[special_idx].byte_len <= byte_offset) {
while (special_idx < render_clusters.len and render_clusters[special_idx].byte_start + render_clusters[special_idx].byte_len <= byte_offset) {
special_idx += 1;
}
text_buffer_loop: while (byte_offset < byte_end and col < col_end) {
const at_special = special_idx < specials.len and specials[special_idx].byte_offset == byte_offset;
const at_special = special_idx < render_clusters.len and render_clusters[special_idx].byte_start == byte_offset;
var grapheme_bytes: []const u8 = undefined;
var g_width: u32 = undefined;
var cluster_width_cols: u32 = undefined;
if (at_special) {
const g = specials[special_idx];
if (g.byte_offset + g.byte_len > byte_end) break;
grapheme_bytes = chunk_bytes[g.byte_offset .. g.byte_offset + g.byte_len];
g_width = g.width;
byte_offset = g.byte_offset + g.byte_len;
const g = render_clusters[special_idx];
if (g.byte_start + g.byte_len > byte_end) break;
grapheme_bytes = chunk_bytes[g.byte_start .. g.byte_start + g.byte_len];
cluster_width_cols = g.width_cols;
byte_offset = g.byte_start + g.byte_len;
special_idx += 1;
} else {
if (byte_offset >= byte_end or byte_offset >= chunk_bytes.len) break;
const cp_len = std.unicode.utf8ByteSequenceLength(chunk_bytes[byte_offset]) catch 1;
const next_byte_offset = @min(byte_offset + cp_len, byte_end);
grapheme_bytes = chunk_bytes[byte_offset..next_byte_offset];
g_width = 1;
cluster_width_cols = 1;
byte_offset = next_byte_offset;
}
if (column_in_line < horizontal_offset) {
globalCharPos += g_width;
column_in_line += g_width;
col += g_width;
if (rendered_col_in_vline < horizontal_offset) {
document_cell_offset += cluster_width_cols;
rendered_col_in_vline += cluster_width_cols;
col += cluster_width_cols;
continue;
}
if (column_in_line >= horizontal_offset + viewport_width) {
globalCharPos += (col_end - col);
if (rendered_col_in_vline >= horizontal_offset + viewport_width) {
document_cell_offset += (col_end - col);
break;
}
// A glyph occupies columns [currentX, currentX + g_width).
// A glyph occupies columns [currentX, currentX + cluster_width_cols).
// If this range ends at or before column 0, the glyph is left of the screen.
// Skip the glyph, but advance the counters to put the next glyph in the correct columns.
// This check permits wide glyphs that cross column 0.
// The g_width > 1 check below discards these glyphs before they reach the unchecked fast-path index.
if (currentX + @as(i32, @intCast(g_width)) <= 0) {
globalCharPos += g_width;
currentX += @as(i32, @intCast(g_width));
column_in_line += g_width;
col += g_width;
// The cluster_width_cols > 1 check below discards these glyphs before they reach the unchecked fast-path index.
if (currentX + @as(i32, @intCast(cluster_width_cols)) <= 0) {
document_cell_offset += cluster_width_cols;
currentX += @as(i32, @intCast(cluster_width_cols));
rendered_col_in_vline += cluster_width_cols;
col += cluster_width_cols;
continue;
}
if (currentX >= @as(i32, @intCast(self.width))) {
globalCharPos += (col_end - col);
document_cell_offset += (col_end - col);
break;
}
const is_tab = grapheme_bytes.len == 1 and grapheme_bytes[0] == '\t';
if (!is_tab and !self.isPointInScissor(currentX, currentY)) {
globalCharPos += g_width;
currentX += @as(i32, @intCast(g_width));
column_in_line += g_width;
col += g_width;
document_cell_offset += cluster_width_cols;
currentX += @as(i32, @intCast(cluster_width_cols));
rendered_col_in_vline += cluster_width_cols;
col += cluster_width_cols;
continue;
}
if (g_width > 1 and !is_tab) {
if (column_in_line + g_width > horizontal_offset + viewport_width or
currentX < 0 or currentX + @as(i32, @intCast(g_width)) > @as(i32, @intCast(self.width)))
if (cluster_width_cols > 1 and !is_tab) {
if (rendered_col_in_vline + cluster_width_cols > horizontal_offset + viewport_width or
currentX < 0 or currentX + @as(i32, @intCast(cluster_width_cols)) > @as(i32, @intCast(self.width)))
{
globalCharPos += g_width;
currentX += @as(i32, @intCast(g_width));
column_in_line += g_width;
col += g_width;
document_cell_offset += cluster_width_cols;
currentX += @as(i32, @intCast(cluster_width_cols));
rendered_col_in_vline += cluster_width_cols;
col += cluster_width_cols;
continue;
}
for (1..g_width) |span_offset| {
for (1..cluster_width_cols) |span_offset| {
if (!self.isPointInScissor(currentX + @as(i32, @intCast(span_offset)), currentY)) {
globalCharPos += g_width;
currentX += @as(i32, @intCast(g_width));
column_in_line += g_width;
col += g_width;
document_cell_offset += cluster_width_cols;
currentX += @as(i32, @intCast(cluster_width_cols));
rendered_col_in_vline += cluster_width_cols;
col += cluster_width_cols;
continue :text_buffer_loop;
}
}
}
var selection_offset = globalCharPos;
if (vline.is_truncated and globalCharPos >= line_col_offset) {
var selection_offset = document_cell_offset;
if (vline.is_truncated and document_cell_offset >= line_col_offset) {
const ellipsis_width: u32 = 3;
const column_offset_in_line = globalCharPos - line_col_offset;
if (column_offset_in_line >= vline.ellipsis_pos and column_offset_in_line < vline.ellipsis_pos + ellipsis_width) {
selection_offset = line_col_offset + vline.ellipsis_pos;
} else if (column_offset_in_line >= vline.ellipsis_pos + ellipsis_width) {
selection_offset = line_col_offset + vline.truncation_suffix_start +
(column_offset_in_line - vline.ellipsis_pos - ellipsis_width);
const column_offset_in_line = document_cell_offset - line_col_offset;
if (column_offset_in_line >= vline.ellipsis_col and column_offset_in_line < vline.ellipsis_col + ellipsis_width) {
selection_offset = line_col_offset + vline.ellipsis_col;
} else if (column_offset_in_line >= vline.ellipsis_col + ellipsis_width) {
selection_offset = line_col_offset + vline.truncation_suffix_col_start +
(column_offset_in_line - vline.ellipsis_col - ellipsis_width);
} else {
selection_offset = line_col_offset + column_offset_in_line;
}
}
// Track the actual column position in the source line (including horizontal offset)
var source_col_pos = col_offset + column_in_line;
var source_col_pos = col_offset + rendered_col_in_vline;
if (vline.is_truncated) {
const ellipsis_width: u32 = 3;
const column_offset_in_line = globalCharPos - line_col_offset;
if (column_offset_in_line >= vline.ellipsis_pos and column_offset_in_line < vline.ellipsis_pos + ellipsis_width) {
const column_offset_in_line = document_cell_offset - line_col_offset;
if (column_offset_in_line >= vline.ellipsis_col and column_offset_in_line < vline.ellipsis_col + ellipsis_width) {
source_col_pos = std.math.maxInt(u32);
} else if (column_offset_in_line >= vline.ellipsis_pos + ellipsis_width) {
source_col_pos = vline.truncation_suffix_start + (column_offset_in_line - vline.ellipsis_pos - ellipsis_width);
} else if (column_offset_in_line >= vline.ellipsis_col + ellipsis_width) {
source_col_pos = vline.truncation_suffix_col_start + (column_offset_in_line - vline.ellipsis_col - ellipsis_width);
}
}
@@ -1884,14 +1884,14 @@ pub const OptimizedBuffer = struct {
}
if (vline.is_truncated) {
const column_offset_in_line = globalCharPos - line_col_offset;
const column_offset_in_line = document_cell_offset - line_col_offset;
const ellipsis_width: u32 = 3;
if (column_offset_in_line >= vline.ellipsis_pos and column_offset_in_line < vline.ellipsis_pos + ellipsis_width) {
if (column_offset_in_line >= vline.ellipsis_col and column_offset_in_line < vline.ellipsis_col + ellipsis_width) {
lineFg = defaultFg;
lineBg = defaultBg;
lineAttributes = defaultAttributes;
} else if (column_offset_in_line >= vline.ellipsis_pos + ellipsis_width) {
const suffix_col_pos = vline.truncation_suffix_start + (column_offset_in_line - vline.ellipsis_pos - ellipsis_width);
} else if (column_offset_in_line >= vline.ellipsis_col + ellipsis_width) {
const suffix_col_pos = vline.truncation_suffix_col_start + (column_offset_in_line - vline.ellipsis_col - ellipsis_width);
if (spans.len == 0) {
lineFg = defaultFg;
lineBg = defaultBg;
@@ -1932,7 +1932,7 @@ pub const OptimizedBuffer = struct {
const finalAttributes = lineAttributes;
var cell_idx: u32 = 0;
while (cell_idx < g_width) : (cell_idx += 1) {
while (cell_idx < cluster_width_cols) : (cell_idx += 1) {
if (view.getSelection()) |sel| {
const isSelected = selection_offset + cell_idx >= sel.start and selection_offset + cell_idx < sel.end;
if (isSelected) {
@@ -1953,7 +1953,7 @@ pub const OptimizedBuffer = struct {
// Skip zero-width characters (ZWJ, VS16, etc.) - don't render them
// Don't increment col since they take no space
if (g_width == 0) {
if (cluster_width_cols == 0) {
continue;
}
@@ -1983,8 +1983,8 @@ pub const OptimizedBuffer = struct {
const tab_indicator_color = view.getTabIndicatorColor();
var tab_col: u32 = 0;
while (tab_col < g_width) : (tab_col += 1) {
if (column_in_line + tab_col >= horizontal_offset + viewport_width) break;
while (tab_col < cluster_width_cols) : (tab_col += 1) {
if (rendered_col_in_vline + tab_col >= horizontal_offset + viewport_width) break;
const tab_x = currentX + @as(i32, @intCast(tab_col));
if (tab_x < 0) continue;
if (tab_x >= @as(i32, @intCast(self.width))) break;
@@ -2008,26 +2008,26 @@ pub const OptimizedBuffer = struct {
}
} else {
var encoded_char: u32 = 0;
if (grapheme_bytes.len == 1 and g_width == 1 and grapheme_bytes[0] >= 32) {
if (grapheme_bytes.len == 1 and cluster_width_cols == 1 and grapheme_bytes[0] >= 32) {
encoded_char = @as(u32, grapheme_bytes[0]);
} else {
const gid = self.pool.alloc(grapheme_bytes) catch |err| {
logger.warn("GraphemePool.alloc FAILED for grapheme (len={d}, bytes={any}): {}", .{ grapheme_bytes.len, grapheme_bytes, err });
globalCharPos += g_width;
currentX += @as(i32, @intCast(g_width));
col += g_width;
document_cell_offset += cluster_width_cols;
currentX += @as(i32, @intCast(cluster_width_cols));
col += cluster_width_cols;
continue;
};
encoded_char = gp.packGraphemeStart(gid & gp.GRAPHEME_ID_MASK, g_width);
encoded_char = gp.packGraphemeStart(gid & gp.GRAPHEME_ID_MASK, cluster_width_cols);
}
if (useTransparentTextFastPath) {
const index = self.coordsToIndex(@intCast(currentX), @intCast(currentY));
if (self.trySetTransparentTextCellFast(index, encoded_char, drawFg, drawAttributes)) {
globalCharPos += g_width;
currentX += @as(i32, @intCast(g_width));
column_in_line += g_width;
col += g_width;
document_cell_offset += cluster_width_cols;
currentX += @as(i32, @intCast(cluster_width_cols));
rendered_col_in_vline += cluster_width_cols;
col += cluster_width_cols;
continue;
}
}
@@ -2039,10 +2039,10 @@ pub const OptimizedBuffer = struct {
);
}
globalCharPos += g_width;
currentX += @as(i32, @intCast(g_width));
column_in_line += g_width;
col += g_width;
document_cell_offset += cluster_width_cols;
currentX += @as(i32, @intCast(cluster_width_cols));
rendered_col_in_vline += cluster_width_cols;
col += cluster_width_cols;
}
}
@@ -2052,7 +2052,7 @@ pub const OptimizedBuffer = struct {
if (is_last_vline_of_logical_line) {
const is_last_logical_line = vline.source_line + 1 >= total_line_count;
if (!is_last_logical_line) {
globalCharPos += 1;
document_cell_offset += 1;
}
}
packages/native/src/edit-buffer.zig
+10 -10
View File
@@ -244,17 +244,17 @@ pub const EditBuffer = struct {
chunk: *const TextChunk,
weight: u32,
) error{ OutOfBounds, OutOfMemory }!struct { left: TextChunk, right: TextChunk } {
const chunk_weight = chunk.width;
const chunk_weight = chunk.width_cols;
if (weight == 0) {
return .{
.left = TextChunk{ .mem_id = 0, .byte_start = 0, .byte_end = 0, .width = 0 },
.left = TextChunk{ .mem_id = 0, .byte_start = 0, .byte_end = 0, .width_cols = 0 },
.right = chunk.*,
};
} else if (weight >= chunk_weight) {
return .{
.left = chunk.*,
.right = TextChunk{ .mem_id = 0, .byte_start = 0, .byte_end = 0, .width = 0 },
.right = TextChunk{ .mem_id = 0, .byte_start = 0, .byte_end = 0, .width_cols = 0 },
};
}
@@ -321,7 +321,7 @@ pub const EditBuffer = struct {
var result = try self.tb.textToSegments(self.allocator, bytes, base_mem_id, base_start, false);
defer result.segments.deinit(result.allocator);
const inserted_width = result.total_width;
const inserted_width_cols = result.total_width_cols;
// Calculate width after last break
var width_after_last_break: u32 = 0;
@@ -331,7 +331,7 @@ pub const EditBuffer = struct {
num_breaks += 1;
width_after_last_break = 0;
} else if (seg.asText()) |chunk| {
width_after_last_break += chunk.width;
width_after_last_break += chunk.width_cols;
}
}
@@ -349,7 +349,7 @@ pub const EditBuffer = struct {
.offset = new_offset,
};
} else {
const new_col = cursor.col + inserted_width;
const new_col = cursor.col + inserted_width_cols;
const new_offset = iter_mod.coordsToOffset(self.tb.rope(), cursor.row, new_col) orelse 0;
self.cursors.items[0] = .{
.row = cursor.row,
@@ -741,7 +741,7 @@ pub const EditBuffer = struct {
const seg = self.tb.rope().get(seg_idx) orelse break;
if (seg.isBreak() or seg.isLineStart()) break;
if (seg.asText()) |chunk| {
const next_cols = cols_before + chunk.width;
const next_cols = cols_before + chunk.width_cols;
const layout = self.tb.getLayoutInfoFor(chunk) catch {
cols_before = next_cols;
passed_cursor = passed_cursor or cursor.col < next_cols;
@@ -764,7 +764,7 @@ pub const EditBuffer = struct {
const local_cursor_col = if (cursor.col > cols_before) cursor.col - cols_before else 0;
for (wrap_breaks) |wrap_break| {
const break_col = wrap_break.col_offset;
const break_col = wrap_break.col_start;
const target_col = cols_before + wrap_break.colEnd();
// If we've passed the cursor chunk, any break is valid
@@ -780,7 +780,7 @@ pub const EditBuffer = struct {
// for script-transition cases like "a日", "日a", or "丽abc".
// Only accept it when the boundary starts on a word codepoint.
if (!passed_cursor and break_col == local_cursor_col) {
const break_byte_offset: usize = @intCast(wrap_break.byte_offset);
const break_byte_offset: usize = @intCast(wrap_break.byte_start);
const chunk_bytes = chunk.getBytes(self.tb.memRegistry());
if (break_byte_offset < chunk_bytes.len) {
const break_cp = utf8.decodeUtf8Unchecked(chunk_bytes, break_byte_offset).cp;
@@ -827,7 +827,7 @@ pub const EditBuffer = struct {
const seg = self.tb.rope().get(seg_idx) orelse break;
if (seg.isBreak() or seg.isLineStart()) break;
if (seg.asText()) |chunk| {
const next_cols = cols_before + chunk.width;
const next_cols = cols_before + chunk.width_cols;
const layout = self.tb.getLayoutInfoFor(chunk) catch {
cols_before = next_cols;
packages/native/src/editor-view.zig
+11 -11
View File
@@ -466,8 +466,8 @@ pub const EditorView = struct {
var new_offset_x = vp.x;
if (self.text_buffer_view.wrap_mode == .none) {
const max_line_width = iter_mod.getMaxLineWidth(self.edit_buffer.tb.rope());
const max_offset_x = if (max_line_width > vp.width) max_line_width - vp.width else 0;
const max_line_width_cols = iter_mod.getMaxLineWidth(self.edit_buffer.tb.rope());
const max_offset_x = if (max_line_width_cols > vp.width) max_line_width_cols - vp.width else 0;
if (vp.x > max_offset_x) {
new_offset_x = max_offset_x;
}
@@ -541,12 +541,12 @@ pub const EditorView = struct {
const cursor = self.edit_buffer.getPrimaryCursor();
const vline = &vlines[visual_row];
if (vline.source_line != cursor.row or cursor.offset < vline.col_offset) {
if (vline.source_line != cursor.row or cursor.offset < vline.document_cell_offset) {
self.cursor_visual_affinity = null;
return;
}
const visual_col = cursor.offset - vline.col_offset;
const visual_col = cursor.offset - vline.document_cell_offset;
var visual_col_max = vline.width_cols;
const viewport_width = if (self.text_buffer_view.getViewport()) |vp| vp.width else vline.width_cols;
// An overwide visual line may be one atomic grapheme; do not reserve a
@@ -624,7 +624,7 @@ pub const EditorView = struct {
}
const vline = &vlines[visual_row_idx];
const vline_start_col = vline.source_col_offset;
const vline_start_col = vline.source_col_start;
// Calculate visual column within this virtual line
const visual_col = if (clamped_col >= vline_start_col)
@@ -653,7 +653,7 @@ pub const EditorView = struct {
const vline = &vlines[visual_row];
const clamped_visual_col = @min(visual_col, vline.width_cols);
const logical_col = vline.source_col_offset + clamped_visual_col;
const logical_col = vline.source_col_start + clamped_visual_col;
const logical_row = @as(u32, @intCast(vline.source_line));
const offset = iter_mod.coordsToOffset(self.edit_buffer.tb.rope(), logical_row, logical_col) orelse 0;
@@ -836,7 +836,7 @@ pub const EditorView = struct {
}
const vline = &vlines[vcursor.visual_row];
const logical_col = vline.source_col_offset; // Start column of this visual line
const logical_col = vline.source_col_start; // Start column of this visual line
const logical_row = @as(u32, @intCast(vline.source_line));
const offset = iter_mod.coordsToOffset(self.edit_buffer.tb.rope(), logical_row, logical_col) orelse 0;
@@ -874,16 +874,16 @@ pub const EditorView = struct {
else
clampVisualColToStayOnVisualRow(vlines, vcursor.visual_row, vline.width_cols);
if (self.text_buffer_view.getSelectionOccupancy() == .cell and !oversized_visual_line) {
const target_offset = vline.col_offset + target_visual_col;
const target_offset = vline.document_cell_offset + target_visual_col;
if (self.edit_buffer.tb.cursorUnitBoundsAtOffset(target_offset)) |bounds| {
if (bounds.start < target_offset) {
const target = if (bounds.start >= vline.col_offset) bounds.start else bounds.end;
target_visual_col = @min(target -| vline.col_offset, vline.width_cols);
const target = if (bounds.start >= vline.document_cell_offset) bounds.start else bounds.end;
target_visual_col = @min(target -| vline.document_cell_offset, vline.width_cols);
}
}
}
const logical_row = @as(u32, @intCast(vline.source_line));
const logical_col = vline.source_col_offset + target_visual_col;
const logical_col = vline.source_col_start + target_visual_col;
const offset = iter_mod.coordsToOffset(self.edit_buffer.tb.rope(), logical_row, logical_col) orelse 0;
return .{
packages/native/src/lib.zig
+15 -15
View File
@@ -3709,13 +3709,13 @@ export fn encodeUnicode(
// Check if ASCII only for optimization
const is_ascii_only = utf8.isAsciiOnly(text);
// Find grapheme info
var grapheme_list: std.ArrayListUnmanaged(utf8.GraphemeInfo) = .empty;
defer grapheme_list.deinit(globalAllocator);
// Find sparse render-cluster metadata.
var render_cluster_list: std.ArrayListUnmanaged(utf8.RenderClusterInfo) = .empty;
defer render_cluster_list.deinit(globalAllocator);
const tab_width: u8 = 2;
utf8.findGraphemeInfo(globalAllocator, text, tab_width, is_ascii_only, wMethod, &grapheme_list) catch return false;
const specials = grapheme_list.items;
utf8.findRenderClusterInfo(globalAllocator, text, tab_width, is_ascii_only, wMethod, &render_cluster_list) catch return false;
const render_clusters = render_cluster_list.items;
// Allocate output array
const estimated_count = if (is_ascii_only) text.len else text.len * 2;
@@ -3751,27 +3751,27 @@ export fn encodeUnicode(
var special_idx: usize = 0;
while (byte_offset < text.len) {
const at_special = special_idx < specials.len and specials[special_idx].col_offset == col;
const at_special = special_idx < render_clusters.len and render_clusters[special_idx].col_start == col;
var grapheme_bytes: []const u8 = undefined;
var g_width: u32 = undefined;
var cluster_width_cols: u32 = undefined;
if (at_special) {
const g = specials[special_idx];
grapheme_bytes = text[g.byte_offset .. g.byte_offset + g.byte_len];
g_width = g.width;
byte_offset = g.byte_offset + g.byte_len;
const g = render_clusters[special_idx];
grapheme_bytes = text[g.byte_start .. g.byte_start + g.byte_len];
cluster_width_cols = g.width_cols;
byte_offset = g.byte_start + g.byte_len;
special_idx += 1;
} else {
if (byte_offset >= text.len) break;
grapheme_bytes = text[byte_offset .. byte_offset + 1];
g_width = 1;
cluster_width_cols = 1;
byte_offset += 1;
}
const cell_width = utf8.getWidthAt(text, if (at_special) specials[special_idx - 1].byte_offset else byte_offset - 1, tab_width, wMethod);
const cell_width = utf8.getWidthAt(text, if (at_special) render_clusters[special_idx - 1].byte_start else byte_offset - 1, tab_width, wMethod);
if (cell_width == 0) {
col += g_width;
col += cluster_width_cols;
continue;
}
@@ -3805,7 +3805,7 @@ export fn encodeUnicode(
};
pending_gid = null; // Successfully stored, no longer pending
result_idx += 1;
col += g_width;
col += cluster_width_cols;
}
// Trim to actual size
packages/native/src/rope.zig
+2 -2
View File
@@ -830,8 +830,8 @@ pub fn Rope(comptime T: type) type {
try writer.writeByte(':');
try writer.print("w{d}", .{metrics.weight()});
if (@hasDecl(T.Metrics, "total_width")) {
try writer.print(",tw{d}", .{metrics.custom.total_width});
if (@hasDecl(T.Metrics, "total_width_cols")) {
try writer.print(",tw{d}", .{metrics.custom.total_width_cols});
}
if (@hasDecl(T.Metrics, "total_bytes")) {
try writer.print(",b{d}", .{metrics.custom.total_bytes});
packages/native/src/tests/text-buffer-drawing_test.zig
+3 -3
View File
@@ -722,7 +722,7 @@ test "drawTextBuffer - word wrap does not split multi-byte UTF-8 characters" {
for (vlines) |vline| {
var line_buffer: [200]u8 = undefined;
const line_start_offset = vline.col_offset;
const line_start_offset = vline.document_cell_offset;
const line_end_offset = line_start_offset + vline.width_cols;
const extracted = tb.getTextRange(line_start_offset, line_end_offset, &line_buffer);
@@ -733,10 +733,10 @@ test "drawTextBuffer - word wrap does not split multi-byte UTF-8 characters" {
try std.testing.expect(vlines.len == 2);
var full_buffer: [200]u8 = undefined;
const line0_len = tb.getTextRange(vlines[0].col_offset, vlines[0].col_offset + vlines[0].width_cols, &full_buffer);
const line0_len = tb.getTextRange(vlines[0].document_cell_offset, vlines[0].document_cell_offset + vlines[0].width_cols, &full_buffer);
const line0_text = full_buffer[0..line0_len];
const line1_len = tb.getTextRange(vlines[1].col_offset, vlines[1].col_offset + vlines[1].width_cols, &full_buffer);
const line1_len = tb.getTextRange(vlines[1].document_cell_offset, vlines[1].document_cell_offset + vlines[1].width_cols, &full_buffer);
const line1_text = full_buffer[0..line1_len];
const line0_ends_with_kai = std.mem.endsWith(u8, line0_text, "界");
packages/native/src/tests/text-buffer-iterators_test.zig
+13 -13
View File
@@ -50,7 +50,7 @@ test "walkLines - single text segment" {
.mem_id = 0,
.byte_start = 0,
.byte_end = 10,
.width = 10,
.width_cols = 10,
.flags = 0,
},
});
@@ -86,7 +86,7 @@ test "walkLines - text + break + text" {
.mem_id = 0,
.byte_start = 0,
.byte_end = 10,
.width = 10,
.width_cols = 10,
.flags = 0,
},
});
@@ -97,7 +97,7 @@ test "walkLines - text + break + text" {
.mem_id = 0,
.byte_start = 10,
.byte_end = 15,
.width = 5,
.width_cols = 5,
.flags = 0,
},
});
@@ -140,7 +140,7 @@ test "walkLines - exclude newlines in offset" {
.mem_id = 0,
.byte_start = 0,
.byte_end = 10,
.width = 10,
.width_cols = 10,
.flags = 0,
},
});
@@ -151,7 +151,7 @@ test "walkLines - exclude newlines in offset" {
.mem_id = 0,
.byte_start = 10,
.byte_end = 15,
.width = 5,
.width_cols = 5,
.flags = 0,
},
});
@@ -194,7 +194,7 @@ test "coordsToOffset - valid coordinates" {
.mem_id = 0,
.byte_start = 0,
.byte_end = 10,
.width = 10,
.width_cols = 10,
.flags = 0,
},
});
@@ -205,7 +205,7 @@ test "coordsToOffset - valid coordinates" {
.mem_id = 0,
.byte_start = 10,
.byte_end = 15,
.width = 5,
.width_cols = 5,
.flags = 0,
},
});
@@ -235,7 +235,7 @@ test "offsetToCoords - valid offsets" {
.mem_id = 0,
.byte_start = 0,
.byte_end = 10,
.width = 10,
.width_cols = 10,
.flags = 0,
},
});
@@ -246,7 +246,7 @@ test "offsetToCoords - valid offsets" {
.mem_id = 0,
.byte_start = 10,
.byte_end = 15,
.width = 5,
.width_cols = 5,
.flags = 0,
},
});
@@ -284,7 +284,7 @@ test "Helper functions" {
.mem_id = 0,
.byte_start = 0,
.byte_end = 10,
.width = 10,
.width_cols = 10,
.flags = 0,
},
});
@@ -295,7 +295,7 @@ test "Helper functions" {
.mem_id = 0,
.byte_start = 10,
.byte_end = 15,
.width = 5,
.width_cols = 5,
.flags = 0,
},
});
@@ -317,7 +317,7 @@ test "coordsToOffset and offsetToCoords - round trip" {
.mem_id = 0,
.byte_start = 0,
.byte_end = 10,
.width = 10,
.width_cols = 10,
.flags = 0,
},
});
@@ -328,7 +328,7 @@ test "coordsToOffset and offsetToCoords - round trip" {
.mem_id = 0,
.byte_start = 10,
.byte_end = 18,
.width = 8,
.width_cols = 8,
.flags = 0,
},
});
packages/native/src/tests/text-buffer-segment_test.zig
+91 -91
View File
@@ -25,13 +25,13 @@ test "TextChunk.getLayoutInfo returns direct byte and column metadata" {
.mem_id = mem_id,
.byte_start = 0,
.byte_end = @intCast(text.len),
.width = @intCast(utf8.calculateTextWidth(text, 2, false, .unicode)),
.width_cols = @intCast(utf8.calculateTextWidth(text, 2, false, .unicode)),
};
const layout = try chunk.getLayoutInfo(arena.allocator(), &registry, 2, .unicode);
try testing.expectEqual(@as(usize, 1), layout.wrap_breaks.len);
try testing.expectEqual(@as(u32, 6), layout.wrap_breaks[0].byte_offset);
try testing.expectEqual(@as(u32, 4), layout.wrap_breaks[0].col_offset);
try testing.expectEqual(@as(u32, 6), layout.wrap_breaks[0].byte_start);
try testing.expectEqual(@as(u32, 4), layout.wrap_breaks[0].col_start);
try testing.expectEqual(@as(u32, 7), layout.wrap_breaks[0].byteEnd());
try testing.expectEqual(@as(u32, 5), layout.wrap_breaks[0].colEnd());
@@ -44,14 +44,14 @@ test "TextChunk.getLayoutInfo returns direct byte and column metadata" {
.mem_id = zero_width_mem_id,
.byte_start = 0,
.byte_end = @intCast(zero_width_text.len),
.width = @intCast(utf8.calculateTextWidth(zero_width_text, 2, false, .unicode)),
.width_cols = @intCast(utf8.calculateTextWidth(zero_width_text, 2, false, .unicode)),
};
const zero_width_layout = try zero_width_chunk.getLayoutInfo(arena.allocator(), &registry, 2, .unicode);
try testing.expectEqual(@as(usize, 1), zero_width_layout.wrap_breaks.len);
try testing.expectEqual(@as(u32, 1), zero_width_layout.wrap_breaks[0].byte_offset);
try testing.expectEqual(@as(u32, 1), zero_width_layout.wrap_breaks[0].byte_start);
try testing.expectEqual(@as(u32, 3), zero_width_layout.wrap_breaks[0].byte_len);
try testing.expectEqual(@as(u32, 1), zero_width_layout.wrap_breaks[0].col_offset);
try testing.expectEqual(@as(u32, 0), zero_width_layout.wrap_breaks[0].width);
try testing.expectEqual(@as(u32, 1), zero_width_layout.wrap_breaks[0].col_start);
try testing.expectEqual(@as(u32, 0), zero_width_layout.wrap_breaks[0].width_cols);
}
test "findChunkLayoutInfo classifies direct byte and column break metadata" {
@@ -62,15 +62,15 @@ test "findChunkLayoutInfo classifies direct byte and column break metadata" {
};
const cases = [_]Case{
.{ .text = "AB🌟 CD", .expected = &.{.{ .byte_offset = 6, .col_offset = 4, .byte_len = 1, .width = 1, .kind = .whitespace }} },
.{ .text = "中A", .expected = &.{.{ .byte_offset = 0, .col_offset = 0, .byte_len = 3, .width = 2, .kind = .script_transition }} },
.{ .text = "A中", .expected = &.{.{ .byte_offset = 0, .col_offset = 0, .byte_len = 1, .width = 1, .kind = .script_transition }} },
.{ .text = "a\u{0301}中", .expected = &.{.{ .byte_offset = 0, .col_offset = 0, .byte_len = 3, .width = 1, .kind = .script_transition }} },
.{ .text = "0123456789abcdef中", .expected = &.{.{ .byte_offset = 15, .col_offset = 15, .byte_len = 1, .width = 1, .kind = .script_transition }} },
.{ .text = "a\u{200B}b", .expected = &.{.{ .byte_offset = 1, .col_offset = 1, .byte_len = 3, .width = 0, .kind = .whitespace }} },
.{ .text = "a\tb", .tab_width = 2, .expected = &.{.{ .byte_offset = 1, .col_offset = 1, .byte_len = 1, .width = 2, .kind = .whitespace }} },
.{ .text = "a\tb", .tab_width = 4, .expected = &.{.{ .byte_offset = 1, .col_offset = 1, .byte_len = 1, .width = 4, .kind = .whitespace }} },
.{ .text = "ab,cd", .expected = &.{.{ .byte_offset = 2, .col_offset = 2, .byte_len = 1, .width = 1, .kind = .punctuation }} },
.{ .text = "AB🌟 CD", .expected = &.{.{ .byte_start = 6, .col_start = 4, .byte_len = 1, .width_cols = 1, .kind = .whitespace }} },
.{ .text = "中A", .expected = &.{.{ .byte_start = 0, .col_start = 0, .byte_len = 3, .width_cols = 2, .kind = .script_transition }} },
.{ .text = "A中", .expected = &.{.{ .byte_start = 0, .col_start = 0, .byte_len = 1, .width_cols = 1, .kind = .script_transition }} },
.{ .text = "a\u{0301}中", .expected = &.{.{ .byte_start = 0, .col_start = 0, .byte_len = 3, .width_cols = 1, .kind = .script_transition }} },
.{ .text = "0123456789abcdef中", .expected = &.{.{ .byte_start = 15, .col_start = 15, .byte_len = 1, .width_cols = 1, .kind = .script_transition }} },
.{ .text = "a\u{200B}b", .expected = &.{.{ .byte_start = 1, .col_start = 1, .byte_len = 3, .width_cols = 0, .kind = .whitespace }} },
.{ .text = "a\tb", .tab_width = 2, .expected = &.{.{ .byte_start = 1, .col_start = 1, .byte_len = 1, .width_cols = 2, .kind = .whitespace }} },
.{ .text = "a\tb", .tab_width = 4, .expected = &.{.{ .byte_start = 1, .col_start = 1, .byte_len = 1, .width_cols = 4, .kind = .whitespace }} },
.{ .text = "ab,cd", .expected = &.{.{ .byte_start = 2, .col_start = 2, .byte_len = 1, .width_cols = 1, .kind = .punctuation }} },
};
var breaks: std.ArrayListUnmanaged(utf8.LayoutWrapBreak) = .empty;
@@ -89,10 +89,10 @@ test "walkChunkLayoutInfo keeps Prepend joined to an ASCII vector" {
_ = try utf8.findChunkLayoutInfo(testing.allocator, text, 2, false, .unicode, &breaks);
try testing.expectEqual(@as(usize, 1), breaks.items.len);
try testing.expectEqual(@as(u32, 0), breaks.items[0].byte_offset);
try testing.expectEqual(@as(u32, 0), breaks.items[0].byte_start);
try testing.expectEqual(@as(u32, 3), breaks.items[0].byte_len);
try testing.expectEqual(@as(u32, 0), breaks.items[0].col_offset);
try testing.expectEqual(@as(u32, 1), breaks.items[0].width);
try testing.expectEqual(@as(u32, 0), breaks.items[0].col_start);
try testing.expectEqual(@as(u32, 1), breaks.items[0].width_cols);
try testing.expectEqual(utf8.LayoutWrapBreakKind.whitespace, breaks.items[0].kind);
}
@@ -138,20 +138,20 @@ test "TextChunk.getLayoutInfo refreshes tab-dependent metadata" {
.mem_id = mem_id,
.byte_start = 0,
.byte_end = @intCast(text.len),
.width = 4,
.width_cols = 4,
};
const first = try chunk.getLayoutInfo(arena.allocator(), &registry, 2, .unicode);
try testing.expectEqual(@as(u32, 2), first.wrap_breaks[0].width);
try testing.expectEqual(@as(u32, 2), first.wrap_breaks[0].width_cols);
const capacity_after_first = arena.queryCapacity();
const second = try chunk.getLayoutInfo(arena.allocator(), &registry, 8, .unicode);
try testing.expectEqual(@as(u32, 8), second.wrap_breaks[0].width);
try testing.expectEqual(@as(u32, 8), second.wrap_breaks[0].width_cols);
try testing.expectEqual(@intFromPtr(first.wrap_breaks.ptr), @intFromPtr(second.wrap_breaks.ptr));
try testing.expectEqual(capacity_after_first, arena.queryCapacity());
}
test "TextChunk.getGraphemes reuses tab-dependent cache storage" {
test "TextChunk.getRenderClusters reuses tab-dependent cache storage" {
var arena = std.heap.ArenaAllocator.init(testing.allocator);
defer arena.deinit();
@@ -163,19 +163,19 @@ test "TextChunk.getGraphemes reuses tab-dependent cache storage" {
.mem_id = mem_id,
.byte_start = 0,
.byte_end = @intCast(text.len),
.width = 4,
.width_cols = 4,
};
const first = try chunk.getGraphemes(arena.allocator(), &registry, 2, .unicode);
try testing.expectEqual(@as(u32, 2), first[0].width);
const first = try chunk.getRenderClusters(arena.allocator(), &registry, 2, .unicode);
try testing.expectEqual(@as(u32, 2), first[0].width_cols);
const capacity_after_first = arena.queryCapacity();
const wider = try chunk.getGraphemes(arena.allocator(), &registry, 8, .unicode);
try testing.expectEqual(@as(u32, 8), wider[0].width);
const wider = try chunk.getRenderClusters(arena.allocator(), &registry, 8, .unicode);
try testing.expectEqual(@as(u32, 8), wider[0].width_cols);
try testing.expectEqual(@intFromPtr(first.ptr), @intFromPtr(wider.ptr));
const narrower = try chunk.getGraphemes(arena.allocator(), &registry, 4, .unicode);
try testing.expectEqual(@as(u32, 4), narrower[0].width);
const narrower = try chunk.getRenderClusters(arena.allocator(), &registry, 4, .unicode);
try testing.expectEqual(@as(u32, 4), narrower[0].width_cols);
try testing.expectEqual(@intFromPtr(first.ptr), @intFromPtr(narrower.ptr));
try testing.expectEqual(capacity_after_first, arena.queryCapacity());
}
@@ -185,14 +185,14 @@ test "Segment.measure - text chunk" {
.mem_id = 0,
.byte_start = 0,
.byte_end = 10,
.width = 10,
.width_cols = 10,
.flags = TextChunk.Flags.ASCII_ONLY,
};
const seg: Segment = .{ .text = chunk };
const metrics = seg.measure();
try testing.expectEqual(@as(u32, 10), metrics.total_width);
try testing.expectEqual(@as(u32, 10), metrics.max_line_width);
try testing.expectEqual(@as(u32, 10), metrics.total_width_cols);
try testing.expectEqual(@as(u32, 10), metrics.max_line_width_cols);
try testing.expect(metrics.ascii_only);
}
@@ -200,8 +200,8 @@ test "Segment.measure - break" {
const seg: Segment = .{ .brk = {} };
const metrics = seg.measure();
try testing.expectEqual(@as(u32, 0), metrics.total_width);
try testing.expectEqual(@as(u32, 0), metrics.max_line_width);
try testing.expectEqual(@as(u32, 0), metrics.total_width_cols);
try testing.expectEqual(@as(u32, 0), metrics.max_line_width_cols);
try testing.expect(metrics.ascii_only);
}
@@ -216,7 +216,7 @@ test "Segment.isBreak and isText" {
.mem_id = 0,
.byte_start = 0,
.byte_end = 10,
.width = 10,
.width_cols = 10,
.flags = 0,
},
};
@@ -233,13 +233,13 @@ test "Segment.asText" {
.mem_id = 0,
.byte_start = 0,
.byte_end = 10,
.width = 10,
.width_cols = 10,
.flags = 0,
};
const text_seg: Segment = .{ .text = chunk };
const retrieved = text_seg.asText();
try testing.expect(retrieved != null);
try testing.expectEqual(@as(u32, 10), retrieved.?.width);
try testing.expectEqual(@as(u32, 10), retrieved.?.width_cols);
const brk_seg: Segment = .{ .brk = {} };
try testing.expect(brk_seg.asText() == null);
@@ -247,102 +247,102 @@ test "Segment.asText" {
test "Metrics.add - two text segments" {
var left: Segment.Metrics = .{
.total_width = 10,
.max_line_width = 10,
.total_width_cols = 10,
.max_line_width_cols = 10,
.ascii_only = true,
};
const right: Segment.Metrics = .{
.total_width = 5,
.max_line_width = 5,
.total_width_cols = 5,
.max_line_width_cols = 5,
.ascii_only = true,
};
left.add(right);
try testing.expectEqual(@as(u32, 15), left.total_width);
try testing.expectEqual(@as(u32, 10), left.max_line_width);
try testing.expectEqual(@as(u32, 15), left.total_width_cols);
try testing.expectEqual(@as(u32, 10), left.max_line_width_cols);
try testing.expect(left.ascii_only);
}
test "Metrics.add - text, break, text" {
var left: Segment.Metrics = .{
.total_width = 10,
.max_line_width = 10,
.total_width_cols = 10,
.max_line_width_cols = 10,
.ascii_only = true,
};
const middle: Segment.Metrics = .{
.total_width = 0,
.max_line_width = 0,
.total_width_cols = 0,
.max_line_width_cols = 0,
.ascii_only = true,
};
left.add(middle);
try testing.expectEqual(@as(u32, 10), left.total_width);
try testing.expectEqual(@as(u32, 10), left.max_line_width);
try testing.expectEqual(@as(u32, 10), left.total_width_cols);
try testing.expectEqual(@as(u32, 10), left.max_line_width_cols);
const right: Segment.Metrics = .{
.total_width = 5,
.max_line_width = 5,
.total_width_cols = 5,
.max_line_width_cols = 5,
.ascii_only = true,
};
left.add(right);
try testing.expectEqual(@as(u32, 15), left.total_width);
try testing.expectEqual(@as(u32, 10), left.max_line_width);
try testing.expectEqual(@as(u32, 15), left.total_width_cols);
try testing.expectEqual(@as(u32, 10), left.max_line_width_cols);
}
test "Metrics.add - multiple breaks" {
var metrics: Segment.Metrics = .{
.total_width = 10,
.max_line_width = 10,
.total_width_cols = 10,
.max_line_width_cols = 10,
.ascii_only = true,
};
metrics.add(.{
.total_width = 0,
.max_line_width = 0,
.total_width_cols = 0,
.max_line_width_cols = 0,
.ascii_only = true,
});
metrics.add(.{
.total_width = 20,
.max_line_width = 20,
.total_width_cols = 20,
.max_line_width_cols = 20,
.ascii_only = true,
});
try testing.expectEqual(@as(u32, 30), metrics.total_width);
try testing.expectEqual(@as(u32, 20), metrics.max_line_width);
try testing.expectEqual(@as(u32, 30), metrics.total_width_cols);
try testing.expectEqual(@as(u32, 20), metrics.max_line_width_cols);
metrics.add(.{
.total_width = 0,
.max_line_width = 0,
.total_width_cols = 0,
.max_line_width_cols = 0,
.ascii_only = true,
});
metrics.add(.{
.total_width = 5,
.max_line_width = 5,
.total_width_cols = 5,
.max_line_width_cols = 5,
.ascii_only = true,
});
try testing.expectEqual(@as(u32, 35), metrics.total_width);
try testing.expectEqual(@as(u32, 20), metrics.max_line_width);
try testing.expectEqual(@as(u32, 35), metrics.total_width_cols);
try testing.expectEqual(@as(u32, 20), metrics.max_line_width_cols);
}
test "Metrics.add - non-ASCII propagation" {
var left: Segment.Metrics = .{
.total_width = 10,
.max_line_width = 10,
.total_width_cols = 10,
.max_line_width_cols = 10,
.ascii_only = true,
};
const right: Segment.Metrics = .{
.total_width = 5,
.max_line_width = 5,
.total_width_cols = 5,
.max_line_width_cols = 5,
.ascii_only = false,
};
@@ -362,7 +362,7 @@ test "UnifiedRope - basic operations" {
.mem_id = 0,
.byte_start = 0,
.byte_end = 10,
.width = 10,
.width_cols = 10,
.flags = TextChunk.Flags.ASCII_ONLY,
},
};
@@ -376,7 +376,7 @@ test "UnifiedRope - basic operations" {
.mem_id = 0,
.byte_start = 10,
.byte_end = 15,
.width = 5,
.width_cols = 5,
.flags = TextChunk.Flags.ASCII_ONLY,
},
};
@@ -384,8 +384,8 @@ test "UnifiedRope - basic operations" {
const metrics = rope.root.metrics();
try testing.expectEqual(@as(u32, 5), rope.count());
try testing.expectEqual(@as(u32, 15), metrics.custom.total_width);
try testing.expectEqual(@as(u32, 10), metrics.custom.max_line_width);
try testing.expectEqual(@as(u32, 15), metrics.custom.total_width_cols);
try testing.expectEqual(@as(u32, 10), metrics.custom.max_line_width_cols);
}
test "UnifiedRope - empty rope metrics" {
@@ -397,7 +397,7 @@ test "UnifiedRope - empty rope metrics" {
const metrics = rope.root.metrics();
try testing.expectEqual(@as(u32, 1), rope.count());
try testing.expectEqual(@as(u32, 0), metrics.custom.total_width);
try testing.expectEqual(@as(u32, 0), metrics.custom.total_width_cols);
}
test "UnifiedRope - single text segment" {
@@ -411,15 +411,15 @@ test "UnifiedRope - single text segment" {
.mem_id = 0,
.byte_start = 0,
.byte_end = 20,
.width = 20,
.width_cols = 20,
.flags = 0,
},
});
const metrics = rope.root.metrics();
try testing.expectEqual(@as(u32, 2), rope.count());
try testing.expectEqual(@as(u32, 20), metrics.custom.total_width);
try testing.expectEqual(@as(u32, 20), metrics.custom.max_line_width);
try testing.expectEqual(@as(u32, 20), metrics.custom.total_width_cols);
try testing.expectEqual(@as(u32, 20), metrics.custom.max_line_width_cols);
}
test "UnifiedRope - multiple lines with varying widths" {
@@ -434,7 +434,7 @@ test "UnifiedRope - multiple lines with varying widths" {
.mem_id = 0,
.byte_start = 0,
.byte_end = 10,
.width = 10,
.width_cols = 10,
.flags = 0,
},
});
@@ -445,7 +445,7 @@ test "UnifiedRope - multiple lines with varying widths" {
.mem_id = 0,
.byte_start = 10,
.byte_end = 40,
.width = 30,
.width_cols = 30,
.flags = 0,
},
});
@@ -456,15 +456,15 @@ test "UnifiedRope - multiple lines with varying widths" {
.mem_id = 0,
.byte_start = 40,
.byte_end = 55,
.width = 15,
.width_cols = 15,
.flags = 0,
},
});
const metrics = rope.root.metrics();
try testing.expectEqual(@as(u32, 8), rope.count());
try testing.expectEqual(@as(u32, 55), metrics.custom.total_width);
try testing.expectEqual(@as(u32, 30), metrics.custom.max_line_width);
try testing.expectEqual(@as(u32, 55), metrics.custom.total_width_cols);
try testing.expectEqual(@as(u32, 30), metrics.custom.max_line_width_cols);
}
fn combineMetrics(left: Segment.Metrics, right: Segment.Metrics) Segment.Metrics {
@@ -475,20 +475,20 @@ fn combineMetrics(left: Segment.Metrics, right: Segment.Metrics) Segment.Metrics
test "combineMetrics helper function" {
const left: Segment.Metrics = .{
.total_width = 10,
.max_line_width = 10,
.total_width_cols = 10,
.max_line_width_cols = 10,
.ascii_only = true,
};
const right: Segment.Metrics = .{
.total_width = 5,
.max_line_width = 5,
.total_width_cols = 5,
.max_line_width_cols = 5,
.ascii_only = true,
};
const combined = combineMetrics(left, right);
try testing.expectEqual(@as(u32, 15), combined.total_width);
try testing.expectEqual(@as(u32, 10), combined.max_line_width);
try testing.expectEqual(@as(u32, 15), combined.total_width_cols);
try testing.expectEqual(@as(u32, 10), combined.max_line_width_cols);
try testing.expect(combined.ascii_only);
}
packages/native/src/tests/text-buffer-view_test.zig
+20 -20
View File
@@ -488,8 +488,8 @@ test "TextBufferView virtual line spans - with highlights" {
try std.testing.expectEqual(@as(usize, 0), vline0_info.source_line);
try std.testing.expectEqual(@as(usize, 0), vline1_info.source_line);
try std.testing.expectEqual(@as(u32, 0), vline0_info.col_offset);
try std.testing.expectEqual(@as(u32, 10), vline1_info.col_offset);
try std.testing.expectEqual(@as(u32, 0), vline0_info.source_col_start);
try std.testing.expectEqual(@as(u32, 10), vline1_info.source_col_start);
try std.testing.expect(vline0_info.spans.len > 0);
try std.testing.expect(vline1_info.spans.len > 0);
@@ -1034,10 +1034,10 @@ test "TextBufferView word wrapping - fragmented rope with word boundary" {
try std.testing.expectEqual(@as(u32, 6), vlines[1].width_cols);
var line_buf: [32]u8 = undefined;
const line0_len = tb.getTextRange(vlines[0].col_offset, vlines[0].col_offset + vlines[0].width_cols, &line_buf);
const line0_len = tb.getTextRange(vlines[0].document_cell_offset, vlines[0].document_cell_offset + vlines[0].width_cols, &line_buf);
try std.testing.expectEqualStrings("hello my good ", line_buf[0..line0_len]);
const line1_len = tb.getTextRange(vlines[1].col_offset, vlines[1].col_offset + vlines[1].width_cols, &line_buf);
const line1_len = tb.getTextRange(vlines[1].document_cell_offset, vlines[1].document_cell_offset + vlines[1].width_cols, &line_buf);
try std.testing.expectEqualStrings("friend", line_buf[0..line1_len]);
}
@@ -2579,8 +2579,8 @@ test "TextBufferView highlights - work correctly with wrapped lines" {
try std.testing.expectEqual(@as(usize, 0), vline0_info.source_line);
try std.testing.expectEqual(@as(usize, 0), vline1_info.source_line);
try std.testing.expectEqual(@as(u32, 0), vline0_info.col_offset);
try std.testing.expectEqual(@as(u32, 10), vline1_info.col_offset);
try std.testing.expectEqual(@as(u32, 0), vline0_info.source_col_start);
try std.testing.expectEqual(@as(u32, 10), vline1_info.source_col_start);
try std.testing.expect(vline0_info.spans.len > 0);
try std.testing.expect(vline1_info.spans.len > 0);
@@ -3197,8 +3197,8 @@ test "TextBufferView truncation - byte windows snap around wide graphemes" {
const line = view.getVirtualLines()[0];
try std.testing.expectEqual(@as(u32, 7), line.width_cols);
try std.testing.expectEqual(@as(u32, 2), line.ellipsis_pos);
try std.testing.expectEqual(@as(u32, 7), line.truncation_suffix_start);
try std.testing.expectEqual(@as(u32, 2), line.ellipsis_col);
try std.testing.expectEqual(@as(u32, 7), line.truncation_suffix_col_start);
try std.testing.expectEqual(@as(usize, 3), line.chunks.items.len);
for (line.chunks.items) |chunk| {
@@ -3315,7 +3315,7 @@ test "TextBufferView highlights - multiple highlights on wrapped line" {
for (0..vline_count) |i| {
const vline_info = view.getVirtualLineSpans(i);
try std.testing.expectEqual(@as(usize, 0), vline_info.source_line);
try std.testing.expectEqual(@as(u32, @intCast(i * 10)), vline_info.col_offset);
try std.testing.expectEqual(@as(u32, @intCast(i * 10)), vline_info.source_col_start);
}
}
@@ -3347,8 +3347,8 @@ test "TextBufferView highlights - with emojis and wrapping" {
try std.testing.expectEqual(@as(usize, 0), vline0_info.source_line);
try std.testing.expectEqual(@as(usize, 0), vline1_info.source_line);
try std.testing.expectEqual(@as(u32, 0), vline0_info.col_offset);
try std.testing.expect(vline1_info.col_offset == 6);
try std.testing.expectEqual(@as(u32, 0), vline0_info.source_col_start);
try std.testing.expect(vline1_info.source_col_start == 6);
try std.testing.expect(vline0_info.spans.len > 0);
}
@@ -3380,9 +3380,9 @@ test "TextBufferView highlights - with CJK characters and wrapping" {
try std.testing.expectEqual(@as(usize, 0), vline_info.source_line);
if (i == 0) {
try std.testing.expectEqual(@as(u32, 0), vline_info.col_offset);
try std.testing.expectEqual(@as(u32, 0), vline_info.source_col_start);
} else if (i == 1) {
try std.testing.expectEqual(@as(u32, 6), vline_info.col_offset);
try std.testing.expectEqual(@as(u32, 6), vline_info.source_col_start);
}
}
}
@@ -3415,8 +3415,8 @@ test "TextBufferView highlights - mixed ASCII and wide chars with wrapping" {
try std.testing.expectEqual(@as(usize, 0), vline0_info.source_line);
try std.testing.expectEqual(@as(usize, 0), vline1_info.source_line);
try std.testing.expectEqual(@as(u32, 0), vline0_info.col_offset);
try std.testing.expectEqual(@as(u32, 7), vline1_info.col_offset);
try std.testing.expectEqual(@as(u32, 0), vline0_info.source_col_start);
try std.testing.expectEqual(@as(u32, 7), vline1_info.source_col_start);
try std.testing.expect(vline0_info.spans.len > 0);
try std.testing.expect(vline1_info.spans.len > 0);
@@ -3447,8 +3447,8 @@ test "TextBufferView highlights - emoji at wrap boundary" {
const vline0_info = view.getVirtualLineSpans(0);
const vline1_info = view.getVirtualLineSpans(1);
try std.testing.expectEqual(@as(u32, 0), vline0_info.col_offset);
try std.testing.expect(vline1_info.col_offset >= 4);
try std.testing.expectEqual(@as(u32, 0), vline0_info.source_col_start);
try std.testing.expect(vline1_info.source_col_start >= 4);
}
test "TextBufferView highlights - emojis without wrapping" {
@@ -4292,10 +4292,10 @@ test "TextBufferView word wrapping - does not split 'uses' across lines" {
var line_buf: [1024]u8 = undefined;
var next_line_buf: [1024]u8 = undefined;
const line_len = tb.getTextRange(vlines[i].col_offset, vlines[i].col_offset + vlines[i].width_cols, &line_buf);
const line_len = tb.getTextRange(vlines[i].document_cell_offset, vlines[i].document_cell_offset + vlines[i].width_cols, &line_buf);
const next_line_len = tb.getTextRange(
vlines[i + 1].col_offset,
vlines[i + 1].col_offset + vlines[i + 1].width_cols,
vlines[i + 1].document_cell_offset,
vlines[i + 1].document_cell_offset + vlines[i + 1].width_cols,
&next_line_buf,
);
packages/native/src/tests/utf8_no_zwj_test.zig
+8 -8
View File
@@ -90,25 +90,25 @@ test "no_zwj: mixed text with ZWJ emoji" {
try testing.expectEqual(@as(u32, 14), width_no_zwj);
}
test "no_zwj: findGraphemeInfo splits ZWJ sequences" {
var result_unicode: std.ArrayListUnmanaged(utf8.GraphemeInfo) = .empty;
test "no_zwj: findRenderClusterInfo splits ZWJ sequences" {
var result_unicode: std.ArrayListUnmanaged(utf8.RenderClusterInfo) = .empty;
defer result_unicode.deinit(testing.allocator);
var result_no_zwj: std.ArrayListUnmanaged(utf8.GraphemeInfo) = .empty;
var result_no_zwj: std.ArrayListUnmanaged(utf8.RenderClusterInfo) = .empty;
defer result_no_zwj.deinit(testing.allocator);
const text = "Hi👩‍🚀Bye";
try utf8.findGraphemeInfo(testing.allocator, text, 4, false, .unicode, &result_unicode);
try utf8.findGraphemeInfo(testing.allocator, text, 4, false, .no_zwj, &result_no_zwj);
try utf8.findRenderClusterInfo(testing.allocator, text, 4, false, .unicode, &result_unicode);
try utf8.findRenderClusterInfo(testing.allocator, text, 4, false, .no_zwj, &result_no_zwj);
// unicode: 1 grapheme (the whole ZWJ sequence)
try testing.expectEqual(@as(usize, 1), result_unicode.items.len);
try testing.expectEqual(@as(u32, 2), result_unicode.items[0].width);
try testing.expectEqual(@as(u32, 2), result_unicode.items[0].width_cols);
// no_zwj: 2 graphemes (woman and rocket separately)
try testing.expectEqual(@as(usize, 2), result_no_zwj.items.len);
try testing.expectEqual(@as(u32, 2), result_no_zwj.items[0].width);
try testing.expectEqual(@as(u32, 2), result_no_zwj.items[1].width);
try testing.expectEqual(@as(u32, 2), result_no_zwj.items[0].width_cols);
try testing.expectEqual(@as(u32, 2), result_no_zwj.items[1].width_cols);
}
test "no_zwj: findWrapPosByWidth with ZWJ sequences" {
packages/native/src/tests/utf8_test.zig
+115 -115
View File
@@ -804,7 +804,7 @@ fn testLayoutBreaks(test_case: LayoutBreakTestCase, allocator: std.mem.Allocator
try testing.expectEqual(test_case.expected.len, breaks.items.len);
for (test_case.expected, 0..) |exp, i| {
try testing.expectEqual(exp, breaks.items[i].byte_offset);
try testing.expectEqual(exp, breaks.items[i].byte_start);
}
}
@@ -918,7 +918,7 @@ test "chunk layout scanner: break offset beyond 64KB" {
_ = try utf8.findChunkLayoutInfo(testing.allocator, buf, 4, true, .unicode, &breaks);
try testing.expectEqual(@as(usize, 1), breaks.items.len);
try testing.expectEqual(@as(u32, break_pos), breaks.items[0].byte_offset);
try testing.expectEqual(@as(u32, break_pos), breaks.items[0].byte_start);
}
// ============================================================================
@@ -1994,230 +1994,230 @@ test "calculateTextWidth: U+269B atom symbol should be width 2" {
}
// ============================================================================
// GRAPHEME INFO TESTS (for caching multi-byte graphemes and tabs)
// RENDER CLUSTER INFO TESTS (for caching multibyte clusters and tabs)
// ============================================================================
test "findGraphemeInfo: empty string" {
var result: std.ArrayListUnmanaged(utf8.GraphemeInfo) = .empty;
test "findRenderClusterInfo: empty string" {
var result: std.ArrayListUnmanaged(utf8.RenderClusterInfo) = .empty;
defer result.deinit(testing.allocator);
try utf8.findGraphemeInfo(testing.allocator, "", 4, false, .unicode, &result);
try utf8.findRenderClusterInfo(testing.allocator, "", 4, false, .unicode, &result);
try testing.expectEqual(@as(usize, 0), result.items.len);
}
test "findGraphemeInfo: ASCII-only returns empty" {
var result: std.ArrayListUnmanaged(utf8.GraphemeInfo) = .empty;
test "findRenderClusterInfo: ASCII-only returns empty" {
var result: std.ArrayListUnmanaged(utf8.RenderClusterInfo) = .empty;
defer result.deinit(testing.allocator);
try utf8.findGraphemeInfo(testing.allocator, "hello world", 4, true, .unicode, &result);
try utf8.findRenderClusterInfo(testing.allocator, "hello world", 4, true, .unicode, &result);
try testing.expectEqual(@as(usize, 0), result.items.len);
}
test "findGraphemeInfo: ASCII with tab" {
var result: std.ArrayListUnmanaged(utf8.GraphemeInfo) = .empty;
test "findRenderClusterInfo: ASCII with tab" {
var result: std.ArrayListUnmanaged(utf8.RenderClusterInfo) = .empty;
defer result.deinit(testing.allocator);
try utf8.findGraphemeInfo(testing.allocator, "hello\tworld", 4, false, .unicode, &result);
try utf8.findRenderClusterInfo(testing.allocator, "hello\tworld", 4, false, .unicode, &result);
// Should have one entry for the tab
try testing.expectEqual(@as(usize, 1), result.items.len);
try testing.expectEqual(@as(u32, 5), result.items[0].byte_offset);
try testing.expectEqual(@as(u32, 5), result.items[0].byte_start);
try testing.expectEqual(@as(u32, 1), result.items[0].byte_len);
try testing.expectEqual(@as(u32, 4), result.items[0].width);
try testing.expectEqual(@as(u32, 5), result.items[0].col_offset);
try testing.expectEqual(@as(u32, 4), result.items[0].width_cols);
try testing.expectEqual(@as(u32, 5), result.items[0].col_start);
}
test "findGraphemeInfo: multiple tabs" {
var result: std.ArrayListUnmanaged(utf8.GraphemeInfo) = .empty;
test "findRenderClusterInfo: multiple tabs" {
var result: std.ArrayListUnmanaged(utf8.RenderClusterInfo) = .empty;
defer result.deinit(testing.allocator);
try utf8.findGraphemeInfo(testing.allocator, "a\tb\tc", 4, false, .unicode, &result);
try utf8.findRenderClusterInfo(testing.allocator, "a\tb\tc", 4, false, .unicode, &result);
// Should have two entries for the tabs
try testing.expectEqual(@as(usize, 2), result.items.len);
// First tab at byte 1, col 1
try testing.expectEqual(@as(u32, 1), result.items[0].byte_offset);
try testing.expectEqual(@as(u32, 1), result.items[0].byte_start);
try testing.expectEqual(@as(u32, 1), result.items[0].byte_len);
try testing.expectEqual(@as(u32, 4), result.items[0].width);
try testing.expectEqual(@as(u32, 1), result.items[0].col_offset);
try testing.expectEqual(@as(u32, 4), result.items[0].width_cols);
try testing.expectEqual(@as(u32, 1), result.items[0].col_start);
// Second tab at byte 3, col 6 (1 + 4 + 1)
try testing.expectEqual(@as(u32, 3), result.items[1].byte_offset);
try testing.expectEqual(@as(u32, 3), result.items[1].byte_start);
try testing.expectEqual(@as(u32, 1), result.items[1].byte_len);
try testing.expectEqual(@as(u32, 4), result.items[1].width);
try testing.expectEqual(@as(u32, 6), result.items[1].col_offset);
try testing.expectEqual(@as(u32, 4), result.items[1].width_cols);
try testing.expectEqual(@as(u32, 6), result.items[1].col_start);
}
test "findGraphemeInfo: CJK characters" {
var result: std.ArrayListUnmanaged(utf8.GraphemeInfo) = .empty;
test "findRenderClusterInfo: CJK characters" {
var result: std.ArrayListUnmanaged(utf8.RenderClusterInfo) = .empty;
defer result.deinit(testing.allocator);
const text = "hello世界";
try utf8.findGraphemeInfo(testing.allocator, text, 4, false, .unicode, &result);
try utf8.findRenderClusterInfo(testing.allocator, text, 4, false, .unicode, &result);
// Should have two entries for the CJK characters
try testing.expectEqual(@as(usize, 2), result.items.len);
// 世 at byte 5
try testing.expectEqual(@as(u32, 5), result.items[0].byte_offset);
try testing.expectEqual(@as(u32, 5), result.items[0].byte_start);
try testing.expectEqual(@as(u32, 3), result.items[0].byte_len);
try testing.expectEqual(@as(u32, 2), result.items[0].width);
try testing.expectEqual(@as(u32, 5), result.items[0].col_offset);
try testing.expectEqual(@as(u32, 2), result.items[0].width_cols);
try testing.expectEqual(@as(u32, 5), result.items[0].col_start);
// 界 at byte 8
try testing.expectEqual(@as(u32, 8), result.items[1].byte_offset);
try testing.expectEqual(@as(u32, 8), result.items[1].byte_start);
try testing.expectEqual(@as(u32, 3), result.items[1].byte_len);
try testing.expectEqual(@as(u32, 2), result.items[1].width);
try testing.expectEqual(@as(u32, 7), result.items[1].col_offset);
try testing.expectEqual(@as(u32, 2), result.items[1].width_cols);
try testing.expectEqual(@as(u32, 7), result.items[1].col_start);
}
test "findGraphemeInfo: emoji with skin tone" {
var result: std.ArrayListUnmanaged(utf8.GraphemeInfo) = .empty;
test "findRenderClusterInfo: emoji with skin tone" {
var result: std.ArrayListUnmanaged(utf8.RenderClusterInfo) = .empty;
defer result.deinit(testing.allocator);
const text = "Hi👋🏿Bye"; // Hi + wave + dark skin tone + Bye
try utf8.findGraphemeInfo(testing.allocator, text, 4, false, .unicode, &result);
try utf8.findRenderClusterInfo(testing.allocator, text, 4, false, .unicode, &result);
// Should have one entry for the emoji cluster
try testing.expectEqual(@as(usize, 1), result.items.len);
try testing.expectEqual(@as(u32, 2), result.items[0].byte_offset);
try testing.expectEqual(@as(u32, 2), result.items[0].byte_start);
try testing.expectEqual(@as(u32, 8), result.items[0].byte_len); // 4 + 4 bytes
try testing.expectEqual(@as(u32, 2), result.items[0].width);
try testing.expectEqual(@as(u32, 2), result.items[0].col_offset);
try testing.expectEqual(@as(u32, 2), result.items[0].width_cols);
try testing.expectEqual(@as(u32, 2), result.items[0].col_start);
}
test "findGraphemeInfo: emoji with ZWJ" {
var result: std.ArrayListUnmanaged(utf8.GraphemeInfo) = .empty;
test "findRenderClusterInfo: emoji with ZWJ" {
var result: std.ArrayListUnmanaged(utf8.RenderClusterInfo) = .empty;
defer result.deinit(testing.allocator);
const text = "a👩‍🚀b"; // a + woman astronaut + b
try utf8.findGraphemeInfo(testing.allocator, text, 4, false, .unicode, &result);
try utf8.findRenderClusterInfo(testing.allocator, text, 4, false, .unicode, &result);
// Should have one entry for the emoji cluster
try testing.expectEqual(@as(usize, 1), result.items.len);
try testing.expectEqual(@as(u32, 1), result.items[0].byte_offset);
try testing.expectEqual(@as(u32, 2), result.items[0].width);
try testing.expectEqual(@as(u32, 1), result.items[0].col_offset);
try testing.expectEqual(@as(u32, 1), result.items[0].byte_start);
try testing.expectEqual(@as(u32, 2), result.items[0].width_cols);
try testing.expectEqual(@as(u32, 1), result.items[0].col_start);
}
test "findGraphemeInfo: combining mark" {
var result: std.ArrayListUnmanaged(utf8.GraphemeInfo) = .empty;
test "findRenderClusterInfo: combining mark" {
var result: std.ArrayListUnmanaged(utf8.RenderClusterInfo) = .empty;
defer result.deinit(testing.allocator);
const text = "cafe\u{0301}"; // café with combining acute
try utf8.findGraphemeInfo(testing.allocator, text, 4, false, .unicode, &result);
try utf8.findRenderClusterInfo(testing.allocator, text, 4, false, .unicode, &result);
// Should have one entry for e + combining mark
try testing.expectEqual(@as(usize, 1), result.items.len);
try testing.expectEqual(@as(u32, 3), result.items[0].byte_offset); // 'e' position
try testing.expectEqual(@as(u32, 3), result.items[0].byte_start); // 'e' position
try testing.expectEqual(@as(u32, 3), result.items[0].byte_len); // e (1 byte) + combining (2 bytes)
try testing.expectEqual(@as(u32, 1), result.items[0].width);
try testing.expectEqual(@as(u32, 3), result.items[0].col_offset);
try testing.expectEqual(@as(u32, 1), result.items[0].width_cols);
try testing.expectEqual(@as(u32, 3), result.items[0].col_start);
}
test "findGraphemeInfo: flag emoji" {
var result: std.ArrayListUnmanaged(utf8.GraphemeInfo) = .empty;
test "findRenderClusterInfo: flag emoji" {
var result: std.ArrayListUnmanaged(utf8.RenderClusterInfo) = .empty;
defer result.deinit(testing.allocator);
const text = "US🇺🇸"; // US + flag
try utf8.findGraphemeInfo(testing.allocator, text, 4, false, .unicode, &result);
try utf8.findRenderClusterInfo(testing.allocator, text, 4, false, .unicode, &result);
// Should have one entry for the flag (two regional indicators)
try testing.expectEqual(@as(usize, 1), result.items.len);
try testing.expectEqual(@as(u32, 2), result.items[0].byte_offset);
try testing.expectEqual(@as(u32, 2), result.items[0].byte_start);
try testing.expectEqual(@as(u32, 8), result.items[0].byte_len); // Two 4-byte chars
try testing.expectEqual(@as(u32, 2), result.items[0].width);
try testing.expectEqual(@as(u32, 2), result.items[0].col_offset);
try testing.expectEqual(@as(u32, 2), result.items[0].width_cols);
try testing.expectEqual(@as(u32, 2), result.items[0].col_start);
}
test "findGraphemeInfo: mixed content" {
var result: std.ArrayListUnmanaged(utf8.GraphemeInfo) = .empty;
test "findRenderClusterInfo: mixed content" {
var result: std.ArrayListUnmanaged(utf8.RenderClusterInfo) = .empty;
defer result.deinit(testing.allocator);
const text = "Hi\t世界!"; // Hi + tab + CJK + !
try utf8.findGraphemeInfo(testing.allocator, text, 4, false, .unicode, &result);
try utf8.findRenderClusterInfo(testing.allocator, text, 4, false, .unicode, &result);
// Should have three entries: tab, 世, 界
try testing.expectEqual(@as(usize, 3), result.items.len);
// Tab at byte 2, col 2
try testing.expectEqual(@as(u32, 2), result.items[0].byte_offset);
try testing.expectEqual(@as(u32, 2), result.items[0].byte_start);
try testing.expectEqual(@as(u32, 1), result.items[0].byte_len);
try testing.expectEqual(@as(u32, 4), result.items[0].width);
try testing.expectEqual(@as(u32, 2), result.items[0].col_offset);
try testing.expectEqual(@as(u32, 4), result.items[0].width_cols);
try testing.expectEqual(@as(u32, 2), result.items[0].col_start);
// 世 at byte 3, col 6
try testing.expectEqual(@as(u32, 3), result.items[1].byte_offset);
try testing.expectEqual(@as(u32, 3), result.items[1].byte_start);
try testing.expectEqual(@as(u32, 3), result.items[1].byte_len);
try testing.expectEqual(@as(u32, 2), result.items[1].width);
try testing.expectEqual(@as(u32, 6), result.items[1].col_offset);
try testing.expectEqual(@as(u32, 2), result.items[1].width_cols);
try testing.expectEqual(@as(u32, 6), result.items[1].col_start);
// 界 at byte 6, col 8
try testing.expectEqual(@as(u32, 6), result.items[2].byte_offset);
try testing.expectEqual(@as(u32, 6), result.items[2].byte_start);
try testing.expectEqual(@as(u32, 3), result.items[2].byte_len);
try testing.expectEqual(@as(u32, 2), result.items[2].width);
try testing.expectEqual(@as(u32, 8), result.items[2].col_offset);
try testing.expectEqual(@as(u32, 2), result.items[2].width_cols);
try testing.expectEqual(@as(u32, 8), result.items[2].col_start);
}
test "findGraphemeInfo: only ASCII letters no cache" {
var result: std.ArrayListUnmanaged(utf8.GraphemeInfo) = .empty;
test "findRenderClusterInfo: only ASCII letters no cache" {
var result: std.ArrayListUnmanaged(utf8.RenderClusterInfo) = .empty;
defer result.deinit(testing.allocator);
try utf8.findGraphemeInfo(testing.allocator, "abcdefghij", 4, false, .unicode, &result);
try utf8.findRenderClusterInfo(testing.allocator, "abcdefghij", 4, false, .unicode, &result);
// No special characters, should be empty
try testing.expectEqual(@as(usize, 0), result.items.len);
}
test "findGraphemeInfo: emoji with VS16" {
var result: std.ArrayListUnmanaged(utf8.GraphemeInfo) = .empty;
test "findRenderClusterInfo: emoji with VS16" {
var result: std.ArrayListUnmanaged(utf8.RenderClusterInfo) = .empty;
defer result.deinit(testing.allocator);
const text = "I ❤️ U"; // I + space + heart + VS16 + space + U
try utf8.findGraphemeInfo(testing.allocator, text, 4, false, .unicode, &result);
try utf8.findRenderClusterInfo(testing.allocator, text, 4, false, .unicode, &result);
// Should have one entry for the emoji cluster
try testing.expectEqual(@as(usize, 1), result.items.len);
try testing.expectEqual(@as(u32, 2), result.items[0].byte_offset);
try testing.expectEqual(@as(u32, 2), result.items[0].width);
try testing.expectEqual(@as(u32, 2), result.items[0].col_offset);
try testing.expectEqual(@as(u32, 2), result.items[0].byte_start);
try testing.expectEqual(@as(u32, 2), result.items[0].width_cols);
try testing.expectEqual(@as(u32, 2), result.items[0].col_start);
}
test "findGraphemeInfo: realistic text" {
var result: std.ArrayListUnmanaged(utf8.GraphemeInfo) = .empty;
test "findRenderClusterInfo: realistic text" {
var result: std.ArrayListUnmanaged(utf8.RenderClusterInfo) = .empty;
defer result.deinit(testing.allocator);
const text = "function test() {\n\tconst 世界 = 10;\n}";
try utf8.findGraphemeInfo(testing.allocator, text, 4, false, .unicode, &result);
try utf8.findRenderClusterInfo(testing.allocator, text, 4, false, .unicode, &result);
// Should have entries for: tab, 世, 界
try testing.expectEqual(@as(usize, 3), result.items.len);
}
test "findGraphemeInfo: hiragana" {
var result: std.ArrayListUnmanaged(utf8.GraphemeInfo) = .empty;
test "findRenderClusterInfo: hiragana" {
var result: std.ArrayListUnmanaged(utf8.RenderClusterInfo) = .empty;
defer result.deinit(testing.allocator);
const text = "こんにちは";
try utf8.findGraphemeInfo(testing.allocator, text, 4, false, .unicode, &result);
try utf8.findRenderClusterInfo(testing.allocator, text, 4, false, .unicode, &result);
// Should have 5 entries (each hiragana is 3 bytes, width 2)
try testing.expectEqual(@as(usize, 5), result.items.len);
// Check first character
try testing.expectEqual(@as(u32, 0), result.items[0].byte_offset);
try testing.expectEqual(@as(u32, 0), result.items[0].byte_start);
try testing.expectEqual(@as(u32, 3), result.items[0].byte_len);
try testing.expectEqual(@as(u32, 2), result.items[0].width);
try testing.expectEqual(@as(u32, 2), result.items[0].width_cols);
}
test "findGraphemeInfo: at SIMD boundary" {
var result: std.ArrayListUnmanaged(utf8.GraphemeInfo) = .empty;
test "findRenderClusterInfo: at SIMD boundary" {
var result: std.ArrayListUnmanaged(utf8.RenderClusterInfo) = .empty;
defer result.deinit(testing.allocator);
// Create text with multibyte char near SIMD boundary (16 bytes)
@@ -2226,22 +2226,22 @@ test "findGraphemeInfo: at SIMD boundary" {
const cjk = "世";
@memcpy(buf[14..17], cjk); // Place CJK char at boundary
try utf8.findGraphemeInfo(testing.allocator, &buf, 4, false, .unicode, &result);
try utf8.findRenderClusterInfo(testing.allocator, &buf, 4, false, .unicode, &result);
// Should find the CJK character
var found = false;
for (result.items) |g| {
if (g.byte_offset == 14) {
if (g.byte_start == 14) {
found = true;
try testing.expectEqual(@as(u32, 3), g.byte_len);
try testing.expectEqual(@as(u32, 2), g.width);
try testing.expectEqual(@as(u32, 2), g.width_cols);
break;
}
}
try testing.expect(found);
}
test "findGraphemeInfo: long grapheme metadata exceeds u8 ranges" {
test "findRenderClusterInfo: long render-cluster metadata exceeds u8 ranges" {
var text: std.ArrayListUnmanaged(u8) = .empty;
defer text.deinit(testing.allocator);
@@ -2255,14 +2255,14 @@ test "findGraphemeInfo: long grapheme metadata exceeds u8 ranges" {
.{ .method = .unicode, .width = 2 },
.{ .method = .wcwidth, .width = 260 },
}) |case| {
var result: std.ArrayListUnmanaged(utf8.GraphemeInfo) = .empty;
var result: std.ArrayListUnmanaged(utf8.RenderClusterInfo) = .empty;
defer result.deinit(testing.allocator);
try utf8.findGraphemeInfo(testing.allocator, text.items, 4, false, case.method, &result);
try utf8.findRenderClusterInfo(testing.allocator, text.items, 4, false, case.method, &result);
try testing.expectEqual(@as(usize, 1), result.items.len);
try testing.expectEqual(@as(u32, 0), result.items[0].byte_offset);
try testing.expectEqual(@as(u32, 0), result.items[0].byte_start);
try testing.expectEqual(@as(u32, @intCast(text.items.len)), result.items[0].byte_len);
try testing.expectEqual(case.width, result.items[0].width);
try testing.expectEqual(case.width, result.items[0].width_cols);
}
}
@@ -3521,7 +3521,7 @@ test "calculateTextWidth: validate against unicode-width-map.zon" {
try testing.expectEqual(@as(usize, 0), failures);
}
test "findGraphemeInfo: comprehensive multilingual text" {
test "findRenderClusterInfo: comprehensive multilingual text" {
const text =
\\# The Celestial Journey of संस्कृति 🌟🔮✨
\\In the beginning, there was नमस्ते 🙏 and the ancient wisdom of the ॐ symbol echoing through dimensions. The travelers 🧑‍🚀👨‍🚀👩‍🚀 embarked on their quest through the cosmos, guided by the mysterious རྒྱ་མཚོ and the luminous 🌈🦄🧚‍♀️ beings of light. They encountered the great देवनागरी scribes who wrote in flowing अक्षर characters, documenting everything in their sacred texts 📜📖✍️.
@@ -3547,23 +3547,23 @@ test "findGraphemeInfo: comprehensive multilingual text" {
const expected_width = utf8.calculateTextWidth(text, 4, false, .unicode);
var result: std.ArrayListUnmanaged(utf8.GraphemeInfo) = .empty;
var result: std.ArrayListUnmanaged(utf8.RenderClusterInfo) = .empty;
defer result.deinit(testing.allocator);
try utf8.findGraphemeInfo(testing.allocator, text, 4, false, .unicode, &result);
try utf8.findRenderClusterInfo(testing.allocator, text, 4, false, .unicode, &result);
try testing.expect(result.items.len > 0);
var prev_end_byte: usize = 0;
for (result.items) |g| {
try testing.expect(g.byte_offset >= prev_end_byte);
try testing.expect(g.byte_start >= prev_end_byte);
const text_before = text[0..g.byte_offset];
const text_before = text[0..g.byte_start];
const expected_col = utf8.calculateTextWidth(text_before, 4, false, .unicode);
try testing.expectEqual(expected_col, g.col_offset);
try testing.expectEqual(expected_col, g.col_start);
prev_end_byte = g.byte_offset + g.byte_len;
prev_end_byte = g.byte_start + g.byte_len;
}
const final_computed_width = utf8.calculateTextWidth(text, 4, false, .unicode);
@@ -3682,29 +3682,29 @@ test "Thai: wrap by width with tone marks" {
try testing.expectEqual(@as(u32, 3), result3.columns_used);
}
test "Thai: grapheme info for combining marks" {
test "Thai: render-cluster info for combining marks" {
const text = "กี่";
var result: std.ArrayListUnmanaged(utf8.GraphemeInfo) = .empty;
var result: std.ArrayListUnmanaged(utf8.RenderClusterInfo) = .empty;
defer result.deinit(testing.allocator);
try utf8.findGraphemeInfo(testing.allocator, text, 4, false, .unicode, &result);
try utf8.findRenderClusterInfo(testing.allocator, text, 4, false, .unicode, &result);
try testing.expectEqual(@as(usize, 1), result.items.len);
try testing.expectEqual(@as(u32, 1), result.items[0].width);
try testing.expectEqual(@as(u32, 1), result.items[0].width_cols);
}
test "Thai: grapheme info for word with combining marks" {
test "Thai: render-cluster info for word with combining marks" {
const text = "คือ";
var result: std.ArrayListUnmanaged(utf8.GraphemeInfo) = .empty;
var result: std.ArrayListUnmanaged(utf8.RenderClusterInfo) = .empty;
defer result.deinit(testing.allocator);
try utf8.findGraphemeInfo(testing.allocator, text, 4, false, .unicode, &result);
try utf8.findRenderClusterInfo(testing.allocator, text, 4, false, .unicode, &result);
try testing.expectEqual(@as(usize, 2), result.items.len);
try testing.expectEqual(@as(u32, 1), result.items[0].width);
try testing.expectEqual(@as(u32, 1), result.items[1].width);
try testing.expectEqual(@as(u32, 1), result.items[0].width_cols);
try testing.expectEqual(@as(u32, 1), result.items[1].width_cols);
}
test "Thai: mixed Thai and ASCII" {
@@ -3744,11 +3744,11 @@ test "Thai: ว่ wcwidth vs unicode mode comparison" {
test "Thai: ว่ is a single grapheme cluster" {
const text = "ว่";
var result: std.ArrayListUnmanaged(utf8.GraphemeInfo) = .empty;
var result: std.ArrayListUnmanaged(utf8.RenderClusterInfo) = .empty;
defer result.deinit(testing.allocator);
try utf8.findGraphemeInfo(testing.allocator, text, 4, false, .unicode, &result);
try utf8.findRenderClusterInfo(testing.allocator, text, 4, false, .unicode, &result);
try testing.expectEqual(@as(usize, 1), result.items.len);
try testing.expectEqual(@as(u32, 1), result.items[0].width);
try testing.expectEqual(@as(u32, 1), result.items[0].width_cols);
}
packages/native/src/tests/utf8_wcwidth_cursor_test.zig
+15 -15
View File
@@ -199,45 +199,45 @@ test "wcwidth: mixed content with cursor movement" {
try testing.expectEqual(@as(u32, 1), r_a.?.width);
}
test "wcwidth: findGraphemeInfo with emoji" {
var result: std.ArrayListUnmanaged(utf8.GraphemeInfo) = .empty;
test "wcwidth: findRenderClusterInfo with emoji" {
var result: std.ArrayListUnmanaged(utf8.RenderClusterInfo) = .empty;
defer result.deinit(testing.allocator);
const text = "👋🏿"; // Wave + skin tone modifier
try utf8.findGraphemeInfo(testing.allocator, text, 4, false, .wcwidth, &result);
try utf8.findRenderClusterInfo(testing.allocator, text, 4, false, .wcwidth, &result);
try testing.expectEqual(@as(usize, 1), result.items.len);
try testing.expectEqual(@as(u32, 0), result.items[0].byte_offset);
try testing.expectEqual(@as(u32, 0), result.items[0].byte_start);
try testing.expectEqual(@as(u32, 8), result.items[0].byte_len);
try testing.expectEqual(@as(u32, 4), result.items[0].width);
try testing.expectEqual(@as(u32, 4), result.items[0].width_cols);
}
test "wcwidth: findGraphemeInfo with ZWJ sequence" {
var result: std.ArrayListUnmanaged(utf8.GraphemeInfo) = .empty;
test "wcwidth: findRenderClusterInfo with ZWJ sequence" {
var result: std.ArrayListUnmanaged(utf8.RenderClusterInfo) = .empty;
defer result.deinit(testing.allocator);
const text = "👩‍🚀"; // Woman + ZWJ + Rocket
try utf8.findGraphemeInfo(testing.allocator, text, 4, false, .wcwidth, &result);
try utf8.findRenderClusterInfo(testing.allocator, text, 4, false, .wcwidth, &result);
try testing.expectEqual(@as(usize, 1), result.items.len);
try testing.expectEqual(@as(u32, 0), result.items[0].byte_offset);
try testing.expectEqual(@as(u32, 0), result.items[0].byte_start);
try testing.expectEqual(@as(u32, 11), result.items[0].byte_len);
try testing.expectEqual(@as(u32, 4), result.items[0].width);
try testing.expectEqual(@as(u32, 4), result.items[0].width_cols);
}
test "wcwidth: findGraphemeInfo with combining marks" {
var result: std.ArrayListUnmanaged(utf8.GraphemeInfo) = .empty;
test "wcwidth: findRenderClusterInfo with combining marks" {
var result: std.ArrayListUnmanaged(utf8.RenderClusterInfo) = .empty;
defer result.deinit(testing.allocator);
const text = "e\u{0301}"; // e + combining acute
try utf8.findGraphemeInfo(testing.allocator, text, 4, false, .wcwidth, &result);
try utf8.findRenderClusterInfo(testing.allocator, text, 4, false, .wcwidth, &result);
try testing.expectEqual(@as(usize, 1), result.items.len);
try testing.expectEqual(@as(u32, 0), result.items[0].byte_offset);
try testing.expectEqual(@as(u32, 0), result.items[0].byte_start);
try testing.expectEqual(@as(u32, 3), result.items[0].byte_len);
try testing.expectEqual(@as(u32, 1), result.items[0].width);
try testing.expectEqual(@as(u32, 1), result.items[0].width_cols);
}
test "wcwidth: tab width handling" {
packages/native/src/tests/utf8_wcwidth_test.zig
+51 -51
View File
@@ -2,139 +2,139 @@ const std = @import("std");
const testing = std.testing;
const utf8 = @import("../utf8.zig");
test "findGraphemeInfo wcwidth: empty string" {
var result: std.ArrayListUnmanaged(utf8.GraphemeInfo) = .empty;
test "findRenderClusterInfo wcwidth: empty string" {
var result: std.ArrayListUnmanaged(utf8.RenderClusterInfo) = .empty;
defer result.deinit(testing.allocator);
try utf8.findGraphemeInfo(testing.allocator, "", 4, false, .wcwidth, &result);
try utf8.findRenderClusterInfo(testing.allocator, "", 4, false, .wcwidth, &result);
try testing.expectEqual(@as(usize, 0), result.items.len);
}
test "findGraphemeInfo wcwidth: ASCII-only returns empty" {
var result: std.ArrayListUnmanaged(utf8.GraphemeInfo) = .empty;
test "findRenderClusterInfo wcwidth: ASCII-only returns empty" {
var result: std.ArrayListUnmanaged(utf8.RenderClusterInfo) = .empty;
defer result.deinit(testing.allocator);
try utf8.findGraphemeInfo(testing.allocator, "hello world", 4, true, .wcwidth, &result);
try utf8.findRenderClusterInfo(testing.allocator, "hello world", 4, true, .wcwidth, &result);
try testing.expectEqual(@as(usize, 0), result.items.len);
}
test "findGraphemeInfo wcwidth: ASCII with tab" {
var result: std.ArrayListUnmanaged(utf8.GraphemeInfo) = .empty;
test "findRenderClusterInfo wcwidth: ASCII with tab" {
var result: std.ArrayListUnmanaged(utf8.RenderClusterInfo) = .empty;
defer result.deinit(testing.allocator);
try utf8.findGraphemeInfo(testing.allocator, "hello\tworld", 4, false, .wcwidth, &result);
try utf8.findRenderClusterInfo(testing.allocator, "hello\tworld", 4, false, .wcwidth, &result);
// Should have one entry for the tab
try testing.expectEqual(@as(usize, 1), result.items.len);
try testing.expectEqual(@as(u32, 5), result.items[0].byte_offset);
try testing.expectEqual(@as(u32, 5), result.items[0].byte_start);
try testing.expectEqual(@as(u32, 1), result.items[0].byte_len);
try testing.expectEqual(@as(u32, 4), result.items[0].width);
try testing.expectEqual(@as(u32, 5), result.items[0].col_offset);
try testing.expectEqual(@as(u32, 4), result.items[0].width_cols);
try testing.expectEqual(@as(u32, 5), result.items[0].col_start);
}
test "findGraphemeInfo wcwidth: CJK characters" {
var result: std.ArrayListUnmanaged(utf8.GraphemeInfo) = .empty;
test "findRenderClusterInfo wcwidth: CJK characters" {
var result: std.ArrayListUnmanaged(utf8.RenderClusterInfo) = .empty;
defer result.deinit(testing.allocator);
const text = "hello世界";
try utf8.findGraphemeInfo(testing.allocator, text, 4, false, .wcwidth, &result);
try utf8.findRenderClusterInfo(testing.allocator, text, 4, false, .wcwidth, &result);
// Should have two entries for the CJK characters (each codepoint separately)
try testing.expectEqual(@as(usize, 2), result.items.len);
// First CJK char '世' at byte 5
try testing.expectEqual(@as(u32, 5), result.items[0].byte_offset);
try testing.expectEqual(@as(u32, 5), result.items[0].byte_start);
try testing.expectEqual(@as(u32, 3), result.items[0].byte_len);
try testing.expectEqual(@as(u32, 2), result.items[0].width);
try testing.expectEqual(@as(u32, 5), result.items[0].col_offset);
try testing.expectEqual(@as(u32, 2), result.items[0].width_cols);
try testing.expectEqual(@as(u32, 5), result.items[0].col_start);
// Second CJK char '界' at byte 8
try testing.expectEqual(@as(u32, 8), result.items[1].byte_offset);
try testing.expectEqual(@as(u32, 8), result.items[1].byte_start);
try testing.expectEqual(@as(u32, 3), result.items[1].byte_len);
try testing.expectEqual(@as(u32, 2), result.items[1].width);
try testing.expectEqual(@as(u32, 7), result.items[1].col_offset);
try testing.expectEqual(@as(u32, 2), result.items[1].width_cols);
try testing.expectEqual(@as(u32, 7), result.items[1].col_start);
}
test "findGraphemeInfo wcwidth: emoji with skin tone - single grapheme cluster" {
var result: std.ArrayListUnmanaged(utf8.GraphemeInfo) = .empty;
test "findRenderClusterInfo wcwidth: emoji with skin tone - single grapheme cluster" {
var result: std.ArrayListUnmanaged(utf8.RenderClusterInfo) = .empty;
defer result.deinit(testing.allocator);
const text = "👋🏿"; // Wave + skin tone modifier
try utf8.findGraphemeInfo(testing.allocator, text, 4, false, .wcwidth, &result);
try utf8.findRenderClusterInfo(testing.allocator, text, 4, false, .wcwidth, &result);
try testing.expectEqual(@as(usize, 1), result.items.len);
try testing.expectEqual(@as(u32, 0), result.items[0].byte_offset);
try testing.expectEqual(@as(u32, 0), result.items[0].byte_start);
try testing.expectEqual(@as(u32, 8), result.items[0].byte_len);
try testing.expectEqual(@as(u32, 4), result.items[0].width);
try testing.expectEqual(@as(u32, 4), result.items[0].width_cols);
}
test "findGraphemeInfo wcwidth: emoji with ZWJ - single grapheme cluster" {
var result: std.ArrayListUnmanaged(utf8.GraphemeInfo) = .empty;
test "findRenderClusterInfo wcwidth: emoji with ZWJ - single grapheme cluster" {
var result: std.ArrayListUnmanaged(utf8.RenderClusterInfo) = .empty;
defer result.deinit(testing.allocator);
const text = "👩‍🚀"; // Woman + ZWJ + Rocket (11 bytes total)
try utf8.findGraphemeInfo(testing.allocator, text, 4, false, .wcwidth, &result);
try utf8.findRenderClusterInfo(testing.allocator, text, 4, false, .wcwidth, &result);
try testing.expectEqual(@as(usize, 1), result.items.len);
try testing.expectEqual(@as(u32, 11), result.items[0].byte_len);
try testing.expectEqual(@as(u32, 4), result.items[0].width);
try testing.expectEqual(@as(u32, 4), result.items[0].width_cols);
}
test "findGraphemeInfo wcwidth: combining mark - part of base grapheme" {
var result: std.ArrayListUnmanaged(utf8.GraphemeInfo) = .empty;
test "findRenderClusterInfo wcwidth: combining mark - part of base grapheme" {
var result: std.ArrayListUnmanaged(utf8.RenderClusterInfo) = .empty;
defer result.deinit(testing.allocator);
const text = "e\u{0301}test"; // e + combining acute accent + test
try utf8.findGraphemeInfo(testing.allocator, text, 4, false, .wcwidth, &result);
try utf8.findRenderClusterInfo(testing.allocator, text, 4, false, .wcwidth, &result);
try testing.expectEqual(@as(usize, 1), result.items.len);
try testing.expectEqual(@as(u32, 0), result.items[0].byte_offset);
try testing.expectEqual(@as(u32, 0), result.items[0].byte_start);
try testing.expectEqual(@as(u32, 3), result.items[0].byte_len);
try testing.expectEqual(@as(u32, 1), result.items[0].width);
try testing.expectEqual(@as(u32, 1), result.items[0].width_cols);
}
test "findGraphemeInfo wcwidth vs unicode: emoji with skin tone" {
var result_wcwidth: std.ArrayListUnmanaged(utf8.GraphemeInfo) = .empty;
test "findRenderClusterInfo wcwidth vs unicode: emoji with skin tone" {
var result_wcwidth: std.ArrayListUnmanaged(utf8.RenderClusterInfo) = .empty;
defer result_wcwidth.deinit(testing.allocator);
var result_unicode: std.ArrayListUnmanaged(utf8.GraphemeInfo) = .empty;
var result_unicode: std.ArrayListUnmanaged(utf8.RenderClusterInfo) = .empty;
defer result_unicode.deinit(testing.allocator);
const text = "Hi👋🏿Bye";
try utf8.findGraphemeInfo(testing.allocator, text, 4, false, .wcwidth, &result_wcwidth);
try utf8.findGraphemeInfo(testing.allocator, text, 4, false, .unicode, &result_unicode);
try utf8.findRenderClusterInfo(testing.allocator, text, 4, false, .wcwidth, &result_wcwidth);
try utf8.findRenderClusterInfo(testing.allocator, text, 4, false, .unicode, &result_unicode);
try testing.expectEqual(@as(usize, 1), result_wcwidth.items.len);
try testing.expectEqual(@as(usize, 1), result_unicode.items.len);
try testing.expectEqual(@as(u32, 2), result_wcwidth.items[0].byte_offset);
try testing.expectEqual(@as(u32, 2), result_wcwidth.items[0].byte_start);
try testing.expectEqual(@as(u32, 8), result_wcwidth.items[0].byte_len);
try testing.expectEqual(@as(u32, 2), result_unicode.items[0].byte_offset);
try testing.expectEqual(@as(u32, 2), result_unicode.items[0].byte_start);
try testing.expectEqual(@as(u32, 8), result_unicode.items[0].byte_len);
try testing.expectEqual(@as(u32, 4), result_wcwidth.items[0].width);
try testing.expectEqual(@as(u32, 2), result_unicode.items[0].width);
try testing.expectEqual(@as(u32, 4), result_wcwidth.items[0].width_cols);
try testing.expectEqual(@as(u32, 2), result_unicode.items[0].width_cols);
}
test "findGraphemeInfo wcwidth vs unicode: flag emoji" {
var result_wcwidth: std.ArrayListUnmanaged(utf8.GraphemeInfo) = .empty;
test "findRenderClusterInfo wcwidth vs unicode: flag emoji" {
var result_wcwidth: std.ArrayListUnmanaged(utf8.RenderClusterInfo) = .empty;
defer result_wcwidth.deinit(testing.allocator);
var result_unicode: std.ArrayListUnmanaged(utf8.GraphemeInfo) = .empty;
var result_unicode: std.ArrayListUnmanaged(utf8.RenderClusterInfo) = .empty;
defer result_unicode.deinit(testing.allocator);
const text = "🇺🇸"; // US flag (two regional indicators)
try utf8.findGraphemeInfo(testing.allocator, text, 4, false, .wcwidth, &result_wcwidth);
try utf8.findGraphemeInfo(testing.allocator, text, 4, false, .unicode, &result_unicode);
try utf8.findRenderClusterInfo(testing.allocator, text, 4, false, .wcwidth, &result_wcwidth);
try utf8.findRenderClusterInfo(testing.allocator, text, 4, false, .unicode, &result_unicode);
try testing.expectEqual(@as(usize, 1), result_wcwidth.items.len);
try testing.expectEqual(@as(usize, 1), result_unicode.items.len);
try testing.expectEqual(@as(u32, 2), result_wcwidth.items[0].width);
try testing.expectEqual(@as(u32, 2), result_unicode.items[0].width);
try testing.expectEqual(@as(u32, 2), result_wcwidth.items[0].width_cols);
try testing.expectEqual(@as(u32, 2), result_unicode.items[0].width_cols);
}
// ============================================================================
packages/native/src/text-buffer-iterators.zig
+8 -8
View File
@@ -88,7 +88,7 @@ pub fn walkLinesAndSegments(
if (seg.asText()) |chunk| {
walk_ctx.seg_callback(walk_ctx.user_ctx, walk_ctx.current_line_idx, chunk, walk_ctx.chunk_idx_in_line);
walk_ctx.chunk_idx_in_line += 1;
walk_ctx.line_width_cols += chunk.width;
walk_ctx.line_width_cols += chunk.width_cols;
} else if (seg.isBreak()) {
walk_ctx.line_callback(walk_ctx.user_ctx, .{
.line_idx = walk_ctx.current_line_idx,
@@ -140,12 +140,12 @@ pub fn getLineCount(rope: *const UnifiedRope) u32 {
pub fn getMaxLineWidth(rope: *const UnifiedRope) u32 {
const metrics = rope.root.metrics();
return metrics.custom.max_line_width;
return metrics.custom.max_line_width_cols;
}
pub fn getTotalWidth(rope: *const UnifiedRope) u32 {
const metrics = rope.root.metrics();
return metrics.custom.total_width;
return metrics.custom.total_width_cols;
}
/// Optimized O(1) implementation using linestart marker lookups
@@ -249,7 +249,7 @@ fn getTextUnitBoundsAt(rope: *UnifiedRope, mem_registry: *const MemRegistry, row
const seg = rope.get(seg_idx) orelse break;
if (seg.isBreak() or seg.isLineStart()) break;
if (seg.asText()) |chunk| {
const next_cols = cols_before + chunk.width;
const next_cols = cols_before + chunk.width_cols;
if (col < next_cols) {
const local_col: u32 = col - cols_before;
const bytes = chunk.getBytes(mem_registry);
@@ -301,7 +301,7 @@ pub fn getPrevGraphemeWidth(rope: *UnifiedRope, mem_registry: *const MemRegistry
const seg = rope.get(seg_idx) orelse break;
if (seg.isBreak() or seg.isLineStart()) break;
if (seg.asText()) |chunk| {
const next_cols = cols_before + chunk.width;
const next_cols = cols_before + chunk.width_cols;
if (clamped_col <= next_cols) {
if (clamped_col == cols_before and prev_chunk != null) {
@@ -401,7 +401,7 @@ pub fn extractTextBetweenOffsets(
const ctx = @as(*@This(), @ptrCast(@alignCast(ctx_ptr)));
const chunk_start_offset = ctx.col_offset.*;
const chunk_end_offset = chunk_start_offset + chunk.width;
const chunk_end_offset = chunk_start_offset + chunk.width_cols;
// Skip chunk if it's entirely outside range
if (chunk_end_offset <= ctx.start or chunk_start_offset >= ctx.end) {
@@ -413,7 +413,7 @@ pub fn extractTextBetweenOffsets(
const is_ascii_only = (chunk.flags & TextChunk.Flags.ASCII_ONLY) != 0;
const local_start_col: u32 = if (ctx.start > chunk_start_offset) ctx.start - chunk_start_offset else 0;
const local_end_col: u32 = @min(ctx.end - chunk_start_offset, chunk.width);
const local_end_col: u32 = @min(ctx.end - chunk_start_offset, chunk.width_cols);
var byte_start: u32 = 0;
var byte_end: u32 = @intCast(chunk_bytes.len);
@@ -423,7 +423,7 @@ pub fn extractTextBetweenOffsets(
byte_start = start_result.byte_offset;
}
if (local_end_col < chunk.width) {
if (local_end_col < chunk.width_cols) {
const end_result = utf8.findGraphemePosByWidth(chunk_bytes, local_end_col, ctx.tab_width, is_ascii_only, true, ctx.width_method);
byte_end = end_result.byte_offset;
}
packages/native/src/text-buffer-segment.zig
+43 -43
View File
@@ -25,7 +25,7 @@ pub const WrapMode = enum {
word,
};
pub const GraphemeInfo = utf8.GraphemeInfo;
pub const RenderClusterInfo = utf8.RenderClusterInfo;
pub const ChunkLayoutInfo = utf8.ChunkLayoutInfo;
const CachedMeasure = struct {
@@ -37,10 +37,10 @@ const CachedMeasure = struct {
};
pub const TextChunkColdState = struct {
graphemes: ?[]GraphemeInfo = null,
graphemes_capacity: usize = 0,
graphemes_tab_width: ?u8 = null,
graphemes_width_method: ?utf8.WidthMethod = null,
render_clusters: ?[]RenderClusterInfo = null,
render_clusters_capacity: usize = 0,
render_clusters_tab_width: ?u8 = null,
render_clusters_width_method: ?utf8.WidthMethod = null,
wrap_breaks: ?[]utf8.LayoutWrapBreak = null,
wrap_breaks_capacity: usize = 0,
wrap_breaks_tab_width: ?u8 = null,
@@ -59,7 +59,7 @@ pub const TextChunk = struct {
mem_id: u8,
byte_start: u32,
byte_end: u32,
width: u32,
width_cols: u32,
flags: u8 = 0,
cold: ?*TextChunkColdState = null,
@@ -76,12 +76,12 @@ pub const TextChunk = struct {
.mem_id = 0,
.byte_start = 0,
.byte_end = 0,
.width = 0,
.width_cols = 0,
};
}
pub fn is_empty(self: *const TextChunk) bool {
return self.width == 0;
return self.width_cols == 0;
}
pub fn getBytes(self: *const TextChunk, mem_registry: *const MemRegistry) []const u8 {
@@ -148,51 +148,51 @@ pub const TextChunk = struct {
};
}
/// Lazily compute and cache grapheme info for this chunk
/// Lazily compute and cache sparse render-cluster metadata for this chunk
/// Returned storage is arena-owned until reset, but a lookup with different
/// tab-dependent settings may overwrite it.
/// For ASCII-only chunks, returns an empty slice (sentinel)
/// For mixed chunks, returns only multibyte (non-ASCII) graphemes and tabs with their column offsets
pub fn getGraphemes(
/// For mixed chunks, returns only multibyte render clusters and tabs with their cell-column starts
pub fn getRenderClusters(
self: *const TextChunk,
allocator: Allocator,
mem_registry: *const MemRegistry,
tabwidth: u8,
width_method: utf8.WidthMethod,
) TextBufferError![]const GraphemeInfo {
if (self.isAsciiOnly()) return &[_]GraphemeInfo{};
) TextBufferError![]const RenderClusterInfo {
if (self.isAsciiOnly()) return &[_]RenderClusterInfo{};
const cold = try self.getOrCreateCold(allocator);
if (cold.graphemes) |cached| {
if (cold.graphemes_tab_width == tabwidth and cold.graphemes_width_method == width_method) {
if (cold.render_clusters) |cached| {
if (cold.render_clusters_tab_width == tabwidth and cold.render_clusters_width_method == width_method) {
return cached;
}
var reusable: std.ArrayListUnmanaged(GraphemeInfo) = .{
var reusable: std.ArrayListUnmanaged(RenderClusterInfo) = .{
.items = cached,
.capacity = cold.graphemes_capacity,
.capacity = cold.render_clusters_capacity,
};
reusable.clearRetainingCapacity();
try utf8.findGraphemeInfo(allocator, self.getBytes(mem_registry), tabwidth, self.isAsciiOnly(), width_method, &reusable);
cold.graphemes = reusable.items;
cold.graphemes_capacity = reusable.capacity;
cold.graphemes_tab_width = tabwidth;
cold.graphemes_width_method = width_method;
try utf8.findRenderClusterInfo(allocator, self.getBytes(mem_registry), tabwidth, self.isAsciiOnly(), width_method, &reusable);
cold.render_clusters = reusable.items;
cold.render_clusters_capacity = reusable.capacity;
cold.render_clusters_tab_width = tabwidth;
cold.render_clusters_width_method = width_method;
return reusable.items;
}
const chunk_bytes = self.getBytes(mem_registry);
var grapheme_list: std.ArrayListUnmanaged(GraphemeInfo) = .empty;
errdefer grapheme_list.deinit(allocator);
var render_cluster_list: std.ArrayListUnmanaged(RenderClusterInfo) = .empty;
errdefer render_cluster_list.deinit(allocator);
try utf8.findGraphemeInfo(allocator, chunk_bytes, tabwidth, self.isAsciiOnly(), width_method, &grapheme_list);
try utf8.findRenderClusterInfo(allocator, chunk_bytes, tabwidth, self.isAsciiOnly(), width_method, &render_cluster_list);
cold.graphemes = grapheme_list.items;
cold.graphemes_capacity = grapheme_list.capacity;
cold.graphemes_tab_width = tabwidth;
cold.graphemes_width_method = width_method;
return grapheme_list.items;
cold.render_clusters = render_cluster_list.items;
cold.render_clusters_capacity = render_cluster_list.capacity;
cold.render_clusters_tab_width = tabwidth;
cold.render_clusters_width_method = width_method;
return render_cluster_list.items;
}
/// Lazily compute and cache direct byte/column wrap metadata for this chunk.
@@ -288,30 +288,30 @@ pub const Segment = union(enum) {
/// Metrics for aggregation in the rope tree
/// These enable O(log n) row/col coordinate mapping and efficient line queries
pub const Metrics = struct {
total_width: u32 = 0,
total_width_cols: u32 = 0,
total_bytes: u32 = 0,
linestart_count: u32 = 0,
newline_count: u32 = 0,
max_line_width: u32 = 0,
max_line_width_cols: u32 = 0,
/// Whether all text segments in subtree are ASCII-only (for fast wrapping paths)
ascii_only: bool = true,
pub fn add(self: *Metrics, other: Metrics) void {
self.total_width += other.total_width;
self.total_width_cols += other.total_width_cols;
self.total_bytes += other.total_bytes;
self.linestart_count += other.linestart_count;
self.newline_count += other.newline_count;
self.max_line_width = @max(self.max_line_width, other.max_line_width);
self.max_line_width_cols = @max(self.max_line_width_cols, other.max_line_width_cols);
self.ascii_only = self.ascii_only and other.ascii_only;
}
/// Get the balancing weight for the rope
/// We use total_width + newline_count to give each break a weight of 1
/// We use total_width_cols + newline_count to give each break a weight of 1
/// This eliminates boundary ambiguity in coordinate/offset conversions
pub fn weight(self: *const Metrics) u32 {
return self.total_width + self.newline_count;
return self.total_width_cols + self.newline_count;
}
};
@@ -322,28 +322,28 @@ pub const Segment = union(enum) {
const is_ascii = (chunk.flags & TextChunk.Flags.ASCII_ONLY) != 0;
const byte_len = chunk.byte_end - chunk.byte_start;
break :blk Metrics{
.total_width = chunk.width,
.total_width_cols = chunk.width_cols,
.total_bytes = byte_len,
.linestart_count = 0,
.newline_count = 0,
.max_line_width = chunk.width,
.max_line_width_cols = chunk.width_cols,
.ascii_only = is_ascii,
};
},
.brk => Metrics{
.total_width = 0,
.total_width_cols = 0,
.total_bytes = 0,
.linestart_count = 0,
.newline_count = 1,
.max_line_width = 0,
.max_line_width_cols = 0,
.ascii_only = true,
},
.linestart => Metrics{
.total_width = 0,
.total_width_cols = 0,
.total_bytes = 0,
.linestart_count = 1,
.newline_count = 0,
.max_line_width = 0,
.max_line_width_cols = 0,
.ascii_only = true,
},
};
@@ -425,7 +425,7 @@ pub const Segment = union(enum) {
.mem_id = left_chunk.mem_id,
.byte_start = left_chunk.byte_start,
.byte_end = right_chunk.byte_end,
.width = left_chunk.width + right_chunk.width,
.width_cols = left_chunk.width_cols + right_chunk.width_cols,
.flags = left_chunk.flags,
},
};
packages/native/src/text-buffer-view.zig
+169 -169
View File
@@ -129,7 +129,7 @@ pub const MeasureResult = struct {
pub const VirtualLineSpanInfo = struct {
spans: []const StyleSpan,
source_line: usize,
col_offset: u32,
source_col_start: u32,
};
/// Byte and display-column windows describe the same whole-grapheme slice;
@@ -143,38 +143,38 @@ pub const VirtualChunk = struct {
};
const PendingWordPiece = struct {
column_start: u32,
width: u32,
col_start_in_chunk: u32,
width_cols: u32,
byte_start: u32,
byte_end: u32,
chunk: *const TextChunk,
};
const PendingWordPieceFit = struct {
width: u32,
width_cols: u32,
bytes_used: u32,
};
pub const VirtualLine = struct {
chunks: std.ArrayListUnmanaged(VirtualChunk),
width_cols: u32,
col_offset: u32,
document_cell_offset: u32,
source_line: usize,
source_col_offset: u32,
source_col_start: u32,
is_truncated: bool,
ellipsis_pos: u32,
truncation_suffix_start: u32,
ellipsis_col: u32,
truncation_suffix_col_start: u32,
pub fn init() VirtualLine {
return .{
.chunks = .empty,
.width_cols = 0,
.col_offset = 0,
.document_cell_offset = 0,
.source_line = 0,
.source_col_offset = 0,
.source_col_start = 0,
.is_truncated = false,
.ellipsis_pos = 0,
.truncation_suffix_start = 0,
.ellipsis_col = 0,
.truncation_suffix_col_start = 0,
};
}
@@ -529,7 +529,7 @@ pub const UnifiedTextBufferView = struct {
if (vline_idx >= vlines.len) break;
// A soft-wrap boundary belongs to the following visual line. Consumed
// separators before its source start remain on the previous line.
if (logical_col < vlines[vline_idx].source_col_offset) return vline_idx - 1;
if (logical_col < vlines[vline_idx].source_col_start) return vline_idx - 1;
}
// If not found, return last virtual line for this logical line
@@ -960,7 +960,7 @@ pub const UnifiedTextBufferView = struct {
const seg = rope.get(seg_idx) orelse break;
if (seg.isBreak() or seg.isLineStart()) break;
const chunk = seg.asText() orelse continue;
const next_cols = cols_before + chunk.width;
const next_cols = cols_before + chunk.width_cols;
if (coords.col < next_cols) {
const bytes = chunk.getBytes(mem_registry);
const is_ascii = (chunk.flags & TextChunk.Flags.ASCII_ONLY) != 0;
@@ -1052,7 +1052,7 @@ pub const UnifiedTextBufferView = struct {
const last_vline = &self.virtual_lines.items[last_line_idx];
if (last_vline.is_truncated) {
return last_vline.col_offset + last_vline.truncation_suffix_start + (last_vline.width_cols -| last_vline.ellipsis_pos -| 3);
return last_vline.document_cell_offset + last_vline.truncation_suffix_col_start + (last_vline.width_cols -| last_vline.ellipsis_col -| 3);
}
return self.text_buffer.rope().totalWeight();
@@ -1095,7 +1095,7 @@ pub const UnifiedTextBufferView = struct {
const vline_idx: usize = @intCast(clamped_y);
const vline = &self.virtual_lines.items[vline_idx];
const lineStart = vline.col_offset;
const lineStart = vline.document_cell_offset;
const max_local_x = self.maxLocalXOnVisualLine(self.virtual_lines.items, vline_idx);
var localX = @max(0, @min(abs_x, @as(i32, @intCast(max_local_x))));
@@ -1104,28 +1104,28 @@ pub const UnifiedTextBufferView = struct {
const ellipsis_width: u32 = 3;
const localX_u32: u32 = @intCast(localX);
if (localX_u32 >= vline.ellipsis_pos and localX_u32 < vline.ellipsis_pos + ellipsis_width) {
localX = @intCast(vline.ellipsis_pos);
} else if (localX_u32 >= vline.ellipsis_pos + ellipsis_width) {
const suffix_offset = localX_u32 - vline.ellipsis_pos - ellipsis_width;
localX = @intCast(vline.truncation_suffix_start + suffix_offset);
if (localX_u32 >= vline.ellipsis_col and localX_u32 < vline.ellipsis_col + ellipsis_width) {
localX = @intCast(vline.ellipsis_col);
} else if (localX_u32 >= vline.ellipsis_col + ellipsis_width) {
const suffix_offset = localX_u32 - vline.ellipsis_col - ellipsis_width;
localX = @intCast(vline.truncation_suffix_col_start + suffix_offset);
}
}
if (!vline.is_truncated and localX == @as(i32, @intCast(vline.width_cols))) {
const rendered_source_end = vline.source_col_offset + vline.width_cols;
const rendered_source_end = vline.source_col_start + vline.width_cols;
const next_idx = vline_idx + 1;
const has_next_same_source = next_idx < self.virtual_lines.items.len and
self.virtual_lines.items[next_idx].source_line == vline.source_line;
if (has_next_same_source and
self.virtual_lines.items[next_idx].source_col_offset > rendered_source_end)
self.virtual_lines.items[next_idx].source_col_start > rendered_source_end)
{
return self.virtual_lines.items[next_idx].col_offset;
return self.virtual_lines.items[next_idx].document_cell_offset;
}
if (!has_next_same_source) {
const logical_line_width = self.text_buffer.lineWidthAt(@intCast(vline.source_line));
if (logical_line_width > rendered_source_end) {
return lineStart - vline.source_col_offset + logical_line_width;
return lineStart - vline.source_col_start + logical_line_width;
}
}
}
@@ -1157,7 +1157,7 @@ pub const UnifiedTextBufferView = struct {
pub fn getVirtualLineSpans(self: *const Self, vline_idx: usize) VirtualLineSpanInfo {
if (vline_idx >= self.virtual_lines.items.len) {
return .{ .spans = &[_]StyleSpan{}, .source_line = 0, .col_offset = 0 };
return .{ .spans = &[_]StyleSpan{}, .source_line = 0, .source_col_start = 0 };
}
const vline = &self.virtual_lines.items[vline_idx];
@@ -1166,7 +1166,7 @@ pub const UnifiedTextBufferView = struct {
return .{
.spans = spans,
.source_line = vline.source_line,
.col_offset = vline.source_col_offset,
.source_col_start = vline.source_col_start,
};
}
@@ -1272,8 +1272,8 @@ pub const UnifiedTextBufferView = struct {
chunks: std.ArrayListUnmanaged(VirtualChunk) = .empty,
width_cols: u32 = 0,
is_truncated: bool = false,
ellipsis_pos: u32 = 0,
truncation_suffix_start: u32 = 0,
ellipsis_col: u32 = 0,
truncation_suffix_col_start: u32 = 0,
};
// Stage all replacements first so OOM leaves the original layout retryable.
const replacements = self.global_allocator.alloc(Replacement, self.virtual_lines.items.len) catch return false;
@@ -1351,8 +1351,8 @@ pub const UnifiedTextBufferView = struct {
replacement.width_cols = prefix_accumulated + ellipsis_width + suffix_accumulated;
replacement.is_truncated = true;
replacement.ellipsis_pos = prefix_accumulated;
replacement.truncation_suffix_start = actual_suffix_start orelse replacement.width_cols;
replacement.ellipsis_col = prefix_accumulated;
replacement.truncation_suffix_col_start = actual_suffix_start orelse replacement.width_cols;
}
for (self.virtual_lines.items, replacements) |*vline, replacement| {
@@ -1360,8 +1360,8 @@ pub const UnifiedTextBufferView = struct {
vline.chunks = replacement.chunks;
vline.width_cols = replacement.width_cols;
vline.is_truncated = replacement.is_truncated;
vline.ellipsis_pos = replacement.ellipsis_pos;
vline.truncation_suffix_start = replacement.truncation_suffix_start;
vline.ellipsis_col = replacement.ellipsis_col;
vline.truncation_suffix_col_start = replacement.truncation_suffix_col_start;
}
return true;
}
@@ -1457,7 +1457,7 @@ pub const UnifiedTextBufferView = struct {
.byte_start_in_chunk = 0,
.byte_len = chunk.byte_end - chunk.byte_start,
.col_start_in_chunk = 0,
.width_cols = chunk.width,
.width_cols = chunk.width_cols,
}) catch {
ctx.failed = true;
};
@@ -1480,15 +1480,15 @@ pub const UnifiedTextBufferView = struct {
var vline = if (ctx.current_vline) |v| v else VirtualLine.init();
vline.width_cols = line_info.width_cols;
vline.col_offset = line_info.col_offset;
vline.document_cell_offset = line_info.col_offset;
vline.source_line = line_info.line_idx;
vline.source_col_offset = 0;
vline.source_col_start = 0;
ctx.output.virtual_lines.append(ctx.allocator, vline) catch {
ctx.failed = true;
return;
};
ctx.output.cached_line_starts.append(ctx.allocator, vline.col_offset) catch {
ctx.output.cached_line_starts.append(ctx.allocator, vline.document_cell_offset) catch {
ctx.failed = true;
return;
};
@@ -1538,15 +1538,15 @@ pub const UnifiedTextBufferView = struct {
result: @TypeOf(result),
wrap_w: u32,
current_wrap_width: u32,
global_char_offset: u32 = 0,
document_cell_offset: u32 = 0,
line_idx: u32 = 0,
line_col_offset: u32 = 0,
line_position: u32 = 0,
source_line_col_offset: u32 = 0,
current_vline_width_cols: u32 = 0,
current_vline: if (calculation == .render) VirtualLine else void = if (calculation == .render) VirtualLine.init() else {},
current_line_first_vline_idx: if (calculation == .render) u32 else void = if (calculation == .render) 0 else {},
current_line_vline_count: if (calculation == .render) u32 else void = if (calculation == .render) 0 else {},
pending_word_pieces: if (wrap_mode == .word) std.ArrayListUnmanaged(PendingWordPiece) else void = if (wrap_mode == .word) .empty else {},
pending_word_width: if (wrap_mode == .word) u32 else void = if (wrap_mode == .word) 0 else {},
pending_word_width_cols: if (wrap_mode == .word) u32 else void = if (wrap_mode == .word) 0 else {},
pending_word_last_class: if (wrap_mode == .word) utf8.WordClass else void = if (wrap_mode == .word) .other else {},
source_line_has_non_whitespace: if (wrap_mode == .word) bool else void = if (wrap_mode == .word) false else {},
word_chunk: if (wrap_mode == .word) ?*const TextChunk else void = if (wrap_mode == .word) null else {},
@@ -1567,35 +1567,35 @@ pub const UnifiedTextBufferView = struct {
fn commitVirtualLine(wctx: *@This()) Allocator.Error!void {
if (comptime calculation == .render) {
wctx.current_vline.width_cols = wctx.line_position;
wctx.current_vline.width_cols = wctx.current_vline_width_cols;
wctx.current_vline.source_line = wctx.line_idx;
wctx.current_vline.source_col_offset = wctx.line_col_offset;
wctx.current_vline.source_col_start = wctx.source_line_col_offset;
}
try wctx.recordVirtualLine();
if (comptime calculation == .render) wctx.current_line_vline_count += 1;
wctx.line_col_offset += wctx.line_position;
wctx.source_line_col_offset += wctx.current_vline_width_cols;
if (comptime calculation == .render) {
wctx.current_vline = VirtualLine.init();
wctx.current_vline.col_offset = wctx.global_char_offset;
wctx.current_vline.document_cell_offset = wctx.document_cell_offset;
}
wctx.line_position = 0;
wctx.current_vline_width_cols = 0;
wctx.current_wrap_width = wctx.wrap_w;
}
fn recordVirtualLine(wctx: *@This()) Allocator.Error!void {
if (comptime calculation == .measure) {
wctx.result.line_count += 1;
wctx.result.width_cols_max = @max(wctx.result.width_cols_max, wctx.line_position);
wctx.result.width_cols_max = @max(wctx.result.width_cols_max, wctx.current_vline_width_cols);
if (comptime wrap_mode == .word) {
wctx.logical_measure_line_count += 1;
wctx.logical_measure_width_max = @max(wctx.logical_measure_width_max, wctx.line_position);
wctx.logical_measure_width_max = @max(wctx.logical_measure_width_max, wctx.current_vline_width_cols);
}
} else {
const out = wctx.result;
try out.virtual_lines.append(wctx.allocator, wctx.current_vline);
try out.cached_line_starts.append(wctx.allocator, wctx.current_vline.col_offset);
try out.cached_line_starts.append(wctx.allocator, wctx.current_vline.document_cell_offset);
try out.cached_line_widths.append(wctx.allocator, wctx.current_vline.width_cols);
try out.cached_line_sources.append(wctx.allocator, wctx.line_idx);
try out.cached_line_wrap_indices.append(wctx.allocator, wctx.current_line_vline_count);
@@ -1614,8 +1614,8 @@ pub const UnifiedTextBufferView = struct {
{
last.byte_len += byte_len;
last.width_cols += width_cols;
wctx.global_char_offset += width_cols;
wctx.line_position += width_cols;
wctx.document_cell_offset += width_cols;
wctx.current_vline_width_cols += width_cols;
return;
}
}
@@ -1630,8 +1630,8 @@ pub const UnifiedTextBufferView = struct {
.width_cols = width_cols,
});
}
wctx.global_char_offset += width_cols;
wctx.line_position += width_cols;
wctx.document_cell_offset += width_cols;
wctx.current_vline_width_cols += width_cols;
}
fn addVirtualChunkSticky(wctx: *@This(), chunk: *const TextChunk, byte_start: u32, byte_len: u32, col_start: u32, width_cols: u32) bool {
@@ -1650,30 +1650,30 @@ pub const UnifiedTextBufferView = struct {
return true;
}
fn consumeDroppedWhitespace(wctx: *@This(), width: u32) void {
fn consumeDroppedWhitespace(wctx: *@This(), width_cols: u32) void {
// Wrapped separators are hidden on the continuation but remain in
// the preceding visual line's logical source interval.
wctx.global_char_offset += width;
wctx.line_col_offset += width;
if (comptime calculation == .render) wctx.current_vline.col_offset = wctx.global_char_offset;
wctx.document_cell_offset += width_cols;
wctx.source_line_col_offset += width_cols;
if (comptime calculation == .render) wctx.current_vline.document_cell_offset = wctx.document_cell_offset;
}
fn queuePendingWordPiece(wctx: *@This(), chunk: *const TextChunk, start: u32, width: u32, byte_start: u32, byte_end: u32) void {
if (width == 0 or wctx.failed) return;
fn queuePendingWordPiece(wctx: *@This(), chunk: *const TextChunk, col_start_in_chunk: u32, width_cols: u32, byte_start: u32, byte_end: u32) void {
if (width_cols == 0 or wctx.failed) return;
if (wctx.pending_word_pieces.items.len > 0) {
const last = &wctx.pending_word_pieces.items[wctx.pending_word_pieces.items.len - 1];
if (last.chunk == chunk and last.column_start + last.width == start and last.byte_end == byte_start) {
last.width += width;
if (last.chunk == chunk and last.col_start_in_chunk + last.width_cols == col_start_in_chunk and last.byte_end == byte_start) {
last.width_cols += width_cols;
last.byte_end = byte_end;
wctx.pending_word_width += width;
wctx.pending_word_width_cols += width_cols;
return;
}
}
wctx.pending_word_pieces.append(wctx.allocator, .{
.column_start = start,
.width = width,
.col_start_in_chunk = col_start_in_chunk,
.width_cols = width_cols,
.byte_start = byte_start,
.byte_end = byte_end,
.chunk = chunk,
@@ -1681,12 +1681,12 @@ pub const UnifiedTextBufferView = struct {
wctx.failed = true;
return;
};
wctx.pending_word_width += width;
wctx.pending_word_width_cols += width_cols;
}
fn clearPendingWord(wctx: *@This()) void {
wctx.pending_word_pieces.clearRetainingCapacity();
wctx.pending_word_width = 0;
wctx.pending_word_width_cols = 0;
wctx.pending_word_last_class = .other;
}
@@ -1703,38 +1703,38 @@ pub const UnifiedTextBufferView = struct {
wctx.pending_word_pieces.items.len = remaining;
}
fn fitPendingWordPiece(wctx: *@This(), piece: PendingWordPiece, max_width: u32, allow_forced_grapheme: bool) PendingWordPieceFit {
if (piece.width <= max_width) {
return .{ .width = piece.width, .bytes_used = piece.byte_end - piece.byte_start };
fn fitPendingWordPiece(wctx: *@This(), piece: PendingWordPiece, max_width_cols: u32, allow_forced_grapheme: bool) PendingWordPieceFit {
if (piece.width_cols <= max_width_cols) {
return .{ .width_cols = piece.width_cols, .bytes_used = piece.byte_end - piece.byte_start };
}
if (max_width == 0) return .{ .width = 0, .bytes_used = 0 };
if (max_width_cols == 0) return .{ .width_cols = 0, .bytes_used = 0 };
const chunk_bytes = piece.chunk.getBytes(wctx.text_buffer.memRegistry());
if (piece.byte_start > piece.byte_end or piece.byte_end > chunk_bytes.len) {
wctx.failed = true;
return .{ .width = 0, .bytes_used = 0 };
return .{ .width_cols = 0, .bytes_used = 0 };
}
const slice_bytes = chunk_bytes[piece.byte_start..piece.byte_end];
const is_ascii_only = (piece.chunk.flags & TextChunk.Flags.ASCII_ONLY) != 0;
const fit = utf8.findWrapPosByWidthGraphemeSafe(
slice_bytes,
max_width,
max_width_cols,
wctx.text_buffer.tabWidth(),
is_ascii_only,
wctx.text_buffer.widthMethod(),
);
if (fit.columns_used > 0 and fit.byte_offset > 0) {
return .{
.width = @min(fit.columns_used, piece.width),
.width_cols = @min(fit.columns_used, piece.width_cols),
.bytes_used = @min(fit.byte_offset, piece.byte_end - piece.byte_start),
};
}
if (!allow_forced_grapheme) return .{ .width = 0, .bytes_used = 0 };
if (!allow_forced_grapheme) return .{ .width_cols = 0, .bytes_used = 0 };
const forced = utf8.findGraphemePosByWidth(
slice_bytes,
max_width,
max_width_cols,
wctx.text_buffer.tabWidth(),
is_ascii_only,
true,
@@ -1742,144 +1742,144 @@ pub const UnifiedTextBufferView = struct {
);
if (forced.columns_used == 0 or forced.byte_offset == 0) {
wctx.failed = true;
return .{ .width = 0, .bytes_used = 0 };
return .{ .width_cols = 0, .bytes_used = 0 };
}
return .{
.width = @min(forced.columns_used, piece.width),
.width_cols = @min(forced.columns_used, piece.width_cols),
.bytes_used = @min(forced.byte_offset, piece.byte_end - piece.byte_start),
};
}
fn consumePendingWordPrefix(wctx: *@This(), max_width: u32) bool {
if (max_width == 0 or wctx.pending_word_width == 0 or wctx.failed) return false;
fn consumePendingWordPrefix(wctx: *@This(), max_width_cols: u32) bool {
if (max_width_cols == 0 or wctx.pending_word_width_cols == 0 or wctx.failed) return false;
const line_before = wctx.line_position;
var remaining = max_width;
const vline_width_cols_before = wctx.current_vline_width_cols;
var remaining_width_cols = max_width_cols;
var consumed_count: usize = 0;
while (consumed_count < wctx.pending_word_pieces.items.len and remaining > 0) {
while (consumed_count < wctx.pending_word_pieces.items.len and remaining_width_cols > 0) {
const piece = wctx.pending_word_pieces.items[consumed_count];
if (piece.width <= remaining) {
if (!addVirtualChunkSticky(wctx, piece.chunk, piece.byte_start, piece.byte_end - piece.byte_start, piece.column_start, piece.width)) return false;
remaining -= piece.width;
if (piece.width_cols <= remaining_width_cols) {
if (!addVirtualChunkSticky(wctx, piece.chunk, piece.byte_start, piece.byte_end - piece.byte_start, piece.col_start_in_chunk, piece.width_cols)) return false;
remaining_width_cols -= piece.width_cols;
consumed_count += 1;
continue;
}
const fit = fitPendingWordPiece(wctx, piece, remaining, wctx.line_position == line_before);
if (fit.width == 0) break;
if (!addVirtualChunkSticky(wctx, piece.chunk, piece.byte_start, fit.bytes_used, piece.column_start, fit.width)) return false;
wctx.pending_word_pieces.items[consumed_count].column_start += fit.width;
wctx.pending_word_pieces.items[consumed_count].width -= fit.width;
const fit = fitPendingWordPiece(wctx, piece, remaining_width_cols, wctx.current_vline_width_cols == vline_width_cols_before);
if (fit.width_cols == 0) break;
if (!addVirtualChunkSticky(wctx, piece.chunk, piece.byte_start, fit.bytes_used, piece.col_start_in_chunk, fit.width_cols)) return false;
wctx.pending_word_pieces.items[consumed_count].col_start_in_chunk += fit.width_cols;
wctx.pending_word_pieces.items[consumed_count].width_cols -= fit.width_cols;
wctx.pending_word_pieces.items[consumed_count].byte_start += fit.bytes_used;
if (wctx.pending_word_pieces.items[consumed_count].width == 0) consumed_count += 1;
if (wctx.pending_word_pieces.items[consumed_count].width_cols == 0) consumed_count += 1;
break;
}
dropPendingWordPrefix(wctx, consumed_count);
const consumed_width = wctx.line_position - line_before;
wctx.pending_word_width -= consumed_width;
return consumed_width > 0;
const consumed_width_cols = wctx.current_vline_width_cols - vline_width_cols_before;
wctx.pending_word_width_cols -= consumed_width_cols;
return consumed_width_cols > 0;
}
fn appendPendingWordToLine(wctx: *@This()) void {
for (wctx.pending_word_pieces.items) |piece| {
if (!addVirtualChunkSticky(wctx, piece.chunk, piece.byte_start, piece.byte_end - piece.byte_start, piece.column_start, piece.width)) return;
if (!addVirtualChunkSticky(wctx, piece.chunk, piece.byte_start, piece.byte_end - piece.byte_start, piece.col_start_in_chunk, piece.width_cols)) return;
}
clearPendingWord(wctx);
}
fn finalizePendingWord(wctx: *@This()) void {
while (wctx.pending_word_width > 0 and !wctx.failed) {
const wrap_limit = wctx.wordWrapWidth();
if (wctx.line_position > 0 and wctx.line_position + wctx.pending_word_width > wrap_limit) {
while (wctx.pending_word_width_cols > 0 and !wctx.failed) {
const wrap_limit_cols = wctx.wordWrapWidth();
if (wctx.current_vline_width_cols > 0 and wctx.current_vline_width_cols + wctx.pending_word_width_cols > wrap_limit_cols) {
if (!commitVirtualLineSticky(wctx)) return;
continue;
}
if (wctx.line_position == 0 and wctx.pending_word_width > wrap_limit) {
if (!consumePendingWordPrefix(wctx, wrap_limit)) return;
if (wctx.pending_word_width > 0 and !commitVirtualLineSticky(wctx)) return;
if (wctx.current_vline_width_cols == 0 and wctx.pending_word_width_cols > wrap_limit_cols) {
if (!consumePendingWordPrefix(wctx, wrap_limit_cols)) return;
if (wctx.pending_word_width_cols > 0 and !commitVirtualLineSticky(wctx)) return;
continue;
}
appendPendingWordToLine(wctx);
}
}
fn placeCompleteWordPiece(wctx: *@This(), chunk: *const TextChunk, start: u32, width: u32, byte_start: u32, byte_end: u32) void {
if (width == 0 or wctx.failed) return;
fn placeCompleteWordPiece(wctx: *@This(), chunk: *const TextChunk, col_start_in_chunk: u32, width_cols: u32, byte_start: u32, byte_end: u32) void {
if (width_cols == 0 or wctx.failed) return;
var piece: PendingWordPiece = .{
.column_start = start,
.width = width,
.col_start_in_chunk = col_start_in_chunk,
.width_cols = width_cols,
.byte_start = byte_start,
.byte_end = byte_end,
.chunk = chunk,
};
while (piece.width > 0 and !wctx.failed) {
const wrap_limit = wctx.wordWrapWidth();
if (piece.width <= wrap_limit) {
if (wctx.line_position > 0 and wctx.line_position + piece.width > wrap_limit and !commitVirtualLineSticky(wctx)) return;
_ = addVirtualChunkSticky(wctx, chunk, piece.byte_start, piece.byte_end - piece.byte_start, piece.column_start, piece.width);
while (piece.width_cols > 0 and !wctx.failed) {
const wrap_limit_cols = wctx.wordWrapWidth();
if (piece.width_cols <= wrap_limit_cols) {
if (wctx.current_vline_width_cols > 0 and wctx.current_vline_width_cols + piece.width_cols > wrap_limit_cols and !commitVirtualLineSticky(wctx)) return;
_ = addVirtualChunkSticky(wctx, chunk, piece.byte_start, piece.byte_end - piece.byte_start, piece.col_start_in_chunk, piece.width_cols);
return;
}
if (wctx.line_position > 0 and !commitVirtualLineSticky(wctx)) return;
if (wctx.current_vline_width_cols > 0 and !commitVirtualLineSticky(wctx)) return;
const fit = fitPendingWordPiece(wctx, piece, wctx.wordWrapWidth(), true);
if (fit.width == 0) return;
if (!addVirtualChunkSticky(wctx, chunk, piece.byte_start, fit.bytes_used, piece.column_start, fit.width)) return;
piece.column_start += fit.width;
piece.width -= fit.width;
if (fit.width_cols == 0) return;
if (!addVirtualChunkSticky(wctx, chunk, piece.byte_start, fit.bytes_used, piece.col_start_in_chunk, fit.width_cols)) return;
piece.col_start_in_chunk += fit.width_cols;
piece.width_cols -= fit.width_cols;
piece.byte_start += fit.bytes_used;
if (piece.width > 0 and !commitVirtualLineSticky(wctx)) return;
if (piece.width_cols > 0 and !commitVirtualLineSticky(wctx)) return;
}
}
fn flushCompleteWordPiece(wctx: *@This(), chunk: *const TextChunk, start: u32, width: u32, byte_start: u32, byte_end: u32) void {
if (width == 0 or wctx.failed) return;
if (wctx.pending_word_width > 0) {
queuePendingWordPiece(wctx, chunk, start, width, byte_start, byte_end);
fn flushCompleteWordPiece(wctx: *@This(), chunk: *const TextChunk, col_start_in_chunk: u32, width_cols: u32, byte_start: u32, byte_end: u32) void {
if (width_cols == 0 or wctx.failed) return;
if (wctx.pending_word_width_cols > 0) {
queuePendingWordPiece(wctx, chunk, col_start_in_chunk, width_cols, byte_start, byte_end);
finalizePendingWord(wctx);
} else {
placeCompleteWordPiece(wctx, chunk, start, width, byte_start, byte_end);
placeCompleteWordPiece(wctx, chunk, col_start_in_chunk, width_cols, byte_start, byte_end);
}
}
fn processWhitespaceBreak(wctx: *@This(), chunk: *const TextChunk, col_start: u32, byte_start: u32, wrap_break: utf8.LayoutWrapBreak) void {
if (wrap_break.col_offset > col_start) {
if (wrap_break.col_start > col_start) {
flushCompleteWordPiece(
wctx,
chunk,
col_start,
wrap_break.col_offset - col_start,
wrap_break.col_start - col_start,
byte_start,
wrap_break.byte_offset,
wrap_break.byte_start,
);
if (wctx.failed) return;
wctx.source_line_has_non_whitespace = true;
} else if (wctx.pending_word_width > 0) {
} else if (wctx.pending_word_width_cols > 0) {
finalizePendingWord(wctx);
if (wctx.failed) return;
}
// Logical-line indentation is content; only later separators may be elided.
const preserve_leading = !wctx.source_line_has_non_whitespace;
const wrap_limit = wctx.wordWrapWidth();
if (!preserve_leading and wctx.line_position + wrap_break.width > wrap_limit) {
if (wctx.line_position > 0 and !commitVirtualLineSticky(wctx)) return;
consumeDroppedWhitespace(wctx, wrap_break.width);
const wrap_limit_cols = wctx.wordWrapWidth();
if (!preserve_leading and wctx.current_vline_width_cols + wrap_break.width_cols > wrap_limit_cols) {
if (wctx.current_vline_width_cols > 0 and !commitVirtualLineSticky(wctx)) return;
consumeDroppedWhitespace(wctx, wrap_break.width_cols);
return;
}
if (!preserve_leading and wctx.line_position == 0) {
consumeDroppedWhitespace(wctx, wrap_break.width);
if (!preserve_leading and wctx.current_vline_width_cols == 0) {
consumeDroppedWhitespace(wctx, wrap_break.width_cols);
return;
}
placeCompleteWordPiece(
wctx,
chunk,
wrap_break.col_offset,
wrap_break.width,
wrap_break.byte_offset,
wrap_break.col_start,
wrap_break.width_cols,
wrap_break.byte_start,
wrap_break.byteEnd(),
);
}
@@ -1887,7 +1887,7 @@ pub const UnifiedTextBufferView = struct {
fn processWordWrapBreakValue(wctx: *@This(), wrap_break: utf8.LayoutWrapBreak) void {
const chunk = wctx.word_chunk orelse return;
const chunk_bytes = chunk.getBytes(wctx.text_buffer.memRegistry());
const col_end = @min(wrap_break.colEnd(), chunk.width);
const col_end = @min(wrap_break.colEnd(), chunk.width_cols);
const byte_end = @min(wrap_break.byteEnd(), @as(u32, @intCast(chunk_bytes.len)));
if (col_end < wctx.word_chunk_col_start or byte_end < wctx.word_chunk_byte_start) return;
if (wrap_break.kind == .whitespace) {
@@ -1908,7 +1908,7 @@ pub const UnifiedTextBufferView = struct {
);
if (wctx.failed) return;
wctx.source_line_has_non_whitespace = true;
} else if (byte_end > wctx.word_chunk_byte_start and wctx.pending_word_width > 0) {
} else if (byte_end > wctx.word_chunk_byte_start and wctx.pending_word_width_cols > 0) {
finalizePendingWord(wctx);
if (wctx.failed) return;
}
@@ -1946,7 +1946,7 @@ pub const UnifiedTextBufferView = struct {
info.word_classes
else
utf8.chunkWordClassEdges(chunk_bytes);
if (wctx.pending_word_width > 0 and
if (wctx.pending_word_width_cols > 0 and
utf8.isCjkAsciiTransition(wctx.pending_word_last_class, word_classes.first))
{
finalizePendingWord(wctx);
@@ -1976,12 +1976,12 @@ pub const UnifiedTextBufferView = struct {
break :blk streamed_layout.last;
};
if (wctx.word_chunk_col_start < chunk.width) {
if (wctx.word_chunk_col_start < chunk.width_cols) {
queuePendingWordPiece(
wctx,
chunk,
wctx.word_chunk_col_start,
chunk.width - wctx.word_chunk_col_start,
chunk.width_cols - wctx.word_chunk_col_start,
wctx.word_chunk_byte_start,
@intCast(chunk_bytes.len),
);
@@ -1995,52 +1995,52 @@ pub const UnifiedTextBufferView = struct {
const chunk_bytes = chunk.getBytes(wctx.text_buffer.memRegistry());
const is_ascii_only = (chunk.flags & TextChunk.Flags.ASCII_ONLY) != 0;
const tab_width = wctx.text_buffer.tabWidth();
var byte_offset: usize = 0;
var char_offset: u32 = 0;
var chunk_byte_offset: usize = 0;
var chunk_col_offset: u32 = 0;
// Advance bytes with columns; re-deriving each byte boundary would
// make repeated wraps within a long chunk quadratic.
while (char_offset < chunk.width) {
const line_wrap_w = wctx.lineWrapWidth();
const remaining_width = if (wctx.line_position < line_wrap_w) line_wrap_w - wctx.line_position else 0;
while (chunk_col_offset < chunk.width_cols) {
const line_wrap_width_cols = wctx.lineWrapWidth();
const remaining_width_cols = if (wctx.current_vline_width_cols < line_wrap_width_cols) line_wrap_width_cols - wctx.current_vline_width_cols else 0;
if (remaining_width == 0) {
if (wctx.line_position > 0) {
if (remaining_width_cols == 0) {
if (wctx.current_vline_width_cols > 0) {
try commitVirtualLine(wctx);
continue;
}
const remaining_bytes = chunk_bytes[byte_offset..];
const remaining_bytes = chunk_bytes[chunk_byte_offset..];
const force_result = utf8.findGraphemePosByWidth(remaining_bytes, 1, tab_width, is_ascii_only, true, width_method);
if (force_result.grapheme_count > 0) {
try addVirtualChunk(wctx, chunk, @intCast(byte_offset), force_result.byte_offset, char_offset, force_result.columns_used);
char_offset += force_result.columns_used;
byte_offset += force_result.byte_offset;
try addVirtualChunk(wctx, chunk, @intCast(chunk_byte_offset), force_result.byte_offset, chunk_col_offset, force_result.columns_used);
chunk_col_offset += force_result.columns_used;
chunk_byte_offset += force_result.byte_offset;
} else {
break;
}
continue;
}
const remaining_bytes = chunk_bytes[byte_offset..];
const remaining_bytes = chunk_bytes[chunk_byte_offset..];
const wrap_result = utf8.findWrapPosByWidthGraphemeSafe(
remaining_bytes,
remaining_width,
remaining_width_cols,
tab_width,
is_ascii_only,
width_method,
);
if (wrap_result.grapheme_count == 0) {
if (wctx.line_position > 0) {
if (wctx.current_vline_width_cols > 0) {
try commitVirtualLine(wctx);
continue;
}
const force_result = utf8.findGraphemePosByWidth(remaining_bytes, 1, tab_width, is_ascii_only, true, width_method);
if (force_result.grapheme_count > 0) {
try addVirtualChunk(wctx, chunk, @intCast(byte_offset), force_result.byte_offset, char_offset, force_result.columns_used);
char_offset += force_result.columns_used;
byte_offset += force_result.byte_offset;
if (char_offset < chunk.width) {
try addVirtualChunk(wctx, chunk, @intCast(chunk_byte_offset), force_result.byte_offset, chunk_col_offset, force_result.columns_used);
chunk_col_offset += force_result.columns_used;
chunk_byte_offset += force_result.byte_offset;
if (chunk_col_offset < chunk.width_cols) {
try commitVirtualLine(wctx);
continue;
}
@@ -2048,11 +2048,11 @@ pub const UnifiedTextBufferView = struct {
break;
}
try addVirtualChunk(wctx, chunk, @intCast(byte_offset), wrap_result.byte_offset, char_offset, wrap_result.columns_used);
char_offset += wrap_result.columns_used;
byte_offset += wrap_result.byte_offset;
try addVirtualChunk(wctx, chunk, @intCast(chunk_byte_offset), wrap_result.byte_offset, chunk_col_offset, wrap_result.columns_used);
chunk_col_offset += wrap_result.columns_used;
chunk_byte_offset += wrap_result.byte_offset;
if (wctx.line_position >= line_wrap_w and char_offset < chunk.width) {
if (wctx.current_vline_width_cols >= line_wrap_width_cols and chunk_col_offset < chunk.width_cols) {
try commitVirtualLine(wctx);
}
}
@@ -2107,7 +2107,7 @@ pub const UnifiedTextBufferView = struct {
)) |summary| {
wctx.result.line_count += summary.line_count;
wctx.result.width_cols_max = @max(wctx.result.width_cols_max, summary.width_max);
wctx.global_char_offset += chunk.width;
wctx.document_cell_offset += chunk.width_cols;
used_measure_cache = true;
} else {
processWordChunk(wctx, chunk);
@@ -2124,12 +2124,12 @@ pub const UnifiedTextBufferView = struct {
const has_content = if (comptime calculation == .render)
wctx.current_vline.chunks.items.len > 0
else
wctx.line_position > 0;
wctx.current_vline_width_cols > 0;
if (!used_measure_cache and (has_content or line_info.width_cols == 0)) {
if (comptime calculation == .render) {
wctx.current_vline.width_cols = wctx.line_position;
wctx.current_vline.width_cols = wctx.current_vline_width_cols;
wctx.current_vline.source_line = wctx.line_idx;
wctx.current_vline.source_col_offset = wctx.line_col_offset;
wctx.current_vline.source_col_start = wctx.source_line_col_offset;
}
wctx.recordVirtualLine() catch {
wctx.failed = true;
@@ -2175,15 +2175,15 @@ pub const UnifiedTextBufferView = struct {
};
}
wctx.global_char_offset += 1;
wctx.document_cell_offset += 1;
wctx.line_idx += 1;
wctx.line_col_offset = 0;
wctx.line_position = 0;
wctx.source_line_col_offset = 0;
wctx.current_vline_width_cols = 0;
wctx.current_wrap_width = wctx.wrap_w;
if (comptime calculation == .render) {
wctx.current_vline = VirtualLine.init();
wctx.current_vline.col_offset = wctx.global_char_offset;
wctx.current_vline.document_cell_offset = wctx.document_cell_offset;
wctx.current_line_first_vline_idx = @intCast(wctx.result.virtual_lines.items.len);
wctx.current_line_vline_count = 0;
}
packages/native/src/text-buffer.zig
+9 -9
View File
@@ -28,7 +28,7 @@ pub const TextBufferError = seg_mod.TextBufferError;
pub const Highlight = seg_mod.Highlight;
pub const StyleSpan = seg_mod.StyleSpan;
pub const WrapMode = seg_mod.WrapMode;
pub const GraphemeInfo = seg_mod.GraphemeInfo;
pub const RenderClusterInfo = seg_mod.RenderClusterInfo;
pub const SyntaxStyle = ss.SyntaxStyle;
@@ -381,7 +381,7 @@ pub const UnifiedTextBuffer = struct {
// Basic queries using unified rope
pub fn getLength(self: *const Self) u32 {
const metrics = self._rope.root.metrics();
return metrics.custom.total_width;
return metrics.custom.total_width_cols;
}
pub fn getByteSize(self: *const Self) u32 {
@@ -584,7 +584,7 @@ pub const UnifiedTextBuffer = struct {
.mem_id = mem_id,
.byte_start = byte_start,
.byte_end = byte_end,
.width = chunk_width,
.width_cols = chunk_width,
.flags = flags,
};
}
@@ -598,7 +598,7 @@ pub const UnifiedTextBuffer = struct {
mem_id: u8,
byte_offset: u32,
prepend_linestart: bool,
) TextBufferError!struct { segments: std.ArrayListUnmanaged(Segment), total_width: u32, allocator: Allocator } {
) TextBufferError!struct { segments: std.ArrayListUnmanaged(Segment), total_width_cols: u32, allocator: Allocator } {
var break_result = utf8.LineBreakResult.init(allocator);
defer break_result.deinit();
try utf8.findLineBreaks(text, &break_result);
@@ -611,7 +611,7 @@ pub const UnifiedTextBuffer = struct {
}
var local_start: u32 = 0;
var total_width: u32 = 0;
var total_width_cols: u32 = 0;
for (break_result.breaks.items) |line_break| {
const break_pos: u32 = @intCast(line_break.pos);
@@ -623,7 +623,7 @@ pub const UnifiedTextBuffer = struct {
if (local_end > local_start) {
const chunk = self.createChunk(mem_id, byte_offset + local_start, byte_offset + local_end);
try segments.append(allocator, .{ .text = chunk });
total_width += chunk.width;
total_width_cols += chunk.width_cols;
}
try segments.append(allocator, .{ .brk = {} });
@@ -635,10 +635,10 @@ pub const UnifiedTextBuffer = struct {
if (local_start < text.len) {
const chunk = self.createChunk(mem_id, byte_offset + local_start, byte_offset + @as(u32, @intCast(text.len)));
try segments.append(allocator, .{ .text = chunk });
total_width += chunk.width;
total_width_cols += chunk.width_cols;
}
return .{ .segments = segments, .total_width = total_width, .allocator = allocator };
return .{ .segments = segments, .total_width_cols = total_width_cols, .allocator = allocator };
}
pub fn getLineCount(self: *const Self) u32 {
@@ -1297,7 +1297,7 @@ pub const UnifiedTextBuffer = struct {
if (segment.asText()) |chunk| {
const mutable = @constCast(chunk);
const bytes = chunk.getBytes(&ctx.buffer.mem_registry);
mutable.width = utf8.calculateTextWidth(
mutable.width_cols = utf8.calculateTextWidth(
bytes,
ctx.buffer.tab_width,
chunk.isAsciiOnly(),
packages/native/src/utf8.zig
+55 -56
View File
@@ -113,18 +113,18 @@ pub const LayoutWrapBreakKind = enum(u8) {
/// The window identifies the grapheme that creates the break, not the position
/// after it; consumers use byteEnd()/colEnd() to cross the boundary.
pub const LayoutWrapBreak = struct {
byte_offset: u32,
col_offset: u32,
byte_start: u32,
col_start: u32,
byte_len: u32,
width: u32,
width_cols: u32,
kind: LayoutWrapBreakKind,
pub fn colEnd(self: LayoutWrapBreak) u32 {
return self.col_offset + self.width;
return self.col_start + self.width_cols;
}
pub fn byteEnd(self: LayoutWrapBreak) u32 {
return self.byte_offset + self.byte_len;
return self.byte_start + self.byte_len;
}
};
@@ -1356,7 +1356,7 @@ fn calculateTextWidthUnicode(text: []const u8, tab_width: u8, isASCIIOnly: bool,
}
// General case with Unicode support and grapheme cluster handling
var total_width: u32 = 0;
var total_width_cols: u32 = 0;
var pos: usize = 0;
var prev_cp: ?u21 = null;
var break_state: uucode.grapheme.BreakState = .default;
@@ -1374,7 +1374,7 @@ fn calculateTextWidthUnicode(text: []const u8, tab_width: u8, isASCIIOnly: bool,
if (is_break) {
if (prev_cp != null) {
total_width += state.width;
total_width_cols += state.width;
}
const cp_width = charWidth(b0, curr_cp, tab_width);
@@ -1389,10 +1389,10 @@ fn calculateTextWidthUnicode(text: []const u8, tab_width: u8, isASCIIOnly: bool,
}
if (prev_cp != null) {
total_width += state.width;
total_width_cols += state.width;
}
return total_width;
return total_width_cols;
}
/// Calculate text width using wcwidth-style codepoint-by-codepoint processing
@@ -1405,7 +1405,7 @@ fn calculateTextWidthWCWidth(text: []const u8, tab_width: u8, isASCIIOnly: bool)
}
// Unicode path - sum width of all codepoints
var total_width: u32 = 0;
var total_width_cols: u32 = 0;
var pos: usize = 0;
while (pos < text.len) {
@@ -1418,20 +1418,21 @@ fn calculateTextWidthWCWidth(text: []const u8, tab_width: u8, isASCIIOnly: bool)
const cp_len: usize = if (b0 < 0x80) 1 else decodeUtf8Unchecked(text, pos).len;
const cp_width = charWidth(b0, curr_cp, tab_width);
total_width += cp_width;
total_width_cols += cp_width;
pos += cp_len;
}
return total_width;
return total_width_cols;
}
/// Grapheme cluster information for caching
pub const GraphemeInfo = struct {
byte_offset: u32,
/// Sparse render-cluster and tab metadata for caching.
/// Boundaries follow the selected mode; widths are terminal cells summed per codepoint in wcwidth mode.
pub const RenderClusterInfo = struct {
byte_start: u32,
byte_len: u32,
width: u32,
col_offset: u32,
width_cols: u32,
col_start: u32,
};
pub const ChunkLayoutInfo = struct {
@@ -1458,17 +1459,17 @@ pub fn chunkWordClassEdges(text: []const u8) WordClassEdges {
inline fn emitLayoutWrapBreak(
visitor: anytype,
byte_offset: usize,
byte_start: usize,
byte_len: usize,
col_offset: u32,
width: u32,
col_start: u32,
width_cols: u32,
kind: LayoutWrapBreakKind,
) !bool {
return visitor.emit(.{
.byte_offset = @intCast(byte_offset),
.col_offset = col_offset,
.byte_start = @intCast(byte_start),
.col_start = col_start,
.byte_len = @intCast(byte_len),
.width = width,
.width_cols = width_cols,
.kind = kind,
});
}
@@ -1728,31 +1729,30 @@ fn walkChunkLayoutInfoGeneric(
return .{ .first = first_word_class, .last = if (cluster_started) cluster_class else .other };
}
/// Find all grapheme clusters in text and return info for multi-byte graphemes and tabs
/// This is a proxy function that dispatches to the appropriate implementation based on width_method
pub fn findGraphemeInfo(
/// Find sparse render-cluster metadata for multibyte clusters and tabs.
/// Cluster boundaries follow the selected mode; wcwidth mode sums codepoint cell widths within each cluster.
pub fn findRenderClusterInfo(
allocator: std.mem.Allocator,
text: []const u8,
tab_width: u8,
isASCIIOnly: bool,
width_method: WidthMethod,
result: *std.ArrayListUnmanaged(GraphemeInfo),
result: *std.ArrayListUnmanaged(RenderClusterInfo),
) !void {
switch (width_method) {
.unicode, .unicode_wide, .no_zwj => try findGraphemeInfoUnicode(allocator, text, tab_width, isASCIIOnly, width_method, result),
.wcwidth => try findGraphemeInfoWCWidth(allocator, text, tab_width, isASCIIOnly, result),
.unicode, .unicode_wide, .no_zwj => try findRenderClusterInfoUnicode(allocator, text, tab_width, isASCIIOnly, width_method, result),
.wcwidth => try findRenderClusterInfoWCWidth(allocator, text, tab_width, isASCIIOnly, result),
}
}
/// Find all grapheme clusters using Unicode grapheme cluster segmentation
/// This version treats grapheme clusters as single units for width calculation
fn findGraphemeInfoUnicode(
/// Find render clusters using the selected Unicode boundary mode.
fn findRenderClusterInfoUnicode(
allocator: std.mem.Allocator,
text: []const u8,
tab_width: u8,
isASCIIOnly: bool,
width_method: WidthMethod,
result: *std.ArrayListUnmanaged(GraphemeInfo),
result: *std.ArrayListUnmanaged(RenderClusterInfo),
) !void {
// In wcwidth mode, always process to capture combining marks on ASCII
if (isASCIIOnly and width_method != .wcwidth) {
@@ -1794,10 +1794,10 @@ fn findGraphemeInfoUnicode(
if (cluster_width_state.width > 0 or width_method == .wcwidth) {
const cluster_byte_len = (pos + i) - cluster_start;
try result.append(allocator, .{
.byte_offset = @intCast(cluster_start),
.byte_start = @intCast(cluster_start),
.byte_len = @intCast(cluster_byte_len),
.width = cluster_width_state.width,
.col_offset = cluster_start_col,
.width_cols = cluster_width_state.width,
.col_start = cluster_start_col,
});
}
col += cluster_width_state.width;
@@ -1840,10 +1840,10 @@ fn findGraphemeInfoUnicode(
if (cluster_width_state.width > 0 or width_method == .wcwidth) {
const cluster_byte_len = (pos + i) - cluster_start;
try result.append(allocator, .{
.byte_offset = @intCast(cluster_start),
.byte_start = @intCast(cluster_start),
.byte_len = @intCast(cluster_byte_len),
.width = cluster_width_state.width,
.col_offset = cluster_start_col,
.width_cols = cluster_width_state.width,
.col_start = cluster_start_col,
});
}
col += cluster_width_state.width;
@@ -1885,10 +1885,10 @@ fn findGraphemeInfoUnicode(
if (cluster_width_state.width > 0 or width_method == .wcwidth) {
const cluster_byte_len = pos - cluster_start;
try result.append(allocator, .{
.byte_offset = @intCast(cluster_start),
.byte_start = @intCast(cluster_start),
.byte_len = @intCast(cluster_byte_len),
.width = cluster_width_state.width,
.col_offset = cluster_start_col,
.width_cols = cluster_width_state.width,
.col_start = cluster_start_col,
});
}
col += cluster_width_state.width;
@@ -1917,23 +1917,22 @@ fn findGraphemeInfoUnicode(
if (cluster_width_state.width > 0 or width_method == .wcwidth) {
const cluster_byte_len = text.len - cluster_start;
try result.append(allocator, .{
.byte_offset = @intCast(cluster_start),
.byte_start = @intCast(cluster_start),
.byte_len = @intCast(cluster_byte_len),
.width = cluster_width_state.width,
.col_offset = cluster_start_col,
.width_cols = cluster_width_state.width,
.col_start = cluster_start_col,
});
}
}
}
/// Find all grapheme clusters using wcwidth-style codepoint-by-codepoint processing
/// This version treats each codepoint as a separate character (tmux/wcwidth behavior)
fn findGraphemeInfoWCWidth(
/// Find Unicode render-cluster byte boundaries with codepoint-summed wcwidth cell widths.
fn findRenderClusterInfoWCWidth(
allocator: std.mem.Allocator,
text: []const u8,
tab_width: u8,
isASCIIOnly: bool,
result: *std.ArrayListUnmanaged(GraphemeInfo),
result: *std.ArrayListUnmanaged(RenderClusterInfo),
) !void {
// wcwidth mode should still produce the same grapheme cluster boundaries as Unicode
// (so ZWJ sequences and combining marks stay together), but the width of each cluster
@@ -1971,16 +1970,16 @@ fn findGraphemeInfoWCWidth(
if (pos + cp_len > text.len) break;
// Use wcwidth break detection (each codepoint is separate, tmux-style)
// Keep Unicode render-cluster boundaries while summing codepoint cell widths.
const is_break = isGraphemeBreak(prev_cp, curr_cp, &break_state, .wcwidth);
if (is_break) {
if (cluster_started and (cluster_is_multibyte or cluster_is_tab)) {
try result.append(allocator, .{
.byte_offset = @intCast(cluster_start),
.byte_start = @intCast(cluster_start),
.byte_len = @intCast(pos - cluster_start),
.width = cluster_width_state.width,
.col_offset = cluster_start_col,
.width_cols = cluster_width_state.width,
.col_start = cluster_start_col,
});
col += cluster_width_state.width;
} else if (cluster_started) {
@@ -2011,10 +2010,10 @@ fn findGraphemeInfoWCWidth(
if (cluster_started) {
if (cluster_is_multibyte or cluster_is_tab) {
try result.append(allocator, .{
.byte_offset = @intCast(cluster_start),
.byte_start = @intCast(cluster_start),
.byte_len = @intCast(text.len - cluster_start),
.width = cluster_width_state.width,
.col_offset = cluster_start_col,
.width_cols = cluster_width_state.width,
.col_start = cluster_start_col,
});
col += cluster_width_state.width;
} else {