fix(core): decode process output using Windows console code page

On Windows with non-UTF-8 console code pages (e.g., GBK for Chinese
locale), child process output is mis-decoded by String::from_utf8_lossy,
producing mojibake. Add decode_process_output() that detects the console
output code page via GetConsoleOutputCP() and decodes with encoding_rs.

Replaces from_utf8_lossy in the core capture paths (exec_capture,
exec_capture_stdin, TOML filter path, proxy streaming path). Module-
specific call sites left for follow-up.

Fixes #2452
This commit is contained in:
guy oron
2026-06-29 09:09:32 +03:00
committed by guyoron1
parent 6f0b0cad29
commit 5bd410eb51
5 changed files with 133 additions and 8 deletions
Generated
+10
View File
@@ -333,6 +333,15 @@ dependencies = [
"syn",
]
[[package]]
name = "encoding_rs"
version = "0.8.35"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3"
dependencies = [
"cfg-if",
]
[[package]]
name = "env_home"
version = "0.1.0"
@@ -893,6 +902,7 @@ dependencies = [
"clap",
"colored",
"dirs",
"encoding_rs",
"flate2",
"getrandom 0.4.2",
"ignore",
+3
View File
@@ -37,6 +37,9 @@ automod = "1"
[target.'cfg(unix)'.dependencies]
libc = "0.2"
[target.'cfg(windows)'.dependencies]
encoding_rs = "0.8"
[build-dependencies]
toml = "0.8"
+4 -4
View File
@@ -558,8 +558,8 @@ pub fn exec_capture(cmd: &mut Command) -> Result<CaptureResult> {
cmd.stdin(Stdio::null());
let output = cmd.output().context("Failed to execute command")?;
Ok(CaptureResult {
stdout: String::from_utf8_lossy(&output.stdout).into_owned(),
stderr: String::from_utf8_lossy(&output.stderr).into_owned(),
stdout: super::utils::decode_process_output(&output.stdout),
stderr: super::utils::decode_process_output(&output.stderr),
exit_code: status_to_exit_code(output.status),
})
}
@@ -569,8 +569,8 @@ pub fn exec_capture_stdin(cmd: &mut Command) -> Result<CaptureResult> {
cmd.stdin(Stdio::inherit());
let output = cmd.output().context("Failed to execute command")?;
Ok(CaptureResult {
stdout: String::from_utf8_lossy(&output.stdout).into_owned(),
stderr: String::from_utf8_lossy(&output.stderr).into_owned(),
stdout: super::utils::decode_process_output(&output.stdout),
stderr: super::utils::decode_process_output(&output.stderr),
exit_code: status_to_exit_code(output.status),
})
}
+112
View File
@@ -496,6 +496,66 @@ pub fn human_bytes(bytes: u64) -> String {
}
}
/// Decode child process output bytes, respecting the Windows console code page.
///
/// On all platforms, tries UTF-8 first. On Windows, falls back to the console's
/// output code page (e.g., GBK for Chinese locale) via `encoding_rs`. On
/// non-Windows or unknown code pages, falls back to lossy UTF-8.
pub fn decode_process_output(bytes: &[u8]) -> String {
if let Ok(s) = std::str::from_utf8(bytes) {
return s.to_owned();
}
#[cfg(windows)]
{
let cp = windows_console_output_cp();
if let Some(encoding) = codepage_to_encoding(cp) {
let (cow, _, _) = encoding.decode(bytes);
return cow.into_owned();
}
}
String::from_utf8_lossy(bytes).into_owned()
}
#[cfg(windows)]
fn windows_console_output_cp() -> u32 {
#[allow(unsafe_code)]
unsafe {
extern "system" {
fn GetConsoleOutputCP() -> u32;
}
GetConsoleOutputCP()
}
}
#[cfg(windows)]
fn codepage_to_encoding(cp: u32) -> Option<&'static encoding_rs::Encoding> {
let label = match cp {
936 | 54936 => "gbk",
950 => "big5",
932 => "shift_jis",
949 => "euc-kr",
874 => "windows-874",
1250 => "windows-1250",
1251 => "windows-1251",
1252 => "windows-1252",
1253 => "windows-1253",
1254 => "windows-1254",
1255 => "windows-1255",
1256 => "windows-1256",
1257 => "windows-1257",
1258 => "windows-1258",
28591 => "iso-8859-1",
28592 => "iso-8859-2",
20866 => "koi8-r",
21866 => "koi8-u",
65001 => return None,
_ => return None,
};
encoding_rs::Encoding::for_label(label.as_bytes())
}
#[cfg(test)]
mod tests {
use super::*;
@@ -1045,4 +1105,56 @@ mod tests {
let tmp = tempfile::tempdir().unwrap();
restrict_file(&tmp.path().join("absent.db-wal"));
}
#[test]
fn test_decode_process_output_valid_utf8() {
assert_eq!(decode_process_output(b"hello world"), "hello world");
}
#[test]
fn test_decode_process_output_chinese_utf8() {
let input = "测试中文".as_bytes();
assert_eq!(decode_process_output(input), "测试中文");
}
#[test]
fn test_decode_process_output_empty() {
assert_eq!(decode_process_output(b""), "");
}
#[test]
fn test_decode_process_output_invalid_utf8_no_panic() {
let bytes: &[u8] = &[0xFF, 0xFE, 0x41, 0x42];
let result = decode_process_output(bytes);
assert!(!result.is_empty());
}
#[cfg(windows)]
#[test]
fn test_decode_process_output_gbk() {
// "测试" in GBK encoding
let gbk_bytes: &[u8] = &[0xB2, 0xE2, 0xCA, 0xD4];
let result = decode_process_output(gbk_bytes);
assert_eq!(result, "测试");
}
#[cfg(windows)]
#[test]
fn test_codepage_to_encoding_known() {
assert!(codepage_to_encoding(936).is_some());
assert!(codepage_to_encoding(932).is_some());
assert!(codepage_to_encoding(949).is_some());
}
#[cfg(windows)]
#[test]
fn test_codepage_to_encoding_utf8_returns_none() {
assert!(codepage_to_encoding(65001).is_none());
}
#[cfg(windows)]
#[test]
fn test_codepage_to_encoding_unknown_returns_none() {
assert!(codepage_to_encoding(99999).is_none());
}
}
+4 -4
View File
@@ -1344,8 +1344,8 @@ fn run_fallback(parse_error: clap::Error) -> Result<i32> {
match result {
Ok(output) => {
let exit_code = core::utils::exit_code_from_output(&output, &raw_command);
let stdout_raw = String::from_utf8_lossy(&output.stdout);
let stderr_raw = String::from_utf8_lossy(&output.stderr);
let stdout_raw = core::utils::decode_process_output(&output.stdout);
let stderr_raw = core::utils::decode_process_output(&output.stderr);
// Merge stderr into the text to filter when filter_stderr is enabled;
// otherwise emit stderr directly so it is always visible.
@@ -2676,8 +2676,8 @@ fn run_cli() -> Result<i32> {
.join()
.map_err(|_| anyhow::anyhow!("stderr streaming thread panicked"))??;
let stdout = String::from_utf8_lossy(&stdout_bytes);
let stderr = String::from_utf8_lossy(&stderr_bytes);
let stdout = core::utils::decode_process_output(&stdout_bytes);
let stderr = core::utils::decode_process_output(&stderr_bytes);
let full_output = format!("{}{}", stdout, stderr);
// Track usage (input = output since no filtering)