From 5bd410eb516032e54f0f4b5bc34b5962c5d2785d Mon Sep 17 00:00:00 2001 From: guy oron Date: Mon, 29 Jun 2026 09:09:32 +0300 Subject: [PATCH] fix(core): decode process output using Windows console code page On Windows with non-UTF-8 console code pages (e.g., GBK for Chinese locale), child process output is mis-decoded by String::from_utf8_lossy, producing mojibake. Add decode_process_output() that detects the console output code page via GetConsoleOutputCP() and decodes with encoding_rs. Replaces from_utf8_lossy in the core capture paths (exec_capture, exec_capture_stdin, TOML filter path, proxy streaming path). Module- specific call sites left for follow-up. Fixes #2452 --- Cargo.lock | 10 ++++ Cargo.toml | 3 ++ src/core/stream.rs | 8 ++-- src/core/utils.rs | 112 +++++++++++++++++++++++++++++++++++++++++++++ src/main.rs | 8 ++-- 5 files changed, 133 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5503fca9..a1ca3a05 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -333,6 +333,15 @@ dependencies = [ "syn", ] +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + [[package]] name = "env_home" version = "0.1.0" @@ -893,6 +902,7 @@ dependencies = [ "clap", "colored", "dirs", + "encoding_rs", "flate2", "getrandom 0.4.2", "ignore", diff --git a/Cargo.toml b/Cargo.toml index d945b734..aa6dd881 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,6 +37,9 @@ automod = "1" [target.'cfg(unix)'.dependencies] libc = "0.2" +[target.'cfg(windows)'.dependencies] +encoding_rs = "0.8" + [build-dependencies] toml = "0.8" diff --git a/src/core/stream.rs b/src/core/stream.rs index d964c574..3a23fd67 100644 --- a/src/core/stream.rs +++ b/src/core/stream.rs @@ -558,8 +558,8 @@ pub fn exec_capture(cmd: &mut Command) -> Result { cmd.stdin(Stdio::null()); let output = cmd.output().context("Failed to execute command")?; Ok(CaptureResult { - stdout: String::from_utf8_lossy(&output.stdout).into_owned(), - stderr: String::from_utf8_lossy(&output.stderr).into_owned(), + stdout: super::utils::decode_process_output(&output.stdout), + stderr: super::utils::decode_process_output(&output.stderr), exit_code: status_to_exit_code(output.status), }) } @@ -569,8 +569,8 @@ pub fn exec_capture_stdin(cmd: &mut Command) -> Result { cmd.stdin(Stdio::inherit()); let output = cmd.output().context("Failed to execute command")?; Ok(CaptureResult { - stdout: String::from_utf8_lossy(&output.stdout).into_owned(), - stderr: String::from_utf8_lossy(&output.stderr).into_owned(), + stdout: super::utils::decode_process_output(&output.stdout), + stderr: super::utils::decode_process_output(&output.stderr), exit_code: status_to_exit_code(output.status), }) } diff --git a/src/core/utils.rs b/src/core/utils.rs index d10f4b01..52e4cf3f 100644 --- a/src/core/utils.rs +++ b/src/core/utils.rs @@ -496,6 +496,66 @@ pub fn human_bytes(bytes: u64) -> String { } } +/// Decode child process output bytes, respecting the Windows console code page. +/// +/// On all platforms, tries UTF-8 first. On Windows, falls back to the console's +/// output code page (e.g., GBK for Chinese locale) via `encoding_rs`. On +/// non-Windows or unknown code pages, falls back to lossy UTF-8. +pub fn decode_process_output(bytes: &[u8]) -> String { + if let Ok(s) = std::str::from_utf8(bytes) { + return s.to_owned(); + } + + #[cfg(windows)] + { + let cp = windows_console_output_cp(); + if let Some(encoding) = codepage_to_encoding(cp) { + let (cow, _, _) = encoding.decode(bytes); + return cow.into_owned(); + } + } + + String::from_utf8_lossy(bytes).into_owned() +} + +#[cfg(windows)] +fn windows_console_output_cp() -> u32 { + #[allow(unsafe_code)] + unsafe { + extern "system" { + fn GetConsoleOutputCP() -> u32; + } + GetConsoleOutputCP() + } +} + +#[cfg(windows)] +fn codepage_to_encoding(cp: u32) -> Option<&'static encoding_rs::Encoding> { + let label = match cp { + 936 | 54936 => "gbk", + 950 => "big5", + 932 => "shift_jis", + 949 => "euc-kr", + 874 => "windows-874", + 1250 => "windows-1250", + 1251 => "windows-1251", + 1252 => "windows-1252", + 1253 => "windows-1253", + 1254 => "windows-1254", + 1255 => "windows-1255", + 1256 => "windows-1256", + 1257 => "windows-1257", + 1258 => "windows-1258", + 28591 => "iso-8859-1", + 28592 => "iso-8859-2", + 20866 => "koi8-r", + 21866 => "koi8-u", + 65001 => return None, + _ => return None, + }; + encoding_rs::Encoding::for_label(label.as_bytes()) +} + #[cfg(test)] mod tests { use super::*; @@ -1045,4 +1105,56 @@ mod tests { let tmp = tempfile::tempdir().unwrap(); restrict_file(&tmp.path().join("absent.db-wal")); } + + #[test] + fn test_decode_process_output_valid_utf8() { + assert_eq!(decode_process_output(b"hello world"), "hello world"); + } + + #[test] + fn test_decode_process_output_chinese_utf8() { + let input = "测试中文".as_bytes(); + assert_eq!(decode_process_output(input), "测试中文"); + } + + #[test] + fn test_decode_process_output_empty() { + assert_eq!(decode_process_output(b""), ""); + } + + #[test] + fn test_decode_process_output_invalid_utf8_no_panic() { + let bytes: &[u8] = &[0xFF, 0xFE, 0x41, 0x42]; + let result = decode_process_output(bytes); + assert!(!result.is_empty()); + } + + #[cfg(windows)] + #[test] + fn test_decode_process_output_gbk() { + // "测试" in GBK encoding + let gbk_bytes: &[u8] = &[0xB2, 0xE2, 0xCA, 0xD4]; + let result = decode_process_output(gbk_bytes); + assert_eq!(result, "测试"); + } + + #[cfg(windows)] + #[test] + fn test_codepage_to_encoding_known() { + assert!(codepage_to_encoding(936).is_some()); + assert!(codepage_to_encoding(932).is_some()); + assert!(codepage_to_encoding(949).is_some()); + } + + #[cfg(windows)] + #[test] + fn test_codepage_to_encoding_utf8_returns_none() { + assert!(codepage_to_encoding(65001).is_none()); + } + + #[cfg(windows)] + #[test] + fn test_codepage_to_encoding_unknown_returns_none() { + assert!(codepage_to_encoding(99999).is_none()); + } } diff --git a/src/main.rs b/src/main.rs index b29cf076..30ba275b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1344,8 +1344,8 @@ fn run_fallback(parse_error: clap::Error) -> Result { match result { Ok(output) => { let exit_code = core::utils::exit_code_from_output(&output, &raw_command); - let stdout_raw = String::from_utf8_lossy(&output.stdout); - let stderr_raw = String::from_utf8_lossy(&output.stderr); + let stdout_raw = core::utils::decode_process_output(&output.stdout); + let stderr_raw = core::utils::decode_process_output(&output.stderr); // Merge stderr into the text to filter when filter_stderr is enabled; // otherwise emit stderr directly so it is always visible. @@ -2676,8 +2676,8 @@ fn run_cli() -> Result { .join() .map_err(|_| anyhow::anyhow!("stderr streaming thread panicked"))??; - let stdout = String::from_utf8_lossy(&stdout_bytes); - let stderr = String::from_utf8_lossy(&stderr_bytes); + let stdout = core::utils::decode_process_output(&stdout_bytes); + let stderr = core::utils::decode_process_output(&stderr_bytes); let full_output = format!("{}{}", stdout, stderr); // Track usage (input = output since no filtering)