mirror of
https://github.com/rtk-ai/rtk.git
synced 2026-09-19 07:33:17 +08:00
fix(core): decode process output using Windows console code page
On Windows with non-UTF-8 console code pages (e.g., GBK for Chinese locale), child process output is mis-decoded by String::from_utf8_lossy, producing mojibake. Add decode_process_output() that detects the console output code page via GetConsoleOutputCP() and decodes with encoding_rs. Replaces from_utf8_lossy in the core capture paths (exec_capture, exec_capture_stdin, TOML filter path, proxy streaming path). Module- specific call sites left for follow-up. Fixes #2452
This commit is contained in:
Generated
+10
@@ -333,6 +333,15 @@ dependencies = [
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "encoding_rs"
|
||||
version = "0.8.35"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "env_home"
|
||||
version = "0.1.0"
|
||||
@@ -893,6 +902,7 @@ dependencies = [
|
||||
"clap",
|
||||
"colored",
|
||||
"dirs",
|
||||
"encoding_rs",
|
||||
"flate2",
|
||||
"getrandom 0.4.2",
|
||||
"ignore",
|
||||
|
||||
@@ -37,6 +37,9 @@ automod = "1"
|
||||
[target.'cfg(unix)'.dependencies]
|
||||
libc = "0.2"
|
||||
|
||||
[target.'cfg(windows)'.dependencies]
|
||||
encoding_rs = "0.8"
|
||||
|
||||
[build-dependencies]
|
||||
toml = "0.8"
|
||||
|
||||
|
||||
+4
-4
@@ -558,8 +558,8 @@ pub fn exec_capture(cmd: &mut Command) -> Result<CaptureResult> {
|
||||
cmd.stdin(Stdio::null());
|
||||
let output = cmd.output().context("Failed to execute command")?;
|
||||
Ok(CaptureResult {
|
||||
stdout: String::from_utf8_lossy(&output.stdout).into_owned(),
|
||||
stderr: String::from_utf8_lossy(&output.stderr).into_owned(),
|
||||
stdout: super::utils::decode_process_output(&output.stdout),
|
||||
stderr: super::utils::decode_process_output(&output.stderr),
|
||||
exit_code: status_to_exit_code(output.status),
|
||||
})
|
||||
}
|
||||
@@ -569,8 +569,8 @@ pub fn exec_capture_stdin(cmd: &mut Command) -> Result<CaptureResult> {
|
||||
cmd.stdin(Stdio::inherit());
|
||||
let output = cmd.output().context("Failed to execute command")?;
|
||||
Ok(CaptureResult {
|
||||
stdout: String::from_utf8_lossy(&output.stdout).into_owned(),
|
||||
stderr: String::from_utf8_lossy(&output.stderr).into_owned(),
|
||||
stdout: super::utils::decode_process_output(&output.stdout),
|
||||
stderr: super::utils::decode_process_output(&output.stderr),
|
||||
exit_code: status_to_exit_code(output.status),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -496,6 +496,66 @@ pub fn human_bytes(bytes: u64) -> String {
|
||||
}
|
||||
}
|
||||
|
||||
/// Decode child process output bytes, respecting the Windows console code page.
|
||||
///
|
||||
/// On all platforms, tries UTF-8 first. On Windows, falls back to the console's
|
||||
/// output code page (e.g., GBK for Chinese locale) via `encoding_rs`. On
|
||||
/// non-Windows or unknown code pages, falls back to lossy UTF-8.
|
||||
pub fn decode_process_output(bytes: &[u8]) -> String {
|
||||
if let Ok(s) = std::str::from_utf8(bytes) {
|
||||
return s.to_owned();
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
{
|
||||
let cp = windows_console_output_cp();
|
||||
if let Some(encoding) = codepage_to_encoding(cp) {
|
||||
let (cow, _, _) = encoding.decode(bytes);
|
||||
return cow.into_owned();
|
||||
}
|
||||
}
|
||||
|
||||
String::from_utf8_lossy(bytes).into_owned()
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
fn windows_console_output_cp() -> u32 {
|
||||
#[allow(unsafe_code)]
|
||||
unsafe {
|
||||
extern "system" {
|
||||
fn GetConsoleOutputCP() -> u32;
|
||||
}
|
||||
GetConsoleOutputCP()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
fn codepage_to_encoding(cp: u32) -> Option<&'static encoding_rs::Encoding> {
|
||||
let label = match cp {
|
||||
936 | 54936 => "gbk",
|
||||
950 => "big5",
|
||||
932 => "shift_jis",
|
||||
949 => "euc-kr",
|
||||
874 => "windows-874",
|
||||
1250 => "windows-1250",
|
||||
1251 => "windows-1251",
|
||||
1252 => "windows-1252",
|
||||
1253 => "windows-1253",
|
||||
1254 => "windows-1254",
|
||||
1255 => "windows-1255",
|
||||
1256 => "windows-1256",
|
||||
1257 => "windows-1257",
|
||||
1258 => "windows-1258",
|
||||
28591 => "iso-8859-1",
|
||||
28592 => "iso-8859-2",
|
||||
20866 => "koi8-r",
|
||||
21866 => "koi8-u",
|
||||
65001 => return None,
|
||||
_ => return None,
|
||||
};
|
||||
encoding_rs::Encoding::for_label(label.as_bytes())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -1045,4 +1105,56 @@ mod tests {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
restrict_file(&tmp.path().join("absent.db-wal"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_decode_process_output_valid_utf8() {
|
||||
assert_eq!(decode_process_output(b"hello world"), "hello world");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_decode_process_output_chinese_utf8() {
|
||||
let input = "测试中文".as_bytes();
|
||||
assert_eq!(decode_process_output(input), "测试中文");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_decode_process_output_empty() {
|
||||
assert_eq!(decode_process_output(b""), "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_decode_process_output_invalid_utf8_no_panic() {
|
||||
let bytes: &[u8] = &[0xFF, 0xFE, 0x41, 0x42];
|
||||
let result = decode_process_output(bytes);
|
||||
assert!(!result.is_empty());
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
#[test]
|
||||
fn test_decode_process_output_gbk() {
|
||||
// "测试" in GBK encoding
|
||||
let gbk_bytes: &[u8] = &[0xB2, 0xE2, 0xCA, 0xD4];
|
||||
let result = decode_process_output(gbk_bytes);
|
||||
assert_eq!(result, "测试");
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
#[test]
|
||||
fn test_codepage_to_encoding_known() {
|
||||
assert!(codepage_to_encoding(936).is_some());
|
||||
assert!(codepage_to_encoding(932).is_some());
|
||||
assert!(codepage_to_encoding(949).is_some());
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
#[test]
|
||||
fn test_codepage_to_encoding_utf8_returns_none() {
|
||||
assert!(codepage_to_encoding(65001).is_none());
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
#[test]
|
||||
fn test_codepage_to_encoding_unknown_returns_none() {
|
||||
assert!(codepage_to_encoding(99999).is_none());
|
||||
}
|
||||
}
|
||||
|
||||
+4
-4
@@ -1344,8 +1344,8 @@ fn run_fallback(parse_error: clap::Error) -> Result<i32> {
|
||||
match result {
|
||||
Ok(output) => {
|
||||
let exit_code = core::utils::exit_code_from_output(&output, &raw_command);
|
||||
let stdout_raw = String::from_utf8_lossy(&output.stdout);
|
||||
let stderr_raw = String::from_utf8_lossy(&output.stderr);
|
||||
let stdout_raw = core::utils::decode_process_output(&output.stdout);
|
||||
let stderr_raw = core::utils::decode_process_output(&output.stderr);
|
||||
|
||||
// Merge stderr into the text to filter when filter_stderr is enabled;
|
||||
// otherwise emit stderr directly so it is always visible.
|
||||
@@ -2676,8 +2676,8 @@ fn run_cli() -> Result<i32> {
|
||||
.join()
|
||||
.map_err(|_| anyhow::anyhow!("stderr streaming thread panicked"))??;
|
||||
|
||||
let stdout = String::from_utf8_lossy(&stdout_bytes);
|
||||
let stderr = String::from_utf8_lossy(&stderr_bytes);
|
||||
let stdout = core::utils::decode_process_output(&stdout_bytes);
|
||||
let stderr = core::utils::decode_process_output(&stderr_bytes);
|
||||
let full_output = format!("{}{}", stdout, stderr);
|
||||
|
||||
// Track usage (input = output since no filtering)
|
||||
|
||||
Reference in New Issue
Block a user