Detailed changes
@@ -153,7 +153,7 @@ dependencies = [
"db",
"derive_more 0.99.20",
"editor",
- "encoding",
+ "encoding_rs",
"env_logger 0.11.8",
"fs",
"futures 0.3.31",
@@ -3355,7 +3355,7 @@ dependencies = [
"dashmap 6.1.0",
"debugger_ui",
"editor",
- "encoding",
+ "encoding_rs",
"envy",
"extension",
"file_finder",
@@ -3722,7 +3722,7 @@ dependencies = [
"dirs 4.0.0",
"edit_prediction",
"editor",
- "encoding",
+ "encoding_rs",
"fs",
"futures 0.3.31",
"gpui",
@@ -5514,70 +5514,6 @@ version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0"
-[[package]]
-name = "encoding"
-version = "0.2.33"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6b0d943856b990d12d3b55b359144ff341533e516d94098b1d3fc1ac666d36ec"
-dependencies = [
- "encoding-index-japanese",
- "encoding-index-korean",
- "encoding-index-simpchinese",
- "encoding-index-singlebyte",
- "encoding-index-tradchinese",
-]
-
-[[package]]
-name = "encoding-index-japanese"
-version = "1.20141219.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "04e8b2ff42e9a05335dbf8b5c6f7567e5591d0d916ccef4e0b1710d32a0d0c91"
-dependencies = [
- "encoding_index_tests",
-]
-
-[[package]]
-name = "encoding-index-korean"
-version = "1.20141219.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4dc33fb8e6bcba213fe2f14275f0963fd16f0a02c878e3095ecfdf5bee529d81"
-dependencies = [
- "encoding_index_tests",
-]
-
-[[package]]
-name = "encoding-index-simpchinese"
-version = "1.20141219.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d87a7194909b9118fc707194baa434a4e3b0fb6a5a757c73c3adb07aa25031f7"
-dependencies = [
- "encoding_index_tests",
-]
-
-[[package]]
-name = "encoding-index-singlebyte"
-version = "1.20141219.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3351d5acffb224af9ca265f435b859c7c01537c0849754d3db3fdf2bfe2ae84a"
-dependencies = [
- "encoding_index_tests",
-]
-
-[[package]]
-name = "encoding-index-tradchinese"
-version = "1.20141219.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fd0e20d5688ce3cab59eb3ef3a2083a5c77bf496cb798dc6fcdb75f323890c18"
-dependencies = [
- "encoding_index_tests",
-]
-
-[[package]]
-name = "encoding_index_tests"
-version = "0.1.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a246d82be1c9d791c5dfde9a2bd045fc3cbba3fa2b11ad558f27d01712f00569"
-
[[package]]
name = "encoding_rs"
version = "0.8.35"
@@ -5593,7 +5529,7 @@ version = "0.1.0"
dependencies = [
"anyhow",
"editor",
- "encoding",
+ "encoding_rs",
"fuzzy",
"gpui",
"language",
@@ -5982,7 +5918,7 @@ dependencies = [
"criterion",
"ctor",
"dap",
- "encoding",
+ "encoding_rs",
"extension",
"fs",
"futures 0.3.31",
@@ -6482,7 +6418,7 @@ dependencies = [
"async-trait",
"cocoa 0.26.0",
"collections",
- "encoding",
+ "encoding_rs",
"fsevent",
"futures 0.3.31",
"git",
@@ -7185,7 +7121,7 @@ dependencies = [
"ctor",
"db",
"editor",
- "encoding",
+ "encoding_rs",
"futures 0.3.31",
"fuzzy",
"git",
@@ -8860,7 +8796,7 @@ dependencies = [
"ctor",
"diffy",
"ec4rs",
- "encoding",
+ "encoding_rs",
"fs",
"futures 0.3.31",
"fuzzy",
@@ -13076,7 +13012,7 @@ dependencies = [
"context_server",
"dap",
"dap_adapters",
- "encoding",
+ "encoding_rs",
"extension",
"fancy-regex 0.14.0",
"fs",
@@ -14052,7 +13988,7 @@ dependencies = [
"dap_adapters",
"debug_adapter_extension",
"editor",
- "encoding",
+ "encoding_rs",
"env_logger 0.11.8",
"extension",
"extension_host",
@@ -20817,7 +20753,7 @@ dependencies = [
"component",
"dap",
"db",
- "encoding",
+ "encoding_rs",
"fs",
"futures 0.3.31",
"gpui",
@@ -20860,7 +20796,7 @@ dependencies = [
"async-lock 2.8.0",
"clock",
"collections",
- "encoding",
+ "encoding_rs",
"fs",
"futures 0.3.31",
"fuzzy",
@@ -21270,7 +21206,7 @@ dependencies = [
"diagnostics",
"edit_prediction_button",
"editor",
- "encoding",
+ "encoding_rs",
"encodings",
"env_logger 0.11.8",
"extension",
@@ -498,7 +498,7 @@ documented = "0.9.1"
dotenvy = "0.15.0"
ec4rs = "1.1"
emojis = "0.6.1"
-encoding = "0.2.33"
+encoding_rs = "0.8"
env_logger = "0.11"
exec = "0.3.1"
fancy-regex = "0.14.0"
@@ -563,7 +563,7 @@ mod tests {
use super::*;
use crate::{ContextServerRegistry, Templates};
use client::TelemetrySettings;
- use encoding::all::UTF_8;
+ use encoding_rs::UTF_8;
use fs::{Fs, encodings::EncodingWrapper};
use gpui::{TestAppContext, UpdateGlobal};
use language_model::fake_provider::FakeLanguageModel;
@@ -32,7 +32,7 @@ cloud_llm_client.workspace = true
collections.workspace = true
context_server.workspace = true
db.workspace = true
-encoding.workspace = true
+encoding_rs.workspace = true
fs.workspace = true
futures.workspace = true
git.workspace = true
@@ -28,7 +28,7 @@ component.workspace = true
derive_more.workspace = true
diffy = "0.4.2"
editor.workspace = true
-encoding.workspace = true
+encoding_rs.workspace = true
feature_flags.workspace = true
futures.workspace = true
gpui.workspace = true
@@ -1231,7 +1231,7 @@ mod tests {
use super::*;
use ::fs::{Fs, encodings::EncodingWrapper};
use client::TelemetrySettings;
- use encoding::all::UTF_8;
+ use encoding_rs::UTF_8;
use gpui::{TestAppContext, UpdateGlobal};
use language_model::fake_provider::FakeLanguageModel;
use serde_json::json;
@@ -31,7 +31,7 @@ chrono.workspace = true
clock.workspace = true
collections.workspace = true
dashmap.workspace = true
-encoding.workspace = true
+encoding_rs.workspace = true
envy = "0.4.2"
futures.workspace = true
gpui.workspace = true
@@ -12,7 +12,7 @@ use buffer_diff::{DiffHunkSecondaryStatus, DiffHunkStatus, assert_hunks};
use call::{ActiveCall, ParticipantLocation, Room, room};
use client::{RECEIVE_TIMEOUT, User};
use collections::{HashMap, HashSet};
-use encoding::all::UTF_8;
+use encoding_rs::UTF_8;
use fs::{FakeFs, Fs as _, RemoveOptions, encodings::EncodingWrapper};
use futures::{StreamExt as _, channel::mpsc};
use git::{
@@ -5,7 +5,7 @@ use async_trait::async_trait;
use call::ActiveCall;
use collections::{BTreeMap, HashMap};
use editor::Bias;
-use encoding::all::UTF_8;
+use encoding_rs::UTF_8;
use fs::{FakeFs, Fs as _, encodings::EncodingWrapper};
use git::status::{FileStatus, StatusCode, TrackedStatus, UnmergedStatus, UnmergedStatusCode};
use gpui::{BackgroundExecutor, Entity, TestAppContext};
@@ -30,7 +30,7 @@ client.workspace = true
collections.workspace = true
command_palette_hooks.workspace = true
dirs.workspace = true
-encoding.workspace = true
+encoding_rs.workspace = true
fs.workspace = true
futures.workspace = true
gpui.workspace = true
@@ -1241,7 +1241,7 @@ async fn get_copilot_lsp(fs: Arc<dyn Fs>, node_runtime: NodeRuntime) -> anyhow::
#[cfg(test)]
mod tests {
use super::*;
- use encoding::Encoding;
+ use encoding_rs::Encoding;
use gpui::TestAppContext;
use util::{path, paths::PathStyle, rel_path::rel_path};
@@ -1460,7 +1460,7 @@ mod tests {
unimplemented!()
}
- fn load_with_encoding(&self, _: &App, _: &'static dyn Encoding) -> Task<Result<String>> {
+ fn load_with_encoding(&self, _: &App, _: &'static Encoding) -> Task<Result<String>> {
unimplemented!()
}
}
@@ -7,7 +7,7 @@ edition.workspace = true
[dependencies]
anyhow.workspace = true
editor.workspace = true
-encoding.workspace = true
+encoding_rs.workspace = true
fuzzy.workspace = true
gpui.workspace = true
language.workspace = true
@@ -1,13 +1,6 @@
///! A crate for handling file encodings in the text editor.
use editor::{Editor, EditorSettings};
-use encoding::Encoding;
-use encoding::all::{
- BIG5_2003, EUC_JP, GB18030, GBK, HZ, IBM866, ISO_2022_JP, ISO_8859_1, ISO_8859_2, ISO_8859_3,
- ISO_8859_4, ISO_8859_5, ISO_8859_6, ISO_8859_7, ISO_8859_8, ISO_8859_10, ISO_8859_13,
- ISO_8859_14, ISO_8859_15, ISO_8859_16, KOI8_R, KOI8_U, MAC_CYRILLIC, MAC_ROMAN, UTF_8,
- UTF_16BE, UTF_16LE, WINDOWS_874, WINDOWS_949, WINDOWS_1250, WINDOWS_1251, WINDOWS_1252,
- WINDOWS_1253, WINDOWS_1254, WINDOWS_1255, WINDOWS_1256, WINDOWS_1257, WINDOWS_1258,
-};
+use encoding_rs::Encoding;
use gpui::{ClickEvent, Entity, Subscription, WeakEntity};
use settings::Settings;
use ui::{Button, ButtonCommon, Context, LabelSize, Render, Tooltip, Window, div};
@@ -18,7 +11,7 @@ use crate::selectors::save_or_reopen::EncodingSaveOrReopenSelector;
/// A status bar item that shows the current file encoding and allows changing it.
pub struct EncodingIndicator {
- pub encoding: Option<&'static dyn Encoding>,
+ pub encoding: Option<&'static Encoding>,
pub workspace: WeakEntity<Workspace>,
observe: Option<Subscription>, // Subscription to observe changes in the active editor
show: bool, // Whether to show the indicator or not, based on whether an editor is active
@@ -37,7 +30,7 @@ impl Render for EncodingIndicator {
}
status_element.child(
- Button::new("encoding", encoding_name(self.encoding.unwrap_or(UTF_8)))
+ Button::new("encoding", encoding_name(self.encoding.unwrap_or(encoding_rs::UTF_8)))
.label_size(LabelSize::Small)
.tooltip(Tooltip::text("Select Encoding"))
.on_click(cx.listener(|indicator, _: &ClickEvent, window, cx| {
@@ -54,7 +47,7 @@ impl Render for EncodingIndicator {
impl EncodingIndicator {
pub fn new(
- encoding: Option<&'static dyn encoding::Encoding>,
+ encoding: Option<&'static Encoding>,
workspace: WeakEntity<Workspace>,
observe: Option<Subscription>,
) -> EncodingIndicator {
@@ -105,140 +98,117 @@ impl StatusItemView for EncodingIndicator {
}
/// Get a human-readable name for the given encoding.
-pub fn encoding_name(encoding: &'static dyn Encoding) -> String {
+pub fn encoding_name(encoding: &'static Encoding) -> String {
let name = encoding.name();
- match () {
- () if name == UTF_8.name() => "UTF-8",
- () if name == UTF_16LE.name() => "UTF-16 LE",
- () if name == UTF_16BE.name() => "UTF-16 BE",
- () if name == IBM866.name() => "IBM866",
- () if name == ISO_8859_1.name() => "ISO 8859-1",
- () if name == ISO_8859_2.name() => "ISO 8859-2",
- () if name == ISO_8859_3.name() => "ISO 8859-3",
- () if name == ISO_8859_4.name() => "ISO 8859-4",
- () if name == ISO_8859_5.name() => "ISO 8859-5",
- () if name == ISO_8859_6.name() => "ISO 8859-6",
- () if name == ISO_8859_7.name() => "ISO 8859-7",
- () if name == ISO_8859_8.name() => "ISO 8859-8",
- () if name == ISO_8859_10.name() => "ISO 8859-10",
- () if name == ISO_8859_13.name() => "ISO 8859-13",
- () if name == ISO_8859_14.name() => "ISO 8859-14",
- () if name == ISO_8859_15.name() => "ISO 8859-15",
- () if name == ISO_8859_16.name() => "ISO 8859-16",
- () if name == KOI8_R.name() => "KOI8-R",
- () if name == KOI8_U.name() => "KOI8-U",
- () if name == MAC_ROMAN.name() => "MacRoman",
- () if name == MAC_CYRILLIC.name() => "Mac Cyrillic",
- () if name == WINDOWS_874.name() => "Windows-874",
- () if name == WINDOWS_1250.name() => "Windows-1250",
- () if name == WINDOWS_1251.name() => "Windows-1251",
- () if name == WINDOWS_1252.name() => "Windows-1252",
- () if name == WINDOWS_1253.name() => "Windows-1253",
- () if name == WINDOWS_1254.name() => "Windows-1254",
- () if name == WINDOWS_1255.name() => "Windows-1255",
- () if name == WINDOWS_1256.name() => "Windows-1256",
- () if name == WINDOWS_1257.name() => "Windows-1257",
- () if name == WINDOWS_1258.name() => "Windows-1258",
- () if name == WINDOWS_949.name() => "Windows-949",
- () if name == EUC_JP.name() => "EUC-JP",
- () if name == ISO_2022_JP.name() => "ISO 2022-JP",
- () if name == GBK.name() => "GBK",
- () if name == GB18030.name() => "GB18030",
- () if name == BIG5_2003.name() => "Big5",
- () if name == HZ.name() => "HZ-GB-2312",
- _ => "",
+ match name {
+ "UTF-8" => "UTF-8",
+ "windows-1252" => "Windows-1252",
+ "windows-1251" => "Windows-1251",
+ "windows-1250" => "Windows-1250",
+ "ISO-8859-2" => "ISO 8859-2",
+ "ISO-8859-3" => "ISO 8859-3",
+ "ISO-8859-4" => "ISO 8859-4",
+ "ISO-8859-5" => "ISO 8859-5",
+ "ISO-8859-6" => "ISO 8859-6",
+ "ISO-8859-7" => "ISO 8859-7",
+ "ISO-8859-8" => "ISO 8859-8",
+ "ISO-8859-13" => "ISO 8859-13",
+ "ISO-8859-15" => "ISO 8859-15",
+ "KOI8-R" => "KOI8-R",
+ "KOI8-U" => "KOI8-U",
+ "macintosh" => "MacRoman",
+ "x-mac-cyrillic" => "Mac Cyrillic",
+ "windows-874" => "Windows-874",
+ "windows-1253" => "Windows-1253",
+ "windows-1254" => "Windows-1254",
+ "windows-1255" => "Windows-1255",
+ "windows-1256" => "Windows-1256",
+ "windows-1257" => "Windows-1257",
+ "windows-1258" => "Windows-1258",
+ "EUC-KR" => "Windows-949",
+ "EUC-JP" => "EUC-JP",
+ "ISO-2022-JP" => "ISO 2022-JP",
+ "GBK" => "GBK",
+ "gb18030" => "GB18030",
+ "Big5" => "Big5",
+ _ => name,
}
.to_string()
}
/// Get an encoding from its index in the predefined list.
/// If the index is out of range, UTF-8 is returned as a default.
-pub fn encoding_from_index(index: usize) -> &'static dyn Encoding {
+pub fn encoding_from_index(index: usize) -> &'static Encoding {
match index {
- 0 => UTF_8,
- 1 => UTF_16LE,
- 2 => UTF_16BE,
- 3 => IBM866,
- 4 => ISO_8859_1,
- 5 => ISO_8859_2,
- 6 => ISO_8859_3,
- 7 => ISO_8859_4,
- 8 => ISO_8859_5,
- 9 => ISO_8859_6,
- 10 => ISO_8859_7,
- 11 => ISO_8859_8,
- 12 => ISO_8859_10,
- 13 => ISO_8859_13,
- 14 => ISO_8859_14,
- 15 => ISO_8859_15,
- 16 => ISO_8859_16,
- 17 => KOI8_R,
- 18 => KOI8_U,
- 19 => MAC_ROMAN,
- 20 => MAC_CYRILLIC,
- 21 => WINDOWS_874,
- 22 => WINDOWS_1250,
- 23 => WINDOWS_1251,
- 24 => WINDOWS_1252,
- 25 => WINDOWS_1253,
- 26 => WINDOWS_1254,
- 27 => WINDOWS_1255,
- 28 => WINDOWS_1256,
- 29 => WINDOWS_1257,
- 30 => WINDOWS_1258,
- 31 => WINDOWS_949,
- 32 => EUC_JP,
- 33 => ISO_2022_JP,
- 34 => GBK,
- 35 => GB18030,
- 36 => BIG5_2003,
- 37 => HZ,
- _ => UTF_8,
+ 0 => encoding_rs::UTF_8,
+ 1 => encoding_rs::WINDOWS_1252,
+ 2 => encoding_rs::WINDOWS_1251,
+ 3 => encoding_rs::WINDOWS_1250,
+ 4 => encoding_rs::ISO_8859_2,
+ 5 => encoding_rs::ISO_8859_3,
+ 6 => encoding_rs::ISO_8859_4,
+ 7 => encoding_rs::ISO_8859_5,
+ 8 => encoding_rs::ISO_8859_6,
+ 9 => encoding_rs::ISO_8859_7,
+ 10 => encoding_rs::ISO_8859_8,
+ 11 => encoding_rs::ISO_8859_13,
+ 12 => encoding_rs::ISO_8859_15,
+ 13 => encoding_rs::KOI8_R,
+ 14 => encoding_rs::KOI8_U,
+ 15 => encoding_rs::MACINTOSH,
+ 16 => encoding_rs::X_MAC_CYRILLIC,
+ 17 => encoding_rs::WINDOWS_874,
+ 18 => encoding_rs::WINDOWS_1253,
+ 19 => encoding_rs::WINDOWS_1254,
+ 20 => encoding_rs::WINDOWS_1255,
+ 21 => encoding_rs::WINDOWS_1256,
+ 22 => encoding_rs::WINDOWS_1257,
+ 23 => encoding_rs::WINDOWS_1258,
+ 24 => encoding_rs::EUC_KR,
+ 25 => encoding_rs::EUC_JP,
+ 26 => encoding_rs::ISO_2022_JP,
+ 27 => encoding_rs::GBK,
+ 28 => encoding_rs::GB18030,
+ 29 => encoding_rs::BIG5,
+ _ => encoding_rs::UTF_8,
}
}
/// Get an encoding from its name.
-pub fn encoding_from_name(name: &str) -> &'static dyn Encoding {
+pub fn encoding_from_name(name: &str) -> &'static Encoding {
match name {
- "UTF-8" => UTF_8,
- "UTF-16 LE" => UTF_16LE,
- "UTF-16 BE" => UTF_16BE,
- "IBM866" => IBM866,
- "ISO 8859-1" => ISO_8859_1,
- "ISO 8859-2" => ISO_8859_2,
- "ISO 8859-3" => ISO_8859_3,
- "ISO 8859-4" => ISO_8859_4,
- "ISO 8859-5" => ISO_8859_5,
- "ISO 8859-6" => ISO_8859_6,
- "ISO 8859-7" => ISO_8859_7,
- "ISO 8859-8" => ISO_8859_8,
- "ISO 8859-10" => ISO_8859_10,
- "ISO 8859-13" => ISO_8859_13,
- "ISO 8859-14" => ISO_8859_14,
- "ISO 8859-15" => ISO_8859_15,
- "ISO 8859-16" => ISO_8859_16,
- "KOI8-R" => KOI8_R,
- "KOI8-U" => KOI8_U,
- "MacRoman" => MAC_ROMAN,
- "Mac Cyrillic" => MAC_CYRILLIC,
- "Windows-874" => WINDOWS_874,
- "Windows-1250" => WINDOWS_1250,
- "Windows-1251" => WINDOWS_1251,
- "Windows-1252" => WINDOWS_1252,
- "Windows-1253" => WINDOWS_1253,
- "Windows-1254" => WINDOWS_1254,
- "Windows-1255" => WINDOWS_1255,
- "Windows-1256" => WINDOWS_1256,
- "Windows-1257" => WINDOWS_1257,
- "Windows-1258" => WINDOWS_1258,
- "Windows-949" => WINDOWS_949,
- "EUC-JP" => EUC_JP,
- "ISO 2022-JP" => ISO_2022_JP,
- "GBK" => GBK,
- "GB18030" => GB18030,
- "Big5" => BIG5_2003,
- "HZ-GB-2312" => HZ,
- _ => UTF_8, // Default to UTF-8 for unknown names
+ "UTF-8" => encoding_rs::UTF_8,
+ "Windows-1252" => encoding_rs::WINDOWS_1252,
+ "Windows-1251" => encoding_rs::WINDOWS_1251,
+ "Windows-1250" => encoding_rs::WINDOWS_1250,
+ "ISO 8859-2" => encoding_rs::ISO_8859_2,
+ "ISO 8859-3" => encoding_rs::ISO_8859_3,
+ "ISO 8859-4" => encoding_rs::ISO_8859_4,
+ "ISO 8859-5" => encoding_rs::ISO_8859_5,
+ "ISO 8859-6" => encoding_rs::ISO_8859_6,
+ "ISO 8859-7" => encoding_rs::ISO_8859_7,
+ "ISO 8859-8" => encoding_rs::ISO_8859_8,
+ "ISO 8859-13" => encoding_rs::ISO_8859_13,
+ "ISO 8859-15" => encoding_rs::ISO_8859_15,
+ "KOI8-R" => encoding_rs::KOI8_R,
+ "KOI8-U" => encoding_rs::KOI8_U,
+ "MacRoman" => encoding_rs::MACINTOSH,
+ "Mac Cyrillic" => encoding_rs::X_MAC_CYRILLIC,
+ "Windows-874" => encoding_rs::WINDOWS_874,
+ "Windows-1253" => encoding_rs::WINDOWS_1253,
+ "Windows-1254" => encoding_rs::WINDOWS_1254,
+ "Windows-1255" => encoding_rs::WINDOWS_1255,
+ "Windows-1256" => encoding_rs::WINDOWS_1256,
+ "Windows-1257" => encoding_rs::WINDOWS_1257,
+ "Windows-1258" => encoding_rs::WINDOWS_1258,
+ "Windows-949" => encoding_rs::EUC_KR,
+ "EUC-JP" => encoding_rs::EUC_JP,
+ "ISO 2022-JP" => encoding_rs::ISO_2022_JP,
+ "GBK" => encoding_rs::GBK,
+ "GB18030" => encoding_rs::GB18030,
+ "Big5" => encoding_rs::BIG5,
+ "HZ-GB-2312" => encoding_rs::UTF_8, // encoding_rs doesn't support HZ, fallback to UTF-8
+ _ => encoding_rs::UTF_8, // Default to UTF-8 for unknown names
}
}
@@ -320,43 +320,35 @@ pub mod encoding {
current_selection: 0,
encodings: vec![
StringMatchCandidate::new(0, "UTF-8"),
- StringMatchCandidate::new(1, "UTF-16 LE"),
- StringMatchCandidate::new(2, "UTF-16 BE"),
- StringMatchCandidate::new(3, "IBM866"),
- StringMatchCandidate::new(4, "ISO 8859-1"),
- StringMatchCandidate::new(5, "ISO 8859-2"),
- StringMatchCandidate::new(6, "ISO 8859-3"),
- StringMatchCandidate::new(7, "ISO 8859-4"),
- StringMatchCandidate::new(8, "ISO 8859-5"),
- StringMatchCandidate::new(9, "ISO 8859-6"),
- StringMatchCandidate::new(10, "ISO 8859-7"),
- StringMatchCandidate::new(11, "ISO 8859-8"),
- StringMatchCandidate::new(12, "ISO 8859-10"),
- StringMatchCandidate::new(13, "ISO 8859-13"),
- StringMatchCandidate::new(14, "ISO 8859-14"),
- StringMatchCandidate::new(15, "ISO 8859-15"),
- StringMatchCandidate::new(16, "ISO 8859-16"),
- StringMatchCandidate::new(17, "KOI8-R"),
- StringMatchCandidate::new(18, "KOI8-U"),
- StringMatchCandidate::new(19, "MacRoman"),
- StringMatchCandidate::new(20, "Mac Cyrillic"),
- StringMatchCandidate::new(21, "Windows-874"),
- StringMatchCandidate::new(22, "Windows-1250"),
- StringMatchCandidate::new(23, "Windows-1251"),
- StringMatchCandidate::new(24, "Windows-1252"),
- StringMatchCandidate::new(25, "Windows-1253"),
- StringMatchCandidate::new(26, "Windows-1254"),
- StringMatchCandidate::new(27, "Windows-1255"),
- StringMatchCandidate::new(28, "Windows-1256"),
- StringMatchCandidate::new(29, "Windows-1257"),
- StringMatchCandidate::new(30, "Windows-1258"),
- StringMatchCandidate::new(31, "Windows-949"),
- StringMatchCandidate::new(32, "EUC-JP"),
- StringMatchCandidate::new(33, "ISO 2022-JP"),
- StringMatchCandidate::new(34, "GBK"),
- StringMatchCandidate::new(35, "GB18030"),
- StringMatchCandidate::new(36, "Big5"),
- StringMatchCandidate::new(37, "HZ-GB-2312"),
+ StringMatchCandidate::new(1, "Windows-1252"),
+ StringMatchCandidate::new(2, "Windows-1251"),
+ StringMatchCandidate::new(3, "Windows-1250"),
+ StringMatchCandidate::new(4, "ISO 8859-2"),
+ StringMatchCandidate::new(5, "ISO 8859-3"),
+ StringMatchCandidate::new(6, "ISO 8859-4"),
+ StringMatchCandidate::new(7, "ISO 8859-5"),
+ StringMatchCandidate::new(8, "ISO 8859-6"),
+ StringMatchCandidate::new(9, "ISO 8859-7"),
+ StringMatchCandidate::new(10, "ISO 8859-8"),
+ StringMatchCandidate::new(11, "ISO 8859-13"),
+ StringMatchCandidate::new(12, "ISO 8859-15"),
+ StringMatchCandidate::new(13, "KOI8-R"),
+ StringMatchCandidate::new(14, "KOI8-U"),
+ StringMatchCandidate::new(15, "MacRoman"),
+ StringMatchCandidate::new(16, "Mac Cyrillic"),
+ StringMatchCandidate::new(17, "Windows-874"),
+ StringMatchCandidate::new(18, "Windows-1253"),
+ StringMatchCandidate::new(19, "Windows-1254"),
+ StringMatchCandidate::new(20, "Windows-1255"),
+ StringMatchCandidate::new(21, "Windows-1256"),
+ StringMatchCandidate::new(22, "Windows-1257"),
+ StringMatchCandidate::new(23, "Windows-1258"),
+ StringMatchCandidate::new(24, "Windows-949"),
+ StringMatchCandidate::new(25, "EUC-JP"),
+ StringMatchCandidate::new(26, "ISO 2022-JP"),
+ StringMatchCandidate::new(27, "GBK"),
+ StringMatchCandidate::new(28, "GB18030"),
+ StringMatchCandidate::new(29, "Big5"),
],
matches: Vec::new(),
selector,
@@ -414,7 +406,7 @@ pub mod encoding {
&query,
true,
false,
- 38,
+ 30,
&AtomicBool::new(false),
executor,
)
@@ -23,7 +23,7 @@ async-trait.workspace = true
client.workspace = true
collections.workspace = true
dap.workspace = true
-encoding.workspace = true
+encoding_rs.workspace = true
extension.workspace = true
fs.workspace = true
futures.workspace = true
@@ -12,7 +12,7 @@ use async_tar::Archive;
use client::ExtensionProvides;
use client::{Client, ExtensionMetadata, GetExtensionsResponse, proto, telemetry::Telemetry};
use collections::{BTreeMap, BTreeSet, HashMap, HashSet, btree_map};
-use encoding::all::UTF_8;
+use encoding_rs::UTF_8;
pub use extension::ExtensionManifest;
use extension::extension_builder::{CompileExtensionOptions, ExtensionBuilder};
use extension::{
@@ -16,7 +16,7 @@ anyhow.workspace = true
async-tar.workspace = true
async-trait.workspace = true
collections.workspace = true
-encoding.workspace = true
+encoding_rs.workspace = true
futures.workspace = true
git.workspace = true
gpui.workspace = true
@@ -1,13 +1,13 @@
-//! Encoding and decoding utilities using the `encoding` crate.
+//! Encoding and decoding utilities using the `encoding_rs` crate.
use std::fmt::Debug;
use anyhow::{Error, Result};
-use encoding::Encoding;
+use encoding_rs::Encoding;
use serde::{Deserialize, de::Visitor};
-/// A wrapper around `encoding::Encoding` to implement `Send` and `Sync`.
+/// A wrapper around `encoding_rs::Encoding` to implement `Send` and `Sync`.
/// Since the reference is static, it is safe to send it across threads.
-pub struct EncodingWrapper(&'static dyn Encoding);
+pub struct EncodingWrapper(&'static Encoding);
impl Debug for EncodingWrapper {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
@@ -28,14 +28,14 @@ impl<'vi> Visitor<'vi> for EncodingWrapperVisitor {
fn visit_str<E: serde::de::Error>(self, encoding: &str) -> Result<EncodingWrapper, E> {
Ok(EncodingWrapper(
- encoding::label::encoding_from_whatwg_label(encoding)
+ Encoding::for_label(encoding.as_bytes())
.ok_or_else(|| serde::de::Error::custom("Invalid Encoding"))?,
))
}
fn visit_string<E: serde::de::Error>(self, encoding: String) -> Result<EncodingWrapper, E> {
Ok(EncodingWrapper(
- encoding::label::encoding_from_whatwg_label(&encoding)
+ Encoding::for_label(encoding.as_bytes())
.ok_or_else(|| serde::de::Error::custom("Invalid Encoding"))?,
))
}
@@ -66,22 +66,24 @@ impl Clone for EncodingWrapper {
}
impl EncodingWrapper {
- pub fn new(encoding: &'static dyn Encoding) -> EncodingWrapper {
+ pub fn new(encoding: &'static Encoding) -> EncodingWrapper {
EncodingWrapper(encoding)
}
pub async fn decode(&self, input: Vec<u8>) -> Result<String> {
- match self.0.decode(&input, encoding::DecoderTrap::Replace) {
- Ok(v) => Ok(v),
- Err(e) => Err(Error::msg(e.to_string())),
- }
+ let (cow, _encoding_used, _had_errors) = self.0.decode(&input);
+ // encoding_rs handles invalid bytes by replacing them with replacement characters
+ // in the output string, so we return the result even if there were errors.
+ // This preserves the original behavior where files with invalid bytes could still be opened.
+ Ok(cow.into_owned())
}
pub async fn encode(&self, input: String) -> Result<Vec<u8>> {
- match self.0.encode(&input, encoding::EncoderTrap::Replace) {
- Ok(v) => Ok(v),
- Err(e) => Err(Error::msg(e.to_string())),
- }
+ let (cow, _encoding_used, _had_errors) = self.0.encode(&input);
+ // encoding_rs handles unencodable characters by replacing them with
+ // appropriate substitutes in the output, so we return the result even if there were errors.
+ // This maintains consistency with the decode behavior.
+ Ok(cow.into_owned())
}
}
@@ -94,3 +96,52 @@ pub async fn to_utf8(input: Vec<u8>, encoding: EncodingWrapper) -> Result<String
pub async fn from_utf8(input: String, target: EncodingWrapper) -> Result<Vec<u8>> {
target.encode(input).await
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use gpui::BackgroundExecutor;
+
+ #[gpui::test]
+ async fn test_decode_with_invalid_bytes(_: BackgroundExecutor) {
+ // Test that files with invalid bytes can still be decoded
+ // This is a regression test for the issue where files couldn't be opened
+ // when they contained invalid bytes for the specified encoding
+
+ // Create some invalid UTF-8 bytes
+ let invalid_bytes = vec![0xFF, 0xFE, 0x00, 0x48]; // Invalid UTF-8 sequence
+
+ let encoding = EncodingWrapper::new(encoding_rs::UTF_8);
+ let result = encoding.decode(invalid_bytes).await;
+
+ // The decode should succeed, not fail
+ assert!(result.is_ok(), "Decode should succeed even with invalid bytes");
+
+ let decoded = result.unwrap();
+ // The result should contain replacement characters for invalid sequences
+ assert!(!decoded.is_empty(), "Decoded string should not be empty");
+
+ // Test with Windows-1252 and some bytes that might be invalid
+ let maybe_invalid_bytes = vec![0x81, 0x8D, 0x8F, 0x90, 0x9D]; // Some potentially problematic bytes
+ let encoding = EncodingWrapper::new(encoding_rs::WINDOWS_1252);
+ let result = encoding.decode(maybe_invalid_bytes).await;
+
+ // Should still succeed
+ assert!(result.is_ok(), "Decode should succeed with Windows-1252 even with potentially invalid bytes");
+ }
+
+ #[gpui::test]
+ async fn test_encode_with_unencodable_chars(_: BackgroundExecutor) {
+ // Test that strings with unencodable characters can still be encoded
+ let input = "Hello δΈη π".to_string(); // Contains Unicode that may not encode to all formats
+
+ let encoding = EncodingWrapper::new(encoding_rs::WINDOWS_1252);
+ let result = encoding.encode(input).await;
+
+ // The encode should succeed, not fail
+ assert!(result.is_ok(), "Encode should succeed even with unencodable characters");
+
+ let encoded = result.unwrap();
+ assert!(!encoded.is_empty(), "Encoded bytes should not be empty");
+ }
+}
@@ -618,7 +618,7 @@ impl Fs for RealFs {
async fn load(&self, path: &Path) -> Result<String> {
let path = path.to_path_buf();
- let encoding = EncodingWrapper::new(encoding::all::UTF_8);
+ let encoding = EncodingWrapper::new(encoding_rs::UTF_8);
let text =
smol::unblock(async || Ok(encodings::to_utf8(std::fs::read(path)?, encoding).await?))
.await
@@ -29,7 +29,7 @@ command_palette_hooks.workspace = true
component.workspace = true
db.workspace = true
editor.workspace = true
-encoding.workspace = true
+encoding_rs.workspace = true
futures.workspace = true
fuzzy.workspace = true
git.workspace = true
@@ -358,7 +358,7 @@ impl Render for FileDiffView {
mod tests {
use super::*;
use editor::test::editor_test_context::assert_state_with_diff;
- use encoding::all::UTF_8;
+ use encoding_rs::UTF_8;
use gpui::TestAppContext;
use language::Rope;
use project::{FakeFs, Fs, Project};
@@ -32,7 +32,7 @@ clock.workspace = true
collections.workspace = true
diffy = "0.4.2"
ec4rs.workspace = true
-encoding.workspace = true
+encoding_rs.workspace = true
fs.workspace = true
futures.workspace = true
fuzzy.workspace = true
@@ -21,7 +21,7 @@ use anyhow::{Context as _, Result};
use clock::Lamport;
pub use clock::ReplicaId;
use collections::HashMap;
-use encoding::Encoding;
+use encoding_rs::Encoding;
use fs::MTime;
use futures::channel::oneshot;
use gpui::{
@@ -127,7 +127,7 @@ pub struct Buffer {
has_unsaved_edits: Cell<(clock::Global, bool)>,
change_bits: Vec<rc::Weak<Cell<bool>>>,
_subscriptions: Vec<gpui::Subscription>,
- pub encoding: &'static dyn encoding::Encoding,
+ pub encoding: &'static Encoding,
}
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
@@ -420,7 +420,7 @@ pub trait LocalFile: File {
fn load_bytes(&self, cx: &App) -> Task<Result<Vec<u8>>>;
/// Loads the file contents from disk, decoding them with the given encoding.
- fn load_with_encoding(&self, cx: &App, encoding: &'static dyn Encoding)
+ fn load_with_encoding(&self, cx: &App, encoding: &'static Encoding)
-> Task<Result<String>>;
}
@@ -1012,7 +1012,7 @@ impl Buffer {
has_conflict: false,
change_bits: Default::default(),
_subscriptions: Vec::new(),
- encoding: encoding::all::UTF_8,
+ encoding: encoding_rs::UTF_8,
}
}
@@ -5238,7 +5238,7 @@ impl LocalFile for TestFile {
unimplemented!()
}
- fn load_with_encoding(&self, _: &App, _: &'static dyn Encoding) -> Task<Result<String>> {
+ fn load_with_encoding(&self, _: &App, _: &'static Encoding) -> Task<Result<String>> {
unimplemented!()
}
}
@@ -39,7 +39,7 @@ clock.workspace = true
collections.workspace = true
context_server.workspace = true
dap.workspace = true
-encoding.workspace = true
+encoding_rs.workspace = true
extension.workspace = true
fancy-regex.workspace = true
fs.workspace = true
@@ -7,7 +7,7 @@ use std::{
use anyhow::{Context as _, Result, anyhow};
use collections::{HashMap, HashSet};
-use encoding::all::UTF_8;
+use encoding_rs::UTF_8;
use fs::{Fs, encodings::EncodingWrapper};
use futures::{
FutureExt,
@@ -12,7 +12,7 @@ use buffer_diff::{
BufferDiffEvent, CALCULATE_DIFF_TASK, DiffHunkSecondaryStatus, DiffHunkStatus,
DiffHunkStatusKind, assert_hunks,
};
-use encoding::all::UTF_8;
+use encoding_rs::UTF_8;
use fs::{FakeFs, encodings::EncodingWrapper};
use futures::{StreamExt, future};
use git::{
@@ -28,7 +28,7 @@ clap.workspace = true
client.workspace = true
dap_adapters.workspace = true
debug_adapter_extension.workspace = true
-encoding.workspace = true
+encoding_rs.workspace = true
env_logger.workspace = true
extension.workspace = true
extension_host.workspace = true
@@ -35,7 +35,7 @@ clock.workspace = true
collections.workspace = true
component.workspace = true
db.workspace = true
-encoding.workspace = true
+encoding_rs.workspace = true
fs.workspace = true
futures.workspace = true
gpui.workspace = true
@@ -19,7 +19,7 @@ mod workspace_settings;
pub use crate::notifications::NotificationFrame;
pub use dock::Panel;
-use encoding::all::UTF_8;
+use encoding_rs::UTF_8;
use fs::encodings::EncodingWrapper;
pub use path_list::PathList;
pub use toast_layer::{ToastAction, ToastLayer, ToastView};
@@ -27,7 +27,7 @@ anyhow.workspace = true
async-lock.workspace = true
clock.workspace = true
collections.workspace = true
-encoding.workspace = true
+encoding_rs.workspace = true
fs.workspace = true
futures.workspace = true
fuzzy.workspace = true
@@ -7,7 +7,7 @@ use ::ignore::gitignore::{Gitignore, GitignoreBuilder};
use anyhow::{Context as _, Result, anyhow};
use clock::ReplicaId;
use collections::{HashMap, HashSet, VecDeque};
-use encoding::Encoding;
+use encoding_rs::Encoding;
use fs::{
Fs, MTime, PathEvent, RemoveOptions, Watcher, copy_recursive, encodings::EncodingWrapper,
read_dir_items,
@@ -735,7 +735,7 @@ impl Worktree {
text: Rope,
line_ending: LineEnding,
cx: &Context<Worktree>,
- encoding: &'static dyn Encoding,
+ encoding: &'static Encoding,
) -> Task<Result<Arc<File>>> {
match self {
Worktree::Local(this) => this.write_file(path, text, line_ending, cx, encoding),
@@ -1451,7 +1451,7 @@ impl LocalWorktree {
text: Rope,
line_ending: LineEnding,
cx: &Context<Worktree>,
- encoding: &'static dyn Encoding,
+ encoding: &'static Encoding,
) -> Task<Result<Arc<File>>> {
let fs = self.fs.clone();
let is_private = self.is_path_private(&path);
@@ -3132,7 +3132,7 @@ impl language::LocalFile for File {
fn load_with_encoding(
&self,
cx: &App,
- encoding: &'static dyn Encoding,
+ encoding: &'static Encoding,
) -> Task<Result<String>> {
let worktree = self.worktree.read(cx).as_local().unwrap();
let path = worktree.absolutize(&self.path);
@@ -3,7 +3,7 @@ use crate::{
worktree_settings::WorktreeSettings,
};
use anyhow::Result;
-use encoding::all::UTF_8;
+use encoding_rs::UTF_8;
use fs::{FakeFs, Fs, RealFs, RemoveOptions, encodings::EncodingWrapper};
use git::GITIGNORE;
use gpui::{AppContext as _, BackgroundExecutor, BorrowAppContext, Context, Task, TestAppContext};
@@ -2176,7 +2176,7 @@ mod tests {
use assets::Assets;
use collections::HashSet;
use editor::{DisplayPoint, Editor, SelectionEffects, display_map::DisplayRow};
- use encoding::all::UTF_8;
+ use encoding_rs::UTF_8;
use fs::encodings::EncodingWrapper;
use gpui::{
Action, AnyWindowHandle, App, AssetSource, BorrowAppContext, SemanticVersion,