Enable a file to be opened with an invalid encoding with the invalid

R Aadarsh created

bytes replaced with replacement characters

- Fix UTF-16 file handling

- Introduce a `ForceOpen` action to allow users to open files despite
encoding errors

- Add `force` and `detect_utf16` flags

- Update UI to provide "Accept the Risk and Open" button for invalid
encoding files

Change summary

Cargo.lock                                         |   2 
crates/agent_ui/src/acp/message_editor.rs          |   2 
crates/agent_ui/src/context.rs                     |   2 
crates/copilot/src/copilot.rs                      |   3 
crates/edit_prediction_context/src/syntax_index.rs |   2 
crates/encodings/Cargo.toml                        |   2 
crates/encodings/src/lib.rs                        |  40 +++++
crates/encodings/src/selectors.rs                  | 120 ++++++++++-----
crates/fs/src/encodings.rs                         |  54 +++++--
crates/fs/src/fs.rs                                |  13 +
crates/language/src/buffer.rs                      |  30 +++
crates/languages/src/json.rs                       |   4 
crates/project/src/buffer_store.rs                 |  57 +++++--
crates/project/src/debugger/breakpoint_store.rs    |   2 
crates/project/src/invalid_item_view.rs            |  56 +++++--
crates/project/src/lsp_store.rs                    |   2 
crates/project/src/lsp_store/rust_analyzer_ext.rs  |   6 
crates/project/src/project.rs                      |  38 ++++
crates/remote_server/src/headless_project.rs       |   4 
crates/workspace/src/workspace.rs                  |  27 ++
crates/worktree/src/worktree.rs                    |  18 ++
crates/worktree/src/worktree_tests.rs              | 113 ++++++++++++++
crates/zed_actions/src/lib.rs                      |   3 
crates/zeta/src/zeta.rs                            |   2 
24 files changed, 477 insertions(+), 125 deletions(-)

Detailed changes

Cargo.lock 🔗

@@ -5529,6 +5529,8 @@ version = "0.1.0"
 dependencies = [
  "editor",
  "encoding_rs",
+ "fs",
+ "futures 0.3.31",
  "fuzzy",
  "gpui",
  "language",

crates/agent_ui/src/acp/message_editor.rs 🔗

@@ -515,7 +515,7 @@ impl MessageEditor {
                                 worktree_id,
                                 path: worktree_path,
                             };
-                            buffer_store.open_buffer(project_path, cx)
+                            buffer_store.open_buffer(project_path, None, false, true, cx)
                         })
                     });
 

crates/agent_ui/src/context.rs 🔗

@@ -287,7 +287,7 @@ impl DirectoryContextHandle {
             let open_task = project.update(cx, |project, cx| {
                 project.buffer_store().update(cx, |buffer_store, cx| {
                     let project_path = ProjectPath { worktree_id, path };
-                    buffer_store.open_buffer(project_path, None, cx)
+                    buffer_store.open_buffer(project_path, None, false, true, cx)
                 })
             });
 

crates/copilot/src/copilot.rs 🔗

@@ -1242,6 +1242,7 @@ async fn get_copilot_lsp(fs: Arc<dyn Fs>, node_runtime: NodeRuntime) -> anyhow::
 mod tests {
     use super::*;
     use fs::encodings::EncodingWrapper;
+    use encoding_rs::Encoding;
     use gpui::TestAppContext;
     use util::{path, paths::PathStyle, rel_path::rel_path};
 
@@ -1452,7 +1453,7 @@ mod tests {
             self.abs_path.clone()
         }
 
-        fn load(&self, _: &App, _: EncodingWrapper, _: bool) -> Task<Result<String>> {
+        fn load(&self, _: &App, _: EncodingWrapper, _: bool, _: bool, _: Option<Arc<std::sync::Mutex<&'static Encoding>>>) -> Task<Result<String>> {
             unimplemented!()
         }
 

crates/edit_prediction_context/src/syntax_index.rs 🔗

@@ -523,7 +523,7 @@ impl SyntaxIndex {
         };
 
         let snapshot_task = worktree.update(cx, |worktree, cx| {
-            let load_task = worktree.load_file(&project_path.path, None, cx);
+            let load_task = worktree.load_file(&project_path.path, None, false, true, None, cx);
             cx.spawn(async move |_this, cx| {
                 let loaded_file = load_task.await?;
                 let language = language.await?;

crates/encodings/Cargo.toml 🔗

@@ -7,6 +7,8 @@ edition.workspace = true
 [dependencies]
 editor.workspace = true
 encoding_rs.workspace = true
+fs.workspace = true
+futures.workspace = true
 fuzzy.workspace = true
 gpui.workspace = true
 language.workspace = true

crates/encodings/src/lib.rs 🔗

@@ -7,8 +7,12 @@ use gpui::{ClickEvent, Entity, Subscription, WeakEntity};
 use language::Buffer;
 use ui::{App, Button, ButtonCommon, Context, LabelSize, Render, Tooltip, Window, div};
 use ui::{Clickable, ParentElement};
-use workspace::{ItemHandle, StatusItemView, Workspace, with_active_or_new_workspace};
-use zed_actions::encodings::Toggle;
+use util::ResultExt;
+use workspace::{
+    CloseActiveItem, ItemHandle, OpenOptions, StatusItemView, Workspace,
+    with_active_or_new_workspace,
+};
+use zed_actions::encodings::{ForceOpen, Toggle};
 
 use crate::selectors::encoding::EncodingSelector;
 use crate::selectors::save_or_reopen::EncodingSaveOrReopenSelector;
@@ -304,4 +308,36 @@ pub fn init(cx: &mut App) {
             });
         });
     });
+
+    cx.on_action(|action: &ForceOpen, cx: &mut App| {
+        let ForceOpen(path) = action.clone();
+        let path = path.to_path_buf();
+
+        with_active_or_new_workspace(cx, |workspace, window, cx| {
+            workspace.active_pane().update(cx, |pane, cx| {
+                pane.close_active_item(&CloseActiveItem::default(), window, cx)
+                    .detach();
+            });
+
+            {
+                let force = workspace.encoding_options.force.get_mut();
+
+                *force = true;
+            }
+
+            let open_task = workspace.open_abs_path(path, OpenOptions::default(), window, cx);
+            let weak_workspace = workspace.weak_handle();
+
+            cx.spawn(async move |_, cx| {
+                let workspace = weak_workspace.upgrade().unwrap();
+                open_task.await.log_err();
+                workspace
+                    .update(cx, |workspace: &mut Workspace, _| {
+                        *workspace.encoding_options.force.get_mut() = false;
+                    })
+                    .log_err();
+            })
+            .detach();
+        });
+    });
 }

crates/encodings/src/selectors.rs 🔗

@@ -1,6 +1,6 @@
 /// This module contains the encoding selectors for saving or reopening files with a different encoding.
 /// It provides a modal view that allows the user to choose between saving with a different encoding
-/// or reopening with a different encoding, and then selecting the desired encoding from a list.
+/// or reopening with a different encoding.
 pub mod save_or_reopen {
     use editor::Editor;
     use gpui::Styled;
@@ -277,10 +277,14 @@ pub mod save_or_reopen {
 
 /// This module contains the encoding selector for choosing an encoding to save or reopen a file with.
 pub mod encoding {
+    use editor::Editor;
+    use fs::encodings::EncodingWrapper;
     use std::{path::PathBuf, sync::atomic::AtomicBool};
 
     use fuzzy::{StringMatch, StringMatchCandidate};
-    use gpui::{AppContext, DismissEvent, Entity, EventEmitter, Focusable, WeakEntity};
+    use gpui::{
+        AppContext, DismissEvent, Entity, EventEmitter, Focusable, WeakEntity, http_client::anyhow,
+    };
     use language::Buffer;
     use picker::{Picker, PickerDelegate};
     use ui::{
@@ -288,7 +292,7 @@ pub mod encoding {
         Window, rems, v_flex,
     };
     use util::{ResultExt, TryFutureExt};
-    use workspace::{ModalView, Workspace};
+    use workspace::{CloseActiveItem, ModalView, OpenOptions, Workspace};
 
     use crate::encoding_from_name;
 
@@ -436,50 +440,84 @@ pub mod encoding {
                 .unwrap();
 
             if let Some(buffer) = &self.buffer
-                && let Some(buffer) = buffer.upgrade()
+                && let Some(buffer_entity) = buffer.upgrade()
             {
-                buffer.update(cx, |buffer, cx| {
+                let buffer = buffer_entity.read(cx);
+
+                // Since the encoding will be accessed in `reload`,
+                // the lock must be released before calling `reload`.
+                // By limiting the scope, we ensure that it is released
+                {
                     let buffer_encoding = buffer.encoding.clone();
-                    let buffer_encoding = &mut *buffer_encoding.lock().unwrap();
-                    *buffer_encoding =
+                    *buffer_encoding.lock().unwrap() =
                         encoding_from_name(self.matches[self.current_selection].string.as_str());
-                    if self.action == Action::Reopen {
-                        let executor = cx.background_executor().clone();
-                        executor.spawn(buffer.reload(cx)).detach();
-                    } else if self.action == Action::Save {
-                        let executor = cx.background_executor().clone();
-
-                        executor
-                            .spawn(workspace.update(cx, |workspace, cx| {
-                                workspace
-                                    .save_active_item(workspace::SaveIntent::Save, window, cx)
-                                    .log_err()
-                            }))
-                            .detach();
-                    }
-                });
+                }
+
+                self.dismissed(window, cx);
+
+                if self.action == Action::Reopen {
+                    buffer_entity.update(cx, |buffer, cx| {
+                        let rec = buffer.reload(cx);
+                        cx.spawn(async move |_, _| rec.await).detach()
+                    });
+                } else if self.action == Action::Save {
+                    let task = workspace.update(cx, |workspace, cx| {
+                        workspace
+                            .save_active_item(workspace::SaveIntent::Save, window, cx)
+                            .log_err()
+                    });
+                    cx.spawn(async |_, _| task).detach();
+                }
             } else {
-                workspace.update(cx, |workspace, cx| {
-                    *workspace.encoding.lock().unwrap() =
+                if let Some(path) = self.selector.upgrade().unwrap().read(cx).path.clone() {
+                    workspace.update(cx, |workspace, cx| {
+                        workspace.active_pane().update(cx, |pane, cx| {
+                            pane.close_active_item(&CloseActiveItem::default(), window, cx)
+                                .detach();
+                        });
+                    });
+
+                    let encoding =
                         encoding_from_name(self.matches[self.current_selection].string.as_str());
-                    workspace
-                        .open_abs_path(
-                            self.selector
-                                .upgrade()
-                                .unwrap()
-                                .read(cx)
-                                .path
-                                .as_ref()
-                                .unwrap()
-                                .clone(),
-                            Default::default(),
-                            window,
-                            cx,
-                        )
-                        .detach();
-                })
+
+                    let open_task = workspace.update(cx, |workspace, cx| {
+                        *workspace.encoding_options.encoding.lock().unwrap() =
+                            EncodingWrapper::new(encoding);
+
+                        workspace.open_abs_path(path, OpenOptions::default(), window, cx)
+                    });
+
+                    cx.spawn(async move |_, cx| {
+                        if let Ok(_) = {
+                            let result = open_task.await;
+                            workspace
+                                .update(cx, |workspace, _| {
+                                    *workspace.encoding_options.force.get_mut() = false;
+                                })
+                                .unwrap();
+
+                            result
+                        } && let Ok(Ok((_, buffer, _))) =
+                            workspace.read_with(cx, |workspace, cx| {
+                                if let Some(active_item) = workspace.active_item(cx)
+                                    && let Some(editor) = active_item.act_as::<Editor>(cx)
+                                {
+                                    Ok(editor.read(cx).active_excerpt(cx).unwrap())
+                                } else {
+                                    Err(anyhow!("error"))
+                                }
+                            })
+                        {
+                            buffer
+                                .read_with(cx, |buffer, _| {
+                                    *buffer.encoding.lock().unwrap() = encoding;
+                                })
+                                .log_err();
+                        }
+                    })
+                    .detach();
+                }
             }
-            self.dismissed(window, cx);
         }
 
         fn dismissed(&mut self, _: &mut Window, cx: &mut Context<Picker<Self>>) {

crates/fs/src/encodings.rs 🔗

@@ -4,6 +4,8 @@ use std::{
     sync::{Arc, Mutex},
 };
 
+use std::sync::atomic::AtomicBool;
+
 use anyhow::Result;
 use encoding_rs::Encoding;
 
@@ -25,8 +27,6 @@ impl Default for EncodingWrapper {
     }
 }
 
-pub struct EncodingWrapperVisitor;
-
 impl PartialEq for EncodingWrapper {
     fn eq(&self, other: &Self) -> bool {
         self.0.name() == other.0.name()
@@ -55,11 +55,13 @@ impl EncodingWrapper {
         &mut self,
         input: Vec<u8>,
         force: bool,
+        detect_utf16: bool,
         buffer_encoding: Option<Arc<Mutex<&'static Encoding>>>,
     ) -> Result<String> {
-        // Check if the input starts with a BOM for UTF-16 encodings only if not forced to
-        // use the encoding specified.
-        if !force {
+        // Check if the input starts with a BOM for UTF-16 encodings only if detect_utf16 is true.
+        println!("{}", force);
+        println!("{}", detect_utf16);
+        if detect_utf16 {
             if let Some(encoding) = match input.get(..2) {
                 Some([0xFF, 0xFE]) => Some(encoding_rs::UTF_16LE),
                 Some([0xFE, 0xFF]) => Some(encoding_rs::UTF_16BE),
@@ -67,20 +69,23 @@ impl EncodingWrapper {
             } {
                 self.0 = encoding;
 
-                if let Some(v) = buffer_encoding {
-                    if let Ok(mut v) = (*v).lock() {
-                        *v = encoding;
-                    }
+                if let Some(v) = buffer_encoding
+                    && let Ok(mut v) = v.lock()
+                {
+                    *v = encoding;
                 }
             }
         }
 
-        let (cow, _had_errors) = self.0.decode_with_bom_removal(&input);
+        let (cow, had_errors) = self.0.decode_with_bom_removal(&input);
+
+        if force {
+            return Ok(cow.to_string());
+        }
 
-        if !_had_errors {
+        if !had_errors {
             Ok(cow.to_string())
         } else {
-            // If there were decoding errors, return an error.
             Err(anyhow::anyhow!(
                 "The file contains invalid bytes for the specified encoding: {}.\nThis usually means that the file is not a regular text file, or is encoded in a different encoding.\nContinuing to open it may result in data loss if saved.",
                 self.0.name()
@@ -107,9 +112,7 @@ impl EncodingWrapper {
             return Ok(data);
         } else {
             let (cow, _encoding_used, _had_errors) = self.0.encode(&input);
-            // `encoding_rs` handles unencodable characters by replacing them with
-            // appropriate substitutes in the output, so we return the result even if there were errors.
-            // This maintains consistency with the decode behaviour.
+
             Ok(cow.into_owned())
         }
     }
@@ -120,12 +123,31 @@ pub async fn to_utf8(
     input: Vec<u8>,
     mut encoding: EncodingWrapper,
     force: bool,
+    detect_utf16: bool,
     buffer_encoding: Option<Arc<Mutex<&'static Encoding>>>,
 ) -> Result<String> {
-    encoding.decode(input, force, buffer_encoding).await
+    encoding
+        .decode(input, force, detect_utf16, buffer_encoding)
+        .await
 }
 
 /// Convert a UTF-8 string to a byte vector in a specified encoding.
 pub async fn from_utf8(input: String, target: EncodingWrapper) -> Result<Vec<u8>> {
     target.encode(input).await
 }
+
+pub struct EncodingOptions {
+    pub encoding: Arc<Mutex<EncodingWrapper>>,
+    pub force: AtomicBool,
+    pub detect_utf16: AtomicBool,
+}
+
+impl Default for EncodingOptions {
+    fn default() -> Self {
+        EncodingOptions {
+            encoding: Arc::new(Mutex::new(EncodingWrapper::default())),
+            force: AtomicBool::new(false),
+            detect_utf16: AtomicBool::new(true),
+        }
+    }
+}

crates/fs/src/fs.rs 🔗

@@ -62,9 +62,9 @@ use std::ffi::OsStr;
 
 #[cfg(any(test, feature = "test-support"))]
 pub use fake_git_repo::{LOAD_HEAD_TEXT_TASK, LOAD_INDEX_TEXT_TASK};
-use crate::encodings::to_utf8;
 use crate::encodings::EncodingWrapper;
 use crate::encodings::from_utf8;
+use crate::encodings::to_utf8;
 
 pub trait Watcher: Send + Sync {
     fn add(&self, path: &Path) -> Result<()>;
@@ -125,9 +125,18 @@ pub trait Fs: Send + Sync {
         &self,
         path: &Path,
         encoding: EncodingWrapper,
+        force: bool,
         detect_utf16: bool,
+        buffer_encoding: Option<Arc<std::sync::Mutex<&'static Encoding>>>,
     ) -> Result<String> {
-        Ok(to_utf8(self.load_bytes(path).await?, encoding, detect_utf16, None).await?)
+        Ok(to_utf8(
+            self.load_bytes(path).await?,
+            encoding,
+            force,
+            detect_utf16,
+            buffer_encoding,
+        )
+        .await?)
     }
 
     async fn load_bytes(&self, path: &Path) -> Result<Vec<u8>>;

crates/language/src/buffer.rs 🔗

@@ -414,8 +414,14 @@ pub trait LocalFile: File {
     fn abs_path(&self, cx: &App) -> PathBuf;
 
     /// Loads the file contents from disk and returns them as a UTF-8 encoded string.
-    fn load(&self, cx: &App, encoding: EncodingWrapper, detect_utf16: bool)
-    -> Task<Result<String>>;
+    fn load(
+        &self,
+        cx: &App,
+        encoding: EncodingWrapper,
+        force: bool,
+        detect_utf16: bool,
+        buffer_encoding: Option<Arc<std::sync::Mutex<&'static Encoding>>>,
+    ) -> Task<Result<String>>;
 
     /// Loads the file's contents from disk.
     fn load_bytes(&self, cx: &App) -> Task<Result<Vec<u8>>>;
@@ -842,6 +848,18 @@ impl Buffer {
         )
     }
 
+    /// Replace the text buffer. This function is in contrast to `set_text` in that it does not
+    /// change the buffer's editing state
+    pub fn replace_text_buffer(&mut self, new: TextBuffer, cx: &mut Context<Self>) {
+        self.text = new;
+        self.saved_version = self.version.clone();
+        self.has_unsaved_edits.set((self.version.clone(), false));
+
+        self.was_changed();
+        cx.emit(BufferEvent::DirtyChanged);
+        cx.notify();
+    }
+
     /// Create a new buffer with the given base text that has proper line endings and other normalization applied.
     pub fn local_normalized(
         base_text_normalized: Rope,
@@ -1346,13 +1364,14 @@ impl Buffer {
     pub fn reload(&mut self, cx: &Context<Self>) -> oneshot::Receiver<Option<Transaction>> {
         let (tx, rx) = futures::channel::oneshot::channel();
         let encoding = EncodingWrapper::new(*(self.encoding.lock().unwrap()));
+        let buffer_encoding = self.encoding.clone();
 
         let prev_version = self.text.version();
         self.reload_task = Some(cx.spawn(async move |this, cx| {
             let Some((new_mtime, new_text)) = this.update(cx, |this, cx| {
                 let file = this.file.as_ref()?.as_local()?;
                 Some((file.disk_state().mtime(), {
-                    file.load(cx, encoding, false)
+                    file.load(cx, encoding, false, true, Some(buffer_encoding))
                 }))
             })?
             else {
@@ -1406,6 +1425,9 @@ impl Buffer {
         cx.notify();
     }
 
+    pub fn replace_file(&mut self, new_file: Arc<dyn File>) {
+        self.file = Some(new_file);
+    }
     /// Updates the [`File`] backing this buffer. This should be called when
     /// the file has changed or has been deleted.
     pub fn file_updated(&mut self, new_file: Arc<dyn File>, cx: &mut Context<Self>) {
@@ -5231,7 +5253,9 @@ impl LocalFile for TestFile {
         &self,
         _cx: &App,
         _encoding: EncodingWrapper,
+        _force: bool,
         _detect_utf16: bool,
+        _buffer_encoding: Option<Arc<std::sync::Mutex<&'static Encoding>>>,
     ) -> Task<Result<String>> {
         unimplemented!()
     }

crates/languages/src/json.rs 🔗

@@ -56,7 +56,9 @@ impl ContextProvider for JsonTaskProvider {
         cx.spawn(async move |cx| {
             let contents = file
                 .worktree
-                .update(cx, |this, cx| this.load_file(&file.path, None, cx))
+                .update(cx, |this, cx| {
+                    this.load_file(&file.path, None, false, true, None, cx)
+                })
                 .ok()?
                 .await
                 .ok()?;

crates/project/src/buffer_store.rs 🔗

@@ -633,26 +633,47 @@ impl LocalBufferStore {
         path: Arc<RelPath>,
         worktree: Entity<Worktree>,
         encoding: Option<EncodingWrapper>,
+        force: bool,
+        detect_utf16: bool,
         cx: &mut Context<BufferStore>,
     ) -> Task<Result<Entity<Buffer>>> {
         let load_buffer = worktree.update(cx, |worktree, cx| {
-            let load_file = worktree.load_file(path.as_ref(), encoding, cx);
             let reservation = cx.reserve_entity();
-
             let buffer_id = BufferId::from(reservation.entity_id().as_non_zero_u64());
-            let path = path.clone();
-            cx.spawn(async move |_, cx| {
-                let loaded = load_file.await.with_context(|| {
-                    format!("Could not open path: {}", path.display(PathStyle::local()))
+
+            // Create the buffer first
+            let buffer = cx.insert_entity(reservation, |_| {
+                Buffer::build(
+                    text::Buffer::new(0, buffer_id, ""),
+                    None,
+                    Capability::ReadWrite,
+                )
+            });
+
+            let buffer_encoding = buffer.read(cx).encoding.clone();
+
+            let load_file_task = worktree.load_file(
+                path.as_ref(),
+                encoding,
+                force,
+                detect_utf16,
+                Some(buffer_encoding),
+                cx,
+            );
+
+            cx.spawn(async move |_, async_cx| {
+                let loaded_file = load_file_task.await?;
+                let mut reload_task = None;
+
+                buffer.update(async_cx, |buffer, cx| {
+                    buffer.replace_file(loaded_file.file);
+                    buffer
+                        .replace_text_buffer(text::Buffer::new(0, buffer_id, loaded_file.text), cx);
+
+                    reload_task = Some(buffer.reload(cx));
                 })?;
-                let text_buffer = cx
-                    .background_spawn(async move {
-                        text::Buffer::new(ReplicaId::LOCAL, buffer_id, loaded.text)
-                    })
-                    .await;
-                cx.insert_entity(reservation, |_| {
-                    Buffer::build(text_buffer, Some(loaded.file), Capability::ReadWrite)
-                })
+
+                Ok(buffer)
             })
         });
 
@@ -834,6 +855,8 @@ impl BufferStore {
         &mut self,
         project_path: ProjectPath,
         encoding: Option<EncodingWrapper>,
+        force: bool,
+        detect_utf16: bool,
         cx: &mut Context<Self>,
     ) -> Task<Result<Entity<Buffer>>> {
         if let Some(buffer) = self.get_by_path(&project_path) {
@@ -857,7 +880,9 @@ impl BufferStore {
                     return Task::ready(Err(anyhow!("no such worktree")));
                 };
                 let load_buffer = match &self.state {
-                    BufferStoreState::Local(this) => this.open_buffer(path, worktree, encoding, cx),
+                    BufferStoreState::Local(this) => {
+                        this.open_buffer(path, worktree, encoding, force, detect_utf16, cx)
+                    }
                     BufferStoreState::Remote(this) => this.open_buffer(path, worktree, cx),
                 };
 
@@ -1170,7 +1195,7 @@ impl BufferStore {
                 let buffers = this.update(cx, |this, cx| {
                     project_paths
                         .into_iter()
-                        .map(|project_path| this.open_buffer(project_path, cx))
+                        .map(|project_path| this.open_buffer(project_path, None, cx))
                         .collect::<Vec<_>>()
                 })?;
                 for buffer_task in buffers {

crates/project/src/debugger/breakpoint_store.rs 🔗

@@ -796,7 +796,7 @@ impl BreakpointStore {
                                 worktree_id: worktree.read(cx).id(),
                                 path: relative_path,
                             };
-                            this.open_buffer(path, None, cx)
+                            this.open_buffer(path, None, false, true, cx)
                         })?
                         .await;
                     let Ok(buffer) = buffer else {

crates/project/src/invalid_item_view.rs 🔗

@@ -4,7 +4,7 @@ use gpui::{EventEmitter, FocusHandle, Focusable};
 use ui::{
     App, Button, ButtonCommon, ButtonStyle, Clickable, Context, FluentBuilder, InteractiveElement,
     KeyBinding, Label, LabelCommon, LabelSize, ParentElement, Render, SharedString, Styled as _,
-    Window, h_flex, v_flex,
+    TintColor, Window, h_flex, v_flex,
 };
 use zed_actions::workspace::OpenWithSystem;
 
@@ -78,7 +78,8 @@ impl Focusable for InvalidItemView {
 impl Render for InvalidItemView {
     fn render(&mut self, window: &mut Window, cx: &mut Context<Self>) -> impl gpui::IntoElement {
         let abs_path = self.abs_path.clone();
-        let path = self.abs_path.clone();
+        let path0 = self.abs_path.clone();
+        let path1 = self.abs_path.clone();
 
         v_flex()
             .size_full()
@@ -115,23 +116,44 @@ impl Render for InvalidItemView {
                                     ),
                                 )
                                 .child(
-                                    h_flex().justify_center().child(
-                                        Button::new(
-                                            "open-with-encoding",
-                                            "Open With a Different Encoding",
+                                    h_flex()
+                                        .justify_center()
+                                        .child(
+                                            Button::new(
+                                                "open-with-encoding",
+                                                "Open With a Different Encoding",
+                                            )
+                                            .style(ButtonStyle::Outlined)
+                                            .on_click(
+                                                move |_, window, cx| {
+                                                    window.dispatch_action(
+                                                        Box::new(zed_actions::encodings::Toggle(
+                                                            path0.clone(),
+                                                        )),
+                                                        cx,
+                                                    )
+                                                },
+                                            ),
                                         )
-                                        .style(ButtonStyle::Outlined)
-                                        .on_click(
-                                            move |_, window, cx| {
-                                                window.dispatch_action(
-                                                    Box::new(zed_actions::encodings::Toggle(
-                                                        path.clone(),
-                                                    )),
-                                                    cx,
-                                                )
-                                            },
+                                        .child(
+                                            Button::new(
+                                                "accept-risk-and-open",
+                                                "Accept the Risk and Open",
+                                            )
+                                            .style(ButtonStyle::Tinted(TintColor::Warning))
+                                            .on_click(
+                                                move |_, window, cx| {
+                                                    window.dispatch_action(
+                                                        Box::new(
+                                                            zed_actions::encodings::ForceOpen(
+                                                                path1.clone(),
+                                                            ),
+                                                        ),
+                                                        cx,
+                                                    );
+                                                },
+                                            ),
                                         ),
-                                    ),
                                 )
                         }),
                 ),

crates/project/src/lsp_store.rs 🔗

@@ -8336,7 +8336,7 @@ impl LspStore {
             lsp_store
                 .update(cx, |lsp_store, cx| {
                     lsp_store.buffer_store().update(cx, |buffer_store, cx| {
-                        buffer_store.open_buffer(project_path, None, cx)
+                        buffer_store.open_buffer(project_path, None, false, true,cx)
                     })
                 })?
                 .await

crates/project/src/lsp_store/rust_analyzer_ext.rs 🔗

@@ -91,7 +91,7 @@ pub fn cancel_flycheck(
     let buffer = buffer_path.map(|buffer_path| {
         project.update(cx, |project, cx| {
             project.buffer_store().update(cx, |buffer_store, cx| {
-                buffer_store.open_buffer(buffer_path, None, cx)
+                buffer_store.open_buffer(buffer_path, None, false, true, cx)
             })
         })
     });
@@ -140,7 +140,7 @@ pub fn run_flycheck(
     let buffer = buffer_path.map(|buffer_path| {
         project.update(cx, |project, cx| {
             project.buffer_store().update(cx, |buffer_store, cx| {
-                buffer_store.open_buffer(buffer_path, None, cx)
+                buffer_store.open_buffer(buffer_path, None, false, true, cx)
             })
         })
     });
@@ -198,7 +198,7 @@ pub fn clear_flycheck(
     let buffer = buffer_path.map(|buffer_path| {
         project.update(cx, |project, cx| {
             project.buffer_store().update(cx, |buffer_store, cx| {
-                buffer_store.open_buffer(buffer_path, None, cx)
+                buffer_store.open_buffer(buffer_path, None, false, true, cx)
             })
         })
     });

crates/project/src/project.rs 🔗

@@ -28,9 +28,12 @@ use buffer_diff::BufferDiff;
 use context_server_store::ContextServerStore;
 use encoding_rs::Encoding;
 pub use environment::ProjectEnvironmentEvent;
+use fs::encodings::EncodingOptions;
 use fs::encodings::EncodingWrapper;
 use git::repository::get_git_committer;
 use git_store::{Repository, RepositoryId};
+use std::sync::atomic::AtomicBool;
+
 pub mod search_history;
 mod yarn;
 
@@ -108,6 +111,7 @@ use settings::{InvalidSettingsError, Settings, SettingsLocation, SettingsStore};
 use smol::channel::Receiver;
 use snippet::Snippet;
 use snippet_provider::SnippetProvider;
+use std::ops::Deref;
 use std::{
     borrow::Cow,
     collections::BTreeMap,
@@ -217,7 +221,7 @@ pub struct Project {
     settings_observer: Entity<SettingsObserver>,
     toolchain_store: Option<Entity<ToolchainStore>>,
     agent_location: Option<AgentLocation>,
-    pub encoding: Arc<std::sync::Mutex<&'static Encoding>>,
+    pub encoding_options: EncodingOptions,
 }
 
 #[derive(Clone, Debug, PartialEq, Eq)]
@@ -1228,7 +1232,11 @@ impl Project {
                 toolchain_store: Some(toolchain_store),
 
                 agent_location: None,
-                encoding: Arc::new(std::sync::Mutex::new(encoding_rs::UTF_8)),
+                encoding_options: EncodingOptions {
+                    encoding: Arc::new(std::sync::Mutex::new(EncodingWrapper::default())),
+                    force: AtomicBool::new(false),
+                    detect_utf16: AtomicBool::new(true),
+                },
             }
         })
     }
@@ -1414,7 +1422,11 @@ impl Project {
 
                 toolchain_store: Some(toolchain_store),
                 agent_location: None,
-                encoding: Arc::new(std::sync::Mutex::new(encoding_rs::UTF_8)),
+                encoding_options: EncodingOptions {
+                    encoding: Arc::new(std::sync::Mutex::new(EncodingWrapper::default())),
+                    force: AtomicBool::new(false),
+                    detect_utf16: AtomicBool::new(false),
+                },
             };
 
             // remote server -> local machine handlers
@@ -1668,8 +1680,14 @@ impl Project {
                 remotely_created_models: Arc::new(Mutex::new(RemotelyCreatedModels::default())),
                 toolchain_store: None,
                 agent_location: None,
-                encoding: Arc::new(std::sync::Mutex::new(encoding_rs::UTF_8)),
+                encoding_options: EncodingOptions {
+                    encoding: Arc::new(std::sync::Mutex::new(EncodingWrapper::default())),
+
+                    force: AtomicBool::new(false),
+                    detect_utf16: AtomicBool::new(false),
+                },
             };
+
             project.set_role(role, cx);
             for worktree in worktrees {
                 project.add_worktree(&worktree, cx);
@@ -2720,7 +2738,17 @@ impl Project {
         self.buffer_store.update(cx, |buffer_store, cx| {
             buffer_store.open_buffer(
                 path.into(),
-                Some(EncodingWrapper::new(self.encoding.lock().as_ref().unwrap())),
+                Some(
+                    self.encoding_options
+                        .encoding
+                        .lock()
+                        .as_ref()
+                        .unwrap()
+                        .deref()
+                        .clone(),
+                ),
+                *self.encoding_options.force.get_mut(),
+                *self.encoding_options.detect_utf16.get_mut(),
                 cx,
             )
         })

crates/remote_server/src/headless_project.rs 🔗

@@ -512,6 +512,8 @@ impl HeadlessProject {
                         path: Arc::<Path>::from_proto(message.payload.path),
                     },
                     None,
+                    false,
+                    true,
                     cx,
                 )
             });
@@ -605,6 +607,8 @@ impl HeadlessProject {
                         path: path,
                     },
                     None,
+                    false,
+                    true,
                     cx,
                 )
             });

crates/workspace/src/workspace.rs 🔗

@@ -19,7 +19,6 @@ mod workspace_settings;
 
 pub use crate::notifications::NotificationFrame;
 pub use dock::Panel;
-use encoding_rs::Encoding;
 use encoding_rs::UTF_8;
 use fs::encodings::EncodingWrapper;
 pub use path_list::PathList;
@@ -33,6 +32,8 @@ use client::{
 };
 use collections::{HashMap, HashSet, hash_map};
 use dock::{Dock, DockPosition, PanelButtons, PanelHandle, RESIZE_HANDLE_SIZE};
+use fs::encodings::EncodingOptions;
+
 use futures::{
     Future, FutureExt, StreamExt,
     channel::{
@@ -650,7 +651,7 @@ impl ProjectItemRegistry {
                 let project_path = project_path.clone();
                 let EncodingWrapper(encoding) = encoding.unwrap_or_default();
 
-                project.update(cx, |project, _| {*project.encoding.lock().unwrap() = encoding});
+                project.update(cx, |project, _| {*project.encoding_options.encoding.lock().unwrap() = EncodingWrapper::new(encoding)});
 
                 let is_file = project
                     .read(cx)
@@ -1190,7 +1191,7 @@ pub struct Workspace {
     session_id: Option<String>,
     scheduled_tasks: Vec<Task<()>>,
     last_open_dock_positions: Vec<DockPosition>,
-    pub encoding: Arc<std::sync::Mutex<&'static Encoding>>,
+    pub encoding_options: EncodingOptions,
 }
 
 impl EventEmitter<Event> for Workspace {}
@@ -1533,7 +1534,7 @@ impl Workspace {
             session_id: Some(session_id),
             scheduled_tasks: Vec::new(),
             last_open_dock_positions: Vec::new(),
-            encoding: Arc::new(std::sync::Mutex::new(encoding_rs::UTF_8)),
+            encoding_options: Default::default(),
         }
     }
 
@@ -3416,7 +3417,6 @@ impl Workspace {
         window: &mut Window,
         cx: &mut Context<Self>,
     ) -> Task<anyhow::Result<Box<dyn ItemHandle>>> {
-        println!("{:?}", *self.encoding.lock().unwrap());
         cx.spawn_in(window, async move |workspace, cx| {
             let open_paths_task_result = workspace
                 .update_in(cx, |workspace, window, cx| {
@@ -3574,11 +3574,24 @@ impl Workspace {
         window: &mut Window,
         cx: &mut App,
     ) -> Task<Result<(Option<ProjectEntryId>, WorkspaceItemBuilder)>> {
+        let project = self.project();
+
+        project.update(cx, |project, _| {
+            project.encoding_options.force.store(
+                self.encoding_options
+                    .force
+                    .load(std::sync::atomic::Ordering::Relaxed),
+                std::sync::atomic::Ordering::Relaxed,
+            );
+        });
+
         let registry = cx.default_global::<ProjectItemRegistry>().clone();
         registry.open_path(
-            self.project(),
+            project,
             &path,
-            Some(EncodingWrapper::new(*self.encoding.lock().unwrap())),
+            Some(EncodingWrapper::new(
+                (self.encoding_options.encoding.lock().unwrap()).0,
+            )),
             window,
             cx,
         )

crates/worktree/src/worktree.rs 🔗

@@ -711,10 +711,15 @@ impl Worktree {
         &self,
         path: &Path,
         encoding: Option<EncodingWrapper>,
+        force: bool,
+        detect_utf16: bool,
+        buffer_encoding: Option<Arc<std::sync::Mutex<&'static Encoding>>>,
         cx: &Context<Worktree>,
     ) -> Task<Result<LoadedFile>> {
         match self {
-            Worktree::Local(this) => this.load_file(path, encoding, cx),
+            Worktree::Local(this) => {
+                this.load_file(path, encoding, force, detect_utf16, buffer_encoding, cx)
+            }
             Worktree::Remote(_) => {
                 Task::ready(Err(anyhow!("remote worktrees can't yet load files")))
             }
@@ -1325,6 +1330,9 @@ impl LocalWorktree {
         &self,
         path: &Path,
         encoding: Option<EncodingWrapper>,
+        force: bool,
+        detect_utf16: bool,
+        buffer_encoding: Option<Arc<std::sync::Mutex<&'static Encoding>>>,
         cx: &Context<Worktree>,
     ) -> Task<Result<LoadedFile>> {
         let path = Arc::from(path);
@@ -1357,7 +1365,9 @@ impl LocalWorktree {
                     } else {
                         EncodingWrapper::new(encoding_rs::UTF_8)
                     },
-                    false,
+                    force,
+                    detect_utf16,
+                    buffer_encoding,
                 )
                 .await?;
 
@@ -3139,13 +3149,15 @@ impl language::LocalFile for File {
         &self,
         cx: &App,
         encoding: EncodingWrapper,
+        force: bool,
         detect_utf16: bool,
+        buffer_encoding: Option<Arc<std::sync::Mutex<&'static Encoding>>>,
     ) -> Task<Result<String>> {
         let worktree = self.worktree.read(cx).as_local().unwrap();
         let abs_path = worktree.absolutize(&self.path);
         let fs = worktree.fs.clone();
         cx.background_spawn(async move {
-            fs.load_with_encoding(&abs_path?, encoding, detect_utf16)
+            fs.load_with_encoding(&abs_path?, encoding, force, detect_utf16, buffer_encoding)
                 .await
         })
     }

crates/worktree/src/worktree_tests.rs 🔗

@@ -468,7 +468,14 @@ async fn test_open_gitignored_files(cx: &mut TestAppContext) {
     let prev_read_dir_count = fs.read_dir_call_count();
     let loaded = tree
         .update(cx, |tree, cx| {
-            tree.load_file("one/node_modules/b/b1.js".as_ref(), None, cx)
+            tree.load_file(
+                "one/node_modules/b/b1.js".as_ref(),
+                None,
+                false,
+                false,
+                None,
+                cx,
+            )
         })
         .await
         .unwrap();
@@ -508,7 +515,14 @@ async fn test_open_gitignored_files(cx: &mut TestAppContext) {
     let prev_read_dir_count = fs.read_dir_call_count();
     let loaded = tree
         .update(cx, |tree, cx| {
-            tree.load_file("one/node_modules/a/a2.js".as_ref(), None, cx)
+            tree.load_file(
+                "one/node_modules/a/a2.js".as_ref(),
+                None,
+                false,
+                false,
+                None,
+                cx,
+            )
         })
         .await
         .unwrap();
@@ -1954,6 +1968,101 @@ fn random_filename(rng: &mut impl Rng) -> String {
         .collect()
 }
 
+#[gpui::test]
+async fn test_rename_file_to_new_directory(cx: &mut TestAppContext) {
+    init_test(cx);
+    let fs = FakeFs::new(cx.background_executor.clone());
+    let expected_contents = "content";
+    fs.as_fake()
+        .insert_tree(
+            "/root",
+            json!({
+                "test.txt": expected_contents
+            }),
+        )
+        .await;
+    let worktree = Worktree::local(
+        Path::new("/root"),
+        true,
+        fs.clone(),
+        Arc::default(),
+        &mut cx.to_async(),
+    )
+    .await
+    .unwrap();
+    cx.read(|cx| worktree.read(cx).as_local().unwrap().scan_complete())
+        .await;
+
+    let entry_id = worktree.read_with(cx, |worktree, _| {
+        worktree.entry_for_path("test.txt").unwrap().id
+    });
+    let _result = worktree
+        .update(cx, |worktree, cx| {
+            worktree.rename_entry(entry_id, Path::new("dir1/dir2/dir3/test.txt"), cx)
+        })
+        .await
+        .unwrap();
+    worktree.read_with(cx, |worktree, _| {
+        assert!(
+            worktree.entry_for_path("test.txt").is_none(),
+            "Old file should have been removed"
+        );
+        assert!(
+            worktree.entry_for_path("dir1/dir2/dir3/test.txt").is_some(),
+            "Whole directory hierarchy and the new file should have been created"
+        );
+    });
+    assert_eq!(
+        worktree
+            .update(cx, |worktree, cx| {
+                worktree.load_file("dir1/dir2/dir3/test.txt".as_ref(), None, cx)
+            })
+            .await
+            .unwrap()
+            .text,
+        expected_contents,
+        "Moved file's contents should be preserved"
+    );
+
+    let entry_id = worktree.read_with(cx, |worktree, _| {
+        worktree
+            .entry_for_path("dir1/dir2/dir3/test.txt")
+            .unwrap()
+            .id
+    });
+    let _result = worktree
+        .update(cx, |worktree, cx| {
+            worktree.rename_entry(entry_id, Path::new("dir1/dir2/test.txt"), cx)
+        })
+        .await
+        .unwrap();
+    worktree.read_with(cx, |worktree, _| {
+        assert!(
+            worktree.entry_for_path("test.txt").is_none(),
+            "First file should not reappear"
+        );
+        assert!(
+            worktree.entry_for_path("dir1/dir2/dir3/test.txt").is_none(),
+            "Old file should have been removed"
+        );
+        assert!(
+            worktree.entry_for_path("dir1/dir2/test.txt").is_some(),
+            "No error should have occurred after moving into existing directory"
+        );
+    });
+    assert_eq!(
+        worktree
+            .update(cx, |worktree, cx| {
+                worktree.load_file("dir1/dir2/test.txt".as_ref(), None, cx)
+            })
+            .await
+            .unwrap()
+            .text,
+        expected_contents,
+        "Moved file's contents should be preserved"
+    );
+}
+
 #[gpui::test]
 async fn test_private_single_file_worktree(cx: &mut TestAppContext) {
     init_test(cx);

crates/zed_actions/src/lib.rs 🔗

@@ -308,6 +308,9 @@ pub mod encodings {
 
     #[derive(PartialEq, Debug, Clone, Action, JsonSchema, Deserialize)]
     pub struct Toggle(pub Arc<std::path::Path>);
+
+    #[derive(PartialEq, Debug, Clone, Action, JsonSchema, Deserialize)]
+    pub struct ForceOpen(pub Arc<std::path::Path>);
 }
 
 pub mod agent {

crates/zeta/src/zeta.rs 🔗

@@ -1986,7 +1986,7 @@ mod tests {
                     .worktree_for_root_name("closed_source_worktree", cx)
                     .unwrap();
                 worktree2.update(cx, |worktree2, cx| {
-                    worktree2.load_file(Path::new("main.rs"), None, cx)
+                    worktree2.load_file(Path::new("main.rs"), None, false, true, None, cx)
                 })
             })
             .await