1use anyhow::Result;
 2use assistant_settings::AgentProfileId;
 3use assistant_tools::{EditFileMode, EditFileToolInput};
 4use async_trait::async_trait;
 5
 6use crate::example::{Example, ExampleContext, ExampleMetadata};
 7
 8pub struct FileOverwriteExample;
 9
10/*
11This eval tests a fix for a destructive behavior of the `edit_file` tool.
12Previously, it would rewrite existing files too aggressively, which often
13resulted in content loss.
14
15Pass rate before the fix: 10%
16Pass rate after the fix:  100%
17*/
18
19#[async_trait(?Send)]
20impl Example for FileOverwriteExample {
21    fn meta(&self) -> ExampleMetadata {
22        let thread_json = include_str!("threads/overwrite-file.json");
23
24        ExampleMetadata {
25            name: "file_overwrite".to_string(),
26            url: "https://github.com/zed-industries/zed.git".to_string(),
27            revision: "023a60806a8cc82e73bd8d88e63b4b07fc7a0040".to_string(),
28            language_server: None,
29            max_assertions: Some(1),
30            profile_id: AgentProfileId::default(),
31            existing_thread_json: Some(thread_json.to_string()),
32        }
33    }
34
35    async fn conversation(&self, cx: &mut ExampleContext) -> Result<()> {
36        let response = cx.run_turns(1).await?;
37        let file_overwritten = if let Some(tool_use) = response.find_tool_call("edit_file") {
38            let input = tool_use.parse_input::<EditFileToolInput>()?;
39            match input.mode {
40                EditFileMode::Edit => false,
41                EditFileMode::Create | EditFileMode::Overwrite => true,
42            }
43        } else {
44            false
45        };
46
47        cx.assert(!file_overwritten, "File should be edited, not overwritten")
48    }
49}