Cargo.lock 🔗
@@ -5019,6 +5019,7 @@ version = "0.1.0"
dependencies = [
"agent",
"anyhow",
+ "assistant_settings",
"assistant_tool",
"assistant_tools",
"async-trait",
agent: Handle attempts to use hallucinated tools (#29946)
Oleksiy Syvokon created
This change:
1. Catches attempts to use missing tools. If this happens, we now send
Agent a message listing available tools, after which Agent can
gracefully recover. Prior behavior: thread would stop in a broken state.
Example of a hallucinated call and a message we send back:
2. Adds evals for hallucinated tool use and imagined edits
3. Adds ability to configure a profile name in evals.
Release Notes:
- N/AChange summary
Cargo.lock | 1
crates/agent/src/active_thread.rs | 16 ++++
crates/agent/src/agent_diff.rs | 1
crates/agent/src/thread.rs | 46 +++++++++++++
crates/eval/Cargo.toml | 1
crates/eval/src/example.rs | 8 ++
crates/eval/src/examples/add_arg_to_trait_method.rs | 2
crates/eval/src/examples/code_block_citations.rs | 2
crates/eval/src/examples/comment_translation.rs | 2
crates/eval/src/examples/file_search.rs | 2
crates/eval/src/examples/hallucinated_tool_calls.toml | 13 +++
crates/eval/src/examples/mod.rs | 10 ++
crates/eval/src/examples/planets.rs | 2
crates/eval/src/instance.rs | 5 +
14 files changed, 111 insertions(+)
Detailed changes
crates/agent/src/active_thread.rs 🔗
@@ -1070,6 +1070,22 @@ impl ActiveThread {
cx,
);
}
+ ThreadEvent::MissingToolUse {
+ tool_use_id,
+ ui_text,
+ } => {
+ self.render_tool_use_markdown(
+ tool_use_id.clone(),
+ ui_text,
+ "",
+ self.thread
+ .read(cx)
+ .output_for_tool(tool_use_id)
+ .map(|output| output.clone().into())
+ .unwrap_or("".into()),
+ cx,
+ );
+ }
}
}
crates/agent/src/agent_diff.rs 🔗
@@ -1372,6 +1372,7 @@ impl AgentDiff {
| ThreadEvent::StreamedAssistantThinking(_, _)
| ThreadEvent::StreamedToolUse { .. }
| ThreadEvent::InvalidToolInput { .. }
+ | ThreadEvent::MissingToolUse { .. }
| ThreadEvent::MessageAdded(_)
| ThreadEvent::MessageEdited(_)
| ThreadEvent::MessageDeleted(_)
crates/agent/src/thread.rs 🔗
@@ -1911,12 +1911,54 @@ impl Thread {
cx,
);
}
+ } else {
+ self.handle_hallucinated_tool_use(
+ tool_use.id.clone(),
+ tool_use.name.clone(),
+ window,
+ cx,
+ );
}
}
pending_tool_uses
}
+ pub fn handle_hallucinated_tool_use(
+ &mut self,
+ tool_use_id: LanguageModelToolUseId,
+ hallucinated_tool_name: Arc<str>,
+ window: Option<AnyWindowHandle>,
+ cx: &mut Context<Thread>,
+ ) {
+ let available_tools = self.tools.read(cx).enabled_tools(cx);
+
+ let tool_list = available_tools
+ .iter()
+ .map(|tool| format!("- {}: {}", tool.name(), tool.description()))
+ .collect::<Vec<_>>()
+ .join("\n");
+
+ let error_message = format!(
+ "The tool '{}' doesn't exist or is not enabled. Available tools:\n{}",
+ hallucinated_tool_name, tool_list
+ );
+
+ let pending_tool_use = self.tool_use.insert_tool_output(
+ tool_use_id.clone(),
+ hallucinated_tool_name,
+ Err(anyhow!("Missing tool call: {error_message}")),
+ self.configured_model.as_ref(),
+ );
+
+ cx.emit(ThreadEvent::MissingToolUse {
+ tool_use_id: tool_use_id.clone(),
+ ui_text: error_message.into(),
+ });
+
+ self.tool_finished(tool_use_id, pending_tool_use, false, window, cx);
+ }
+
pub fn receive_invalid_tool_json(
&mut self,
tool_use_id: LanguageModelToolUseId,
@@ -2574,6 +2616,10 @@ pub enum ThreadEvent {
ui_text: Arc<str>,
input: serde_json::Value,
},
+ MissingToolUse {
+ tool_use_id: LanguageModelToolUseId,
+ ui_text: Arc<str>,
+ },
InvalidToolInput {
tool_use_id: LanguageModelToolUseId,
ui_text: Arc<str>,
crates/eval/Cargo.toml 🔗
@@ -20,6 +20,7 @@ path = "src/explorer.rs"
[dependencies]
agent.workspace = true
anyhow.workspace = true
+assistant_settings.workspace = true
assistant_tool.workspace = true
assistant_tools.workspace = true
async-trait.workspace = true
crates/eval/src/example.rs 🔗
@@ -12,6 +12,7 @@ use crate::{
};
use agent::{ContextLoadResult, Thread, ThreadEvent};
use anyhow::{Result, anyhow};
+use assistant_settings::AgentProfileId;
use async_trait::async_trait;
use buffer_diff::DiffHunkStatus;
use collections::HashMap;
@@ -46,6 +47,7 @@ pub struct ExampleMetadata {
pub revision: String,
pub language_server: Option<LanguageServer>,
pub max_assertions: Option<usize>,
+ pub profile_id: AgentProfileId,
}
#[derive(Clone, Debug)]
@@ -268,6 +270,12 @@ impl ExampleContext {
ThreadEvent::InvalidToolInput { .. } => {
println!("{log_prefix} invalid tool input");
}
+ ThreadEvent::MissingToolUse {
+ tool_use_id: _,
+ ui_text,
+ } => {
+ println!("{log_prefix} {ui_text}");
+ }
ThreadEvent::ToolConfirmationNeeded => {
panic!(
"{}Bug: Tool confirmation should not be required in eval",
crates/eval/src/examples/add_arg_to_trait_method.rs 🔗
@@ -1,6 +1,7 @@
use std::path::Path;
use anyhow::Result;
+use assistant_settings::AgentProfileId;
use async_trait::async_trait;
use crate::example::{Example, ExampleContext, ExampleMetadata, JudgeAssertion, LanguageServer};
@@ -19,6 +20,7 @@ impl Example for AddArgToTraitMethod {
allow_preexisting_diagnostics: false,
}),
max_assertions: None,
+ profile_id: AgentProfileId::default(),
}
}
crates/eval/src/examples/code_block_citations.rs 🔗
@@ -1,4 +1,5 @@
use anyhow::Result;
+use assistant_settings::AgentProfileId;
use async_trait::async_trait;
use markdown::PathWithRange;
@@ -20,6 +21,7 @@ impl Example for CodeBlockCitations {
allow_preexisting_diagnostics: false,
}),
max_assertions: None,
+ profile_id: AgentProfileId::default(),
}
}
crates/eval/src/examples/comment_translation.rs 🔗
@@ -1,5 +1,6 @@
use crate::example::{Example, ExampleContext, ExampleMetadata, JudgeAssertion};
use anyhow::Result;
+use assistant_settings::AgentProfileId;
use assistant_tools::StreamingEditFileToolInput;
use async_trait::async_trait;
@@ -14,6 +15,7 @@ impl Example for CommentTranslation {
revision: "504d084e29bce4f60614bc702e91af7f7d9e60ad".to_string(),
language_server: None,
max_assertions: Some(1),
+ profile_id: AgentProfileId::default(),
}
}
crates/eval/src/examples/file_search.rs 🔗
@@ -1,4 +1,5 @@
use anyhow::Result;
+use assistant_settings::AgentProfileId;
use assistant_tools::FindPathToolInput;
use async_trait::async_trait;
use regex::Regex;
@@ -16,6 +17,7 @@ impl Example for FileSearchExample {
revision: "03ecb88fe30794873f191ddb728f597935b3101c".to_string(),
language_server: None,
max_assertions: Some(3),
+ profile_id: AgentProfileId::default(),
}
}
crates/eval/src/examples/hallucinated_tool_calls.toml 🔗
@@ -0,0 +1,13 @@
+url = "https://github.com/jlowin/fastmcp"
+revision = "a2c1e14e5d83af1c32b76280ab368df199c4e860"
+language_extension = "py"
+
+prompt = "Write a LICENSE file just saying 'Apache 2.0' and nothing else"
+
+profile_name = "ask"
+
+[thread_assertions]
+
+no_edit_attempts = """The agent should not claim that it edited or created the file. It should not pretend making any changes."""
+
+mention_insufficient_tools = """Agent should mention that it doesn't have relevant tools needed to make the change."""
crates/eval/src/examples/mod.rs 🔗
@@ -1,4 +1,5 @@
use anyhow::Result;
+use assistant_settings::AgentProfileId;
use async_trait::async_trait;
use serde::Deserialize;
use std::collections::BTreeMap;
@@ -56,12 +57,19 @@ impl DeclarativeExample {
None
};
+ let profile_id = if let Some(profile_name) = base.profile_name {
+ AgentProfileId(profile_name.into())
+ } else {
+ AgentProfileId::default()
+ };
+
let metadata = ExampleMetadata {
name,
url: base.url,
revision: base.revision,
language_server,
max_assertions: None,
+ profile_id,
};
Ok(DeclarativeExample {
@@ -97,6 +105,8 @@ pub struct ExampleToml {
pub allow_preexisting_diagnostics: bool,
pub prompt: String,
#[serde(default)]
+ pub profile_name: Option<String>,
+ #[serde(default)]
pub diff_assertions: BTreeMap<String, String>,
#[serde(default)]
pub thread_assertions: BTreeMap<String, String>,
crates/eval/src/examples/planets.rs 🔗
@@ -1,4 +1,5 @@
use anyhow::Result;
+use assistant_settings::AgentProfileId;
use assistant_tool::Tool;
use assistant_tools::{OpenTool, TerminalTool};
use async_trait::async_trait;
@@ -16,6 +17,7 @@ impl Example for Planets {
revision: "59e49c75214f60b4dc4a45092292061c8c26ce27".to_string(), // so effectively a blank project.
language_server: None,
max_assertions: None,
+ profile_id: AgentProfileId::default(),
}
}
crates/eval/src/instance.rs 🔗
@@ -307,9 +307,14 @@ impl ExampleInstance {
std::fs::write(&last_diff_file_path, "")?;
let thread_store = thread_store.await?;
+
+ let profile_id = meta.profile_id.clone();
+ thread_store.update(cx, |thread_store, cx| thread_store.load_profile_by_id(profile_id, cx)).expect("Failed to load profile");
+
let thread =
thread_store.update(cx, |thread_store, cx| thread_store.create_thread(cx))?;
+
thread.update(cx, |thread, _cx| {
let mut request_count = 0;
let previous_diff = Rc::new(RefCell::new("".to_string()));