diff --git a/crates/agent/src/agent.rs b/crates/agent/src/agent.rs
index 95ceb26c4f872133e48c2b0b7725a0bf7dd02d1f..f0e4365baec7b63de9eefd2f9ce2cef02e5d361c 100644
--- a/crates/agent/src/agent.rs
+++ b/crates/agent/src/agent.rs
@@ -117,6 +117,7 @@ pub fn init(
     client: Arc<Client>,
     prompt_builder: Arc<PromptBuilder>,
     language_registry: Arc<LanguageRegistry>,
+    is_eval: bool,
     cx: &mut App,
 ) {
     AssistantSettings::register(cx);
@@ -124,7 +125,11 @@ pub fn init(
 
     assistant_context_editor::init(client.clone(), cx);
     rules_library::init(cx);
-    init_language_model_settings(cx);
+    if !is_eval {
+        // Initializing the language model from the user settings messes with the eval, so we only initialize them when
+        // we're not running inside of the eval.
+        init_language_model_settings(cx);
+    }
     assistant_slash_command::init(cx);
     thread_store::init(cx);
     agent_panel::init(cx);
diff --git a/crates/eval/src/eval.rs b/crates/eval/src/eval.rs
index f42d138f282d60067c56568dce556beae96d3438..064a0c688e8fefda0480e0d65cc14e359686a318 100644
--- a/crates/eval/src/eval.rs
+++ b/crates/eval/src/eval.rs
@@ -432,6 +432,7 @@ pub fn init(cx: &mut App) -> Arc<AgentAppState> {
         client.clone(),
         prompt_builder.clone(),
         languages.clone(),
+        true,
         cx,
     );
     assistant_tools::init(client.http_client(), cx);
diff --git a/crates/zed/src/main.rs b/crates/zed/src/main.rs
index 99fa59ee724e61c58b2a3b64e4137438443eaacb..1bdef5456c08dca69fa60ab787753c94bc8c5a9a 100644
--- a/crates/zed/src/main.rs
+++ b/crates/zed/src/main.rs
@@ -519,6 +519,7 @@ fn main() {
             app_state.client.clone(),
             prompt_builder.clone(),
             app_state.languages.clone(),
+            false,
             cx,
         );
         assistant_tools::init(app_state.client.http_client(), cx);
diff --git a/crates/zed/src/zed.rs b/crates/zed/src/zed.rs
index c03a2b6004dad3248fc898deb0f3e029bcddf6a3..77c02e3faacccf8f90792fb63a71dca5c850308e 100644
--- a/crates/zed/src/zed.rs
+++ b/crates/zed/src/zed.rs
@@ -4295,6 +4295,7 @@ mod tests {
                 app_state.client.clone(),
                 prompt_builder.clone(),
                 app_state.languages.clone(),
+                false,
                 cx,
             );
             repl::init(app_state.fs.clone(), cx);