Handle new `refusal` stop reason from Claude 4 models (#31217)

Marshall Bowers created 6 months ago

This PR adds support for handling the new [`refusal` stop
reason](https://docs.anthropic.com/en/docs/test-and-evaluate/strengthen-guardrails/handle-streaming-refusals)
from Claude 4 models.

<img width="409" alt="Screenshot 2025-05-22 at 4 31 56 PM"
src="https://github.com/user-attachments/assets/707b04f5-5a52-4a19-95d9-cbd2be2dd86f"
/>

Release Notes:

- Added handling for `"stop_reason": "refusal"` from Claude 4 models.

Change summary

crates/agent/src/agent_diff.rs                   |  1 +
crates/agent/src/thread.rs                       | 10 ++++++++++
crates/assistant_context_editor/src/context.rs   |  1 +
crates/eval/src/example.rs                       |  4 ++++
crates/language_model/src/language_model.rs      |  1 +
crates/language_models/src/provider/anthropic.rs |  1 +
6 files changed, 18 insertions(+)

Detailed changes

crates/agent/src/agent_diff.rs

@@ -1348,6 +1348,7 @@ impl AgentDiff {
             ThreadEvent::NewRequest
             | ThreadEvent::Stopped(Ok(StopReason::EndTurn))
             | ThreadEvent::Stopped(Ok(StopReason::MaxTokens))
+            | ThreadEvent::Stopped(Ok(StopReason::Refusal))
             | ThreadEvent::Stopped(Err(_))
             | ThreadEvent::ShowError(_)
             | ThreadEvent::CompletionCanceled => {

crates/agent/src/thread.rs

@@ -1693,6 +1693,16 @@ impl Thread {
                                     project.set_agent_location(None, cx);
                                 });
                             }
+                            StopReason::Refusal => {
+                                thread.project.update(cx, |project, cx| {
+                                    project.set_agent_location(None, cx);
+                                });
+
+                                cx.emit (ThreadEvent::ShowError(ThreadError::Message {
+                                    header: "Language model refusal".into(),
+                                    message: "Model refused to generate content for safety reasons.".into(),
+                                }));
+                            }
                         },
                         Err(error) => {
                             thread.project.update(cx, |project, cx| {

crates/assistant_context_editor/src/context.rs

@@ -2204,6 +2204,7 @@ impl AssistantContext {
                             StopReason::ToolUse => {}
                             StopReason::EndTurn => {}
                             StopReason::MaxTokens => {}
+                            StopReason::Refusal => {}
                         }
                     }
                 })

crates/eval/src/example.rs

@@ -231,6 +231,10 @@ impl ExampleContext {
                     Ok(StopReason::MaxTokens) => {
                         tx.try_send(Err(anyhow!("Exceeded maximum tokens"))).ok();
                     }
+                    Ok(StopReason::Refusal) => {
+                        tx.try_send(Err(anyhow!("Model refused to generate content")))
+                            .ok();
+                    }
                     Err(err) => {
                         tx.try_send(Err(anyhow!(err.clone()))).ok();
                     }

crates/language_model/src/language_model.rs

@@ -100,6 +100,7 @@ pub enum StopReason {
     EndTurn,
     MaxTokens,
     ToolUse,
+    Refusal,
 }
 
 #[derive(Debug, Clone, Copy)]

crates/language_models/src/provider/anthropic.rs

@@ -825,6 +825,7 @@ impl AnthropicEventMapper {
                         "end_turn" => StopReason::EndTurn,
                         "max_tokens" => StopReason::MaxTokens,
                         "tool_use" => StopReason::ToolUse,
+                        "refusal" => StopReason::Refusal,
                         _ => {
                             log::error!("Unexpected anthropic stop_reason: {stop_reason}");
                             StopReason::EndTurn