@@ -523,14 +523,7 @@ pub fn into_anthropic(
match message.role {
Role::User | Role::Assistant => {
- let cache_control = if message.cache {
- Some(anthropic::CacheControl {
- cache_type: anthropic::CacheControlType::Ephemeral,
- })
- } else {
- None
- };
- let anthropic_message_content: Vec<anthropic::RequestContent> = message
+ let mut anthropic_message_content: Vec<anthropic::RequestContent> = message
.content
.into_iter()
.filter_map(|content| match content {
@@ -538,7 +531,7 @@ pub fn into_anthropic(
if !text.is_empty() {
Some(anthropic::RequestContent::Text {
text,
- cache_control,
+ cache_control: None,
})
} else {
None
@@ -552,7 +545,7 @@ pub fn into_anthropic(
Some(anthropic::RequestContent::Thinking {
thinking,
signature: signature.unwrap_or_default(),
- cache_control,
+ cache_control: None,
})
} else {
None
@@ -573,14 +566,14 @@ pub fn into_anthropic(
media_type: "image/png".to_string(),
data: image.source.to_string(),
},
- cache_control,
+ cache_control: None,
}),
MessageContent::ToolUse(tool_use) => {
Some(anthropic::RequestContent::ToolUse {
id: tool_use.id.to_string(),
name: tool_use.name.to_string(),
input: tool_use.input,
- cache_control,
+ cache_control: None,
})
}
MessageContent::ToolResult(tool_result) => {
@@ -601,7 +594,7 @@ pub fn into_anthropic(
}])
}
},
- cache_control,
+ cache_control: None,
})
}
})
@@ -617,6 +610,29 @@ pub fn into_anthropic(
continue;
}
}
+
+ // Mark the last segment of the message as cached
+ if message.cache {
+ let cache_control_value = Some(anthropic::CacheControl {
+ cache_type: anthropic::CacheControlType::Ephemeral,
+ });
+ for message_content in anthropic_message_content.iter_mut().rev() {
+ match message_content {
+ anthropic::RequestContent::RedactedThinking { .. } => {
+ // Caching is not possible, fallback to next message
+ }
+ anthropic::RequestContent::Text { cache_control, .. }
+ | anthropic::RequestContent::Thinking { cache_control, .. }
+ | anthropic::RequestContent::Image { cache_control, .. }
+ | anthropic::RequestContent::ToolUse { cache_control, .. }
+ | anthropic::RequestContent::ToolResult { cache_control, .. } => {
+ *cache_control = cache_control_value;
+ break;
+ }
+ }
+ }
+ }
+
new_messages.push(anthropic::Message {
role: anthropic_role,
content: anthropic_message_content,
@@ -1068,3 +1084,75 @@ impl Render for ConfigurationView {
}
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use anthropic::AnthropicModelMode;
+ use language_model::{LanguageModelRequestMessage, MessageContent};
+
+ #[test]
+ fn test_cache_control_only_on_last_segment() {
+ let request = LanguageModelRequest {
+ messages: vec![LanguageModelRequestMessage {
+ role: Role::User,
+ content: vec![
+ MessageContent::Text("Some prompt".to_string()),
+ MessageContent::Image(language_model::LanguageModelImage::empty()),
+ MessageContent::Image(language_model::LanguageModelImage::empty()),
+ MessageContent::Image(language_model::LanguageModelImage::empty()),
+ MessageContent::Image(language_model::LanguageModelImage::empty()),
+ ],
+ cache: true,
+ }],
+ thread_id: None,
+ prompt_id: None,
+ intent: None,
+ mode: None,
+ stop: vec![],
+ temperature: None,
+ tools: vec![],
+ tool_choice: None,
+ };
+
+ let anthropic_request = into_anthropic(
+ request,
+ "claude-3-5-sonnet".to_string(),
+ 0.7,
+ 4096,
+ AnthropicModelMode::Default,
+ );
+
+ assert_eq!(anthropic_request.messages.len(), 1);
+
+ let message = &anthropic_request.messages[0];
+ assert_eq!(message.content.len(), 5);
+
+ assert!(matches!(
+ message.content[0],
+ anthropic::RequestContent::Text {
+ cache_control: None,
+ ..
+ }
+ ));
+ for i in 1..3 {
+ assert!(matches!(
+ message.content[i],
+ anthropic::RequestContent::Image {
+ cache_control: None,
+ ..
+ }
+ ));
+ }
+
+ assert!(matches!(
+ message.content[4],
+ anthropic::RequestContent::Image {
+ cache_control: Some(anthropic::CacheControl {
+ cache_type: anthropic::CacheControlType::Ephemeral,
+ }),
+ ..
+ }
+ ));
+ }
+}