1use std::mem;
2
3use anyhow::{Result, anyhow, bail};
4use futures::{AsyncBufReadExt, AsyncReadExt, StreamExt, io::BufReader, stream::BoxStream};
5use http_client::{AsyncBody, HttpClient, Method, Request as HttpRequest};
6use serde::{Deserialize, Deserializer, Serialize, Serializer};
7
8pub const API_URL: &str = "https://generativelanguage.googleapis.com";
9
10pub async fn stream_generate_content(
11 client: &dyn HttpClient,
12 api_url: &str,
13 api_key: &str,
14 mut request: GenerateContentRequest,
15) -> Result<BoxStream<'static, Result<GenerateContentResponse>>> {
16 validate_generate_content_request(&request)?;
17
18 // The `model` field is emptied as it is provided as a path parameter.
19 let model_id = mem::take(&mut request.model.model_id);
20
21 let uri =
22 format!("{api_url}/v1beta/models/{model_id}:streamGenerateContent?alt=sse&key={api_key}",);
23
24 let request_builder = HttpRequest::builder()
25 .method(Method::POST)
26 .uri(uri)
27 .header("Content-Type", "application/json");
28
29 let request = request_builder.body(AsyncBody::from(serde_json::to_string(&request)?))?;
30 let mut response = client.send(request).await?;
31 if response.status().is_success() {
32 let reader = BufReader::new(response.into_body());
33 Ok(reader
34 .lines()
35 .filter_map(|line| async move {
36 match line {
37 Ok(line) => {
38 if let Some(line) = line.strip_prefix("data: ") {
39 match serde_json::from_str(line) {
40 Ok(response) => Some(Ok(response)),
41 Err(error) => Some(Err(anyhow!(format!(
42 "Error parsing JSON: {error:?}\n{line:?}"
43 )))),
44 }
45 } else {
46 None
47 }
48 }
49 Err(error) => Some(Err(anyhow!(error))),
50 }
51 })
52 .boxed())
53 } else {
54 let mut text = String::new();
55 response.body_mut().read_to_string(&mut text).await?;
56 Err(anyhow!(
57 "error during streamGenerateContent, status code: {:?}, body: {}",
58 response.status(),
59 text
60 ))
61 }
62}
63
64pub async fn count_tokens(
65 client: &dyn HttpClient,
66 api_url: &str,
67 api_key: &str,
68 request: CountTokensRequest,
69) -> Result<CountTokensResponse> {
70 validate_generate_content_request(&request.generate_content_request)?;
71
72 let uri = format!(
73 "{api_url}/v1beta/models/{model_id}:countTokens?key={api_key}",
74 model_id = &request.generate_content_request.model.model_id,
75 );
76
77 let request = serde_json::to_string(&request)?;
78 let request_builder = HttpRequest::builder()
79 .method(Method::POST)
80 .uri(&uri)
81 .header("Content-Type", "application/json");
82 let http_request = request_builder.body(AsyncBody::from(request))?;
83
84 let mut response = client.send(http_request).await?;
85 let mut text = String::new();
86 response.body_mut().read_to_string(&mut text).await?;
87 anyhow::ensure!(
88 response.status().is_success(),
89 "error during countTokens, status code: {:?}, body: {}",
90 response.status(),
91 text
92 );
93 Ok(serde_json::from_str::<CountTokensResponse>(&text)?)
94}
95
96pub fn validate_generate_content_request(request: &GenerateContentRequest) -> Result<()> {
97 if request.model.is_empty() {
98 bail!("Model must be specified");
99 }
100
101 if request.contents.is_empty() {
102 bail!("Request must contain at least one content item");
103 }
104
105 if let Some(user_content) = request
106 .contents
107 .iter()
108 .find(|content| content.role == Role::User)
109 && user_content.parts.is_empty()
110 {
111 bail!("User content must contain at least one part");
112 }
113
114 Ok(())
115}
116
117#[derive(Debug, Serialize, Deserialize)]
118pub enum Task {
119 #[serde(rename = "generateContent")]
120 GenerateContent,
121 #[serde(rename = "streamGenerateContent")]
122 StreamGenerateContent,
123 #[serde(rename = "countTokens")]
124 CountTokens,
125 #[serde(rename = "embedContent")]
126 EmbedContent,
127 #[serde(rename = "batchEmbedContents")]
128 BatchEmbedContents,
129}
130
131#[derive(Debug, Serialize, Deserialize)]
132#[serde(rename_all = "camelCase")]
133pub struct GenerateContentRequest {
134 #[serde(default, skip_serializing_if = "ModelName::is_empty")]
135 pub model: ModelName,
136 pub contents: Vec<Content>,
137 #[serde(skip_serializing_if = "Option::is_none")]
138 pub system_instruction: Option<SystemInstruction>,
139 #[serde(skip_serializing_if = "Option::is_none")]
140 pub generation_config: Option<GenerationConfig>,
141 #[serde(skip_serializing_if = "Option::is_none")]
142 pub safety_settings: Option<Vec<SafetySetting>>,
143 #[serde(skip_serializing_if = "Option::is_none")]
144 pub tools: Option<Vec<Tool>>,
145 #[serde(skip_serializing_if = "Option::is_none")]
146 pub tool_config: Option<ToolConfig>,
147}
148
149#[derive(Debug, Serialize, Deserialize)]
150#[serde(rename_all = "camelCase")]
151pub struct GenerateContentResponse {
152 #[serde(skip_serializing_if = "Option::is_none")]
153 pub candidates: Option<Vec<GenerateContentCandidate>>,
154 #[serde(skip_serializing_if = "Option::is_none")]
155 pub prompt_feedback: Option<PromptFeedback>,
156 #[serde(skip_serializing_if = "Option::is_none")]
157 pub usage_metadata: Option<UsageMetadata>,
158}
159
160#[derive(Debug, Serialize, Deserialize)]
161#[serde(rename_all = "camelCase")]
162pub struct GenerateContentCandidate {
163 #[serde(skip_serializing_if = "Option::is_none")]
164 pub index: Option<usize>,
165 pub content: Content,
166 #[serde(skip_serializing_if = "Option::is_none")]
167 pub finish_reason: Option<String>,
168 #[serde(skip_serializing_if = "Option::is_none")]
169 pub finish_message: Option<String>,
170 #[serde(skip_serializing_if = "Option::is_none")]
171 pub safety_ratings: Option<Vec<SafetyRating>>,
172 #[serde(skip_serializing_if = "Option::is_none")]
173 pub citation_metadata: Option<CitationMetadata>,
174}
175
176#[derive(Debug, Serialize, Deserialize)]
177#[serde(rename_all = "camelCase")]
178pub struct Content {
179 #[serde(default)]
180 pub parts: Vec<Part>,
181 pub role: Role,
182}
183
184#[derive(Debug, Serialize, Deserialize)]
185#[serde(rename_all = "camelCase")]
186pub struct SystemInstruction {
187 pub parts: Vec<Part>,
188}
189
190#[derive(Debug, PartialEq, Deserialize, Serialize)]
191#[serde(rename_all = "camelCase")]
192pub enum Role {
193 User,
194 Model,
195}
196
197#[derive(Debug, Serialize, Deserialize)]
198#[serde(untagged)]
199pub enum Part {
200 TextPart(TextPart),
201 InlineDataPart(InlineDataPart),
202 FunctionCallPart(FunctionCallPart),
203 FunctionResponsePart(FunctionResponsePart),
204 ThoughtPart(ThoughtPart),
205}
206
207#[derive(Debug, Serialize, Deserialize)]
208#[serde(rename_all = "camelCase")]
209pub struct TextPart {
210 pub text: String,
211}
212
213#[derive(Debug, Serialize, Deserialize)]
214#[serde(rename_all = "camelCase")]
215pub struct InlineDataPart {
216 pub inline_data: GenerativeContentBlob,
217}
218
219#[derive(Debug, Serialize, Deserialize)]
220#[serde(rename_all = "camelCase")]
221pub struct GenerativeContentBlob {
222 pub mime_type: String,
223 pub data: String,
224}
225
226#[derive(Debug, Serialize, Deserialize)]
227#[serde(rename_all = "camelCase")]
228pub struct FunctionCallPart {
229 pub function_call: FunctionCall,
230}
231
232#[derive(Debug, Serialize, Deserialize)]
233#[serde(rename_all = "camelCase")]
234pub struct FunctionResponsePart {
235 pub function_response: FunctionResponse,
236}
237
238#[derive(Debug, Serialize, Deserialize)]
239#[serde(rename_all = "camelCase")]
240pub struct ThoughtPart {
241 pub thought: bool,
242 pub thought_signature: String,
243}
244
245#[derive(Debug, Serialize, Deserialize)]
246#[serde(rename_all = "camelCase")]
247pub struct CitationSource {
248 #[serde(skip_serializing_if = "Option::is_none")]
249 pub start_index: Option<usize>,
250 #[serde(skip_serializing_if = "Option::is_none")]
251 pub end_index: Option<usize>,
252 #[serde(skip_serializing_if = "Option::is_none")]
253 pub uri: Option<String>,
254 #[serde(skip_serializing_if = "Option::is_none")]
255 pub license: Option<String>,
256}
257
258#[derive(Debug, Serialize, Deserialize)]
259#[serde(rename_all = "camelCase")]
260pub struct CitationMetadata {
261 pub citation_sources: Vec<CitationSource>,
262}
263
264#[derive(Debug, Serialize, Deserialize)]
265#[serde(rename_all = "camelCase")]
266pub struct PromptFeedback {
267 #[serde(skip_serializing_if = "Option::is_none")]
268 pub block_reason: Option<String>,
269 pub safety_ratings: Option<Vec<SafetyRating>>,
270 #[serde(skip_serializing_if = "Option::is_none")]
271 pub block_reason_message: Option<String>,
272}
273
274#[derive(Debug, Serialize, Deserialize, Default)]
275#[serde(rename_all = "camelCase")]
276pub struct UsageMetadata {
277 #[serde(skip_serializing_if = "Option::is_none")]
278 pub prompt_token_count: Option<u64>,
279 #[serde(skip_serializing_if = "Option::is_none")]
280 pub cached_content_token_count: Option<u64>,
281 #[serde(skip_serializing_if = "Option::is_none")]
282 pub candidates_token_count: Option<u64>,
283 #[serde(skip_serializing_if = "Option::is_none")]
284 pub tool_use_prompt_token_count: Option<u64>,
285 #[serde(skip_serializing_if = "Option::is_none")]
286 pub thoughts_token_count: Option<u64>,
287 #[serde(skip_serializing_if = "Option::is_none")]
288 pub total_token_count: Option<u64>,
289}
290
291#[derive(Debug, Serialize, Deserialize)]
292#[serde(rename_all = "camelCase")]
293pub struct ThinkingConfig {
294 pub thinking_budget: u32,
295}
296
297#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
298#[derive(Copy, Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
299pub enum GoogleModelMode {
300 #[default]
301 Default,
302 Thinking {
303 budget_tokens: Option<u32>,
304 },
305}
306
307#[derive(Debug, Deserialize, Serialize)]
308#[serde(rename_all = "camelCase")]
309pub struct GenerationConfig {
310 #[serde(skip_serializing_if = "Option::is_none")]
311 pub candidate_count: Option<usize>,
312 #[serde(skip_serializing_if = "Option::is_none")]
313 pub stop_sequences: Option<Vec<String>>,
314 #[serde(skip_serializing_if = "Option::is_none")]
315 pub max_output_tokens: Option<usize>,
316 #[serde(skip_serializing_if = "Option::is_none")]
317 pub temperature: Option<f64>,
318 #[serde(skip_serializing_if = "Option::is_none")]
319 pub top_p: Option<f64>,
320 #[serde(skip_serializing_if = "Option::is_none")]
321 pub top_k: Option<usize>,
322 #[serde(skip_serializing_if = "Option::is_none")]
323 pub thinking_config: Option<ThinkingConfig>,
324}
325
326#[derive(Debug, Serialize, Deserialize)]
327#[serde(rename_all = "camelCase")]
328pub struct SafetySetting {
329 pub category: HarmCategory,
330 pub threshold: HarmBlockThreshold,
331}
332
333#[derive(Debug, Serialize, Deserialize)]
334pub enum HarmCategory {
335 #[serde(rename = "HARM_CATEGORY_UNSPECIFIED")]
336 Unspecified,
337 #[serde(rename = "HARM_CATEGORY_DEROGATORY")]
338 Derogatory,
339 #[serde(rename = "HARM_CATEGORY_TOXICITY")]
340 Toxicity,
341 #[serde(rename = "HARM_CATEGORY_VIOLENCE")]
342 Violence,
343 #[serde(rename = "HARM_CATEGORY_SEXUAL")]
344 Sexual,
345 #[serde(rename = "HARM_CATEGORY_MEDICAL")]
346 Medical,
347 #[serde(rename = "HARM_CATEGORY_DANGEROUS")]
348 Dangerous,
349 #[serde(rename = "HARM_CATEGORY_HARASSMENT")]
350 Harassment,
351 #[serde(rename = "HARM_CATEGORY_HATE_SPEECH")]
352 HateSpeech,
353 #[serde(rename = "HARM_CATEGORY_SEXUALLY_EXPLICIT")]
354 SexuallyExplicit,
355 #[serde(rename = "HARM_CATEGORY_DANGEROUS_CONTENT")]
356 DangerousContent,
357}
358
359#[derive(Debug, Serialize, Deserialize)]
360#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
361pub enum HarmBlockThreshold {
362 #[serde(rename = "HARM_BLOCK_THRESHOLD_UNSPECIFIED")]
363 Unspecified,
364 BlockLowAndAbove,
365 BlockMediumAndAbove,
366 BlockOnlyHigh,
367 BlockNone,
368}
369
370#[derive(Debug, Serialize, Deserialize)]
371#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
372pub enum HarmProbability {
373 #[serde(rename = "HARM_PROBABILITY_UNSPECIFIED")]
374 Unspecified,
375 Negligible,
376 Low,
377 Medium,
378 High,
379}
380
381#[derive(Debug, Serialize, Deserialize)]
382#[serde(rename_all = "camelCase")]
383pub struct SafetyRating {
384 pub category: HarmCategory,
385 pub probability: HarmProbability,
386}
387
388#[derive(Debug, Serialize, Deserialize)]
389#[serde(rename_all = "camelCase")]
390pub struct CountTokensRequest {
391 pub generate_content_request: GenerateContentRequest,
392}
393
394#[derive(Debug, Serialize, Deserialize)]
395#[serde(rename_all = "camelCase")]
396pub struct CountTokensResponse {
397 pub total_tokens: u64,
398}
399
400#[derive(Debug, Serialize, Deserialize)]
401pub struct FunctionCall {
402 pub name: String,
403 pub args: serde_json::Value,
404}
405
406#[derive(Debug, Serialize, Deserialize)]
407pub struct FunctionResponse {
408 pub name: String,
409 pub response: serde_json::Value,
410}
411
412#[derive(Debug, Serialize, Deserialize)]
413#[serde(rename_all = "camelCase")]
414pub struct Tool {
415 pub function_declarations: Vec<FunctionDeclaration>,
416}
417
418#[derive(Debug, Serialize, Deserialize)]
419#[serde(rename_all = "camelCase")]
420pub struct ToolConfig {
421 pub function_calling_config: FunctionCallingConfig,
422}
423
424#[derive(Debug, Serialize, Deserialize)]
425#[serde(rename_all = "camelCase")]
426pub struct FunctionCallingConfig {
427 pub mode: FunctionCallingMode,
428 #[serde(skip_serializing_if = "Option::is_none")]
429 pub allowed_function_names: Option<Vec<String>>,
430}
431
432#[derive(Debug, Serialize, Deserialize)]
433#[serde(rename_all = "lowercase")]
434pub enum FunctionCallingMode {
435 Auto,
436 Any,
437 None,
438}
439
440#[derive(Debug, Serialize, Deserialize)]
441pub struct FunctionDeclaration {
442 pub name: String,
443 pub description: String,
444 pub parameters: serde_json::Value,
445}
446
447#[derive(Debug, Default)]
448pub struct ModelName {
449 pub model_id: String,
450}
451
452impl ModelName {
453 pub fn is_empty(&self) -> bool {
454 self.model_id.is_empty()
455 }
456}
457
458const MODEL_NAME_PREFIX: &str = "models/";
459
460impl Serialize for ModelName {
461 fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
462 where
463 S: Serializer,
464 {
465 serializer.serialize_str(&format!("{MODEL_NAME_PREFIX}{}", &self.model_id))
466 }
467}
468
469impl<'de> Deserialize<'de> for ModelName {
470 fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
471 where
472 D: Deserializer<'de>,
473 {
474 let string = String::deserialize(deserializer)?;
475 if let Some(id) = string.strip_prefix(MODEL_NAME_PREFIX) {
476 Ok(Self {
477 model_id: id.to_string(),
478 })
479 } else {
480 Err(serde::de::Error::custom(format!(
481 "Expected model name to begin with {}, got: {}",
482 MODEL_NAME_PREFIX, string
483 )))
484 }
485 }
486}
487
488#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
489#[derive(Clone, Default, Debug, Deserialize, Serialize, PartialEq, Eq, strum::EnumIter)]
490pub enum Model {
491 #[serde(rename = "gemini-1.5-pro")]
492 Gemini15Pro,
493 #[serde(rename = "gemini-1.5-flash-8b")]
494 Gemini15Flash8b,
495 #[serde(rename = "gemini-1.5-flash")]
496 Gemini15Flash,
497 #[serde(
498 rename = "gemini-2.0-flash-lite",
499 alias = "gemini-2.0-flash-lite-preview"
500 )]
501 Gemini20FlashLite,
502 #[serde(rename = "gemini-2.0-flash")]
503 Gemini20Flash,
504 #[serde(
505 rename = "gemini-2.5-flash-lite-preview",
506 alias = "gemini-2.5-flash-lite-preview-06-17"
507 )]
508 Gemini25FlashLitePreview,
509 #[serde(
510 rename = "gemini-2.5-flash",
511 alias = "gemini-2.0-flash-thinking-exp",
512 alias = "gemini-2.5-flash-preview-04-17",
513 alias = "gemini-2.5-flash-preview-05-20",
514 alias = "gemini-2.5-flash-preview-latest"
515 )]
516 #[default]
517 Gemini25Flash,
518 #[serde(
519 rename = "gemini-2.5-pro",
520 alias = "gemini-2.0-pro-exp",
521 alias = "gemini-2.5-pro-preview-latest",
522 alias = "gemini-2.5-pro-exp-03-25",
523 alias = "gemini-2.5-pro-preview-03-25",
524 alias = "gemini-2.5-pro-preview-05-06",
525 alias = "gemini-2.5-pro-preview-06-05"
526 )]
527 Gemini25Pro,
528 #[serde(rename = "custom")]
529 Custom {
530 name: String,
531 /// The name displayed in the UI, such as in the assistant panel model dropdown menu.
532 display_name: Option<String>,
533 max_tokens: u64,
534 #[serde(default)]
535 mode: GoogleModelMode,
536 },
537}
538
539impl Model {
540 pub fn default_fast() -> Self {
541 Self::Gemini20FlashLite
542 }
543
544 pub fn id(&self) -> &str {
545 match self {
546 Self::Gemini15Pro => "gemini-1.5-pro",
547 Self::Gemini15Flash8b => "gemini-1.5-flash-8b",
548 Self::Gemini15Flash => "gemini-1.5-flash",
549 Self::Gemini20FlashLite => "gemini-2.0-flash-lite",
550 Self::Gemini20Flash => "gemini-2.0-flash",
551 Self::Gemini25FlashLitePreview => "gemini-2.5-flash-lite-preview",
552 Self::Gemini25Flash => "gemini-2.5-flash",
553 Self::Gemini25Pro => "gemini-2.5-pro",
554 Self::Custom { name, .. } => name,
555 }
556 }
557 pub fn request_id(&self) -> &str {
558 match self {
559 Self::Gemini15Pro => "gemini-1.5-pro",
560 Self::Gemini15Flash8b => "gemini-1.5-flash-8b",
561 Self::Gemini15Flash => "gemini-1.5-flash",
562 Self::Gemini20FlashLite => "gemini-2.0-flash-lite",
563 Self::Gemini20Flash => "gemini-2.0-flash",
564 Self::Gemini25FlashLitePreview => "gemini-2.5-flash-lite-preview-06-17",
565 Self::Gemini25Flash => "gemini-2.5-flash",
566 Self::Gemini25Pro => "gemini-2.5-pro",
567 Self::Custom { name, .. } => name,
568 }
569 }
570
571 pub fn display_name(&self) -> &str {
572 match self {
573 Self::Gemini15Pro => "Gemini 1.5 Pro",
574 Self::Gemini15Flash8b => "Gemini 1.5 Flash-8b",
575 Self::Gemini15Flash => "Gemini 1.5 Flash",
576 Self::Gemini20FlashLite => "Gemini 2.0 Flash-Lite",
577 Self::Gemini20Flash => "Gemini 2.0 Flash",
578 Self::Gemini25FlashLitePreview => "Gemini 2.5 Flash-Lite Preview",
579 Self::Gemini25Flash => "Gemini 2.5 Flash",
580 Self::Gemini25Pro => "Gemini 2.5 Pro",
581 Self::Custom {
582 name, display_name, ..
583 } => display_name.as_ref().unwrap_or(name),
584 }
585 }
586
587 pub fn max_token_count(&self) -> u64 {
588 match self {
589 Self::Gemini15Pro => 2_097_152,
590 Self::Gemini15Flash8b => 1_048_576,
591 Self::Gemini15Flash => 1_048_576,
592 Self::Gemini20FlashLite => 1_048_576,
593 Self::Gemini20Flash => 1_048_576,
594 Self::Gemini25FlashLitePreview => 1_000_000,
595 Self::Gemini25Flash => 1_048_576,
596 Self::Gemini25Pro => 1_048_576,
597 Self::Custom { max_tokens, .. } => *max_tokens,
598 }
599 }
600
601 pub fn max_output_tokens(&self) -> Option<u64> {
602 match self {
603 Model::Gemini15Pro => Some(8_192),
604 Model::Gemini15Flash8b => Some(8_192),
605 Model::Gemini15Flash => Some(8_192),
606 Model::Gemini20FlashLite => Some(8_192),
607 Model::Gemini20Flash => Some(8_192),
608 Model::Gemini25FlashLitePreview => Some(64_000),
609 Model::Gemini25Flash => Some(65_536),
610 Model::Gemini25Pro => Some(65_536),
611 Model::Custom { .. } => None,
612 }
613 }
614
615 pub fn supports_tools(&self) -> bool {
616 true
617 }
618
619 pub fn supports_images(&self) -> bool {
620 true
621 }
622
623 pub fn mode(&self) -> GoogleModelMode {
624 match self {
625 Self::Gemini15Pro
626 | Self::Gemini15Flash8b
627 | Self::Gemini15Flash
628 | Self::Gemini20FlashLite
629 | Self::Gemini20Flash => GoogleModelMode::Default,
630 Self::Gemini25FlashLitePreview | Self::Gemini25Flash | Self::Gemini25Pro => {
631 GoogleModelMode::Thinking {
632 // By default these models are set to "auto", so we preserve that behavior
633 // but indicate they are capable of thinking mode
634 budget_tokens: None,
635 }
636 }
637 Self::Custom { mode, .. } => *mode,
638 }
639 }
640}
641
642impl std::fmt::Display for Model {
643 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
644 write!(f, "{}", self.id())
645 }
646}