1use std::mem;
2
3use anyhow::{Result, anyhow, bail};
4use futures::{AsyncBufReadExt, AsyncReadExt, StreamExt, io::BufReader, stream::BoxStream};
5use http_client::{AsyncBody, HttpClient, Method, Request as HttpRequest};
6use serde::{Deserialize, Deserializer, Serialize, Serializer};
7
8pub const API_URL: &str = "https://generativelanguage.googleapis.com";
9
10pub async fn stream_generate_content(
11 client: &dyn HttpClient,
12 api_url: &str,
13 api_key: &str,
14 mut request: GenerateContentRequest,
15) -> Result<BoxStream<'static, Result<GenerateContentResponse>>> {
16 validate_generate_content_request(&request)?;
17
18 // The `model` field is emptied as it is provided as a path parameter.
19 let model_id = mem::take(&mut request.model.model_id);
20
21 let uri =
22 format!("{api_url}/v1beta/models/{model_id}:streamGenerateContent?alt=sse&key={api_key}",);
23
24 let request_builder = HttpRequest::builder()
25 .method(Method::POST)
26 .uri(uri)
27 .header("Content-Type", "application/json");
28
29 let request = request_builder.body(AsyncBody::from(serde_json::to_string(&request)?))?;
30 let mut response = client.send(request).await?;
31 if response.status().is_success() {
32 let reader = BufReader::new(response.into_body());
33 Ok(reader
34 .lines()
35 .filter_map(|line| async move {
36 match line {
37 Ok(line) => {
38 if let Some(line) = line.strip_prefix("data: ") {
39 match serde_json::from_str(line) {
40 Ok(response) => Some(Ok(response)),
41 Err(error) => Some(Err(anyhow!(format!(
42 "Error parsing JSON: {error:?}\n{line:?}"
43 )))),
44 }
45 } else {
46 None
47 }
48 }
49 Err(error) => Some(Err(anyhow!(error))),
50 }
51 })
52 .boxed())
53 } else {
54 let mut text = String::new();
55 response.body_mut().read_to_string(&mut text).await?;
56 Err(anyhow!(
57 "error during streamGenerateContent, status code: {:?}, body: {}",
58 response.status(),
59 text
60 ))
61 }
62}
63
64pub async fn count_tokens(
65 client: &dyn HttpClient,
66 api_url: &str,
67 api_key: &str,
68 request: CountTokensRequest,
69) -> Result<CountTokensResponse> {
70 validate_generate_content_request(&request.generate_content_request)?;
71
72 let uri = format!(
73 "{api_url}/v1beta/models/{model_id}:countTokens?key={api_key}",
74 model_id = &request.generate_content_request.model.model_id,
75 );
76
77 let request = serde_json::to_string(&request)?;
78 let request_builder = HttpRequest::builder()
79 .method(Method::POST)
80 .uri(&uri)
81 .header("Content-Type", "application/json");
82 let http_request = request_builder.body(AsyncBody::from(request))?;
83
84 let mut response = client.send(http_request).await?;
85 let mut text = String::new();
86 response.body_mut().read_to_string(&mut text).await?;
87 anyhow::ensure!(
88 response.status().is_success(),
89 "error during countTokens, status code: {:?}, body: {}",
90 response.status(),
91 text
92 );
93 Ok(serde_json::from_str::<CountTokensResponse>(&text)?)
94}
95
96pub fn validate_generate_content_request(request: &GenerateContentRequest) -> Result<()> {
97 if request.model.is_empty() {
98 bail!("Model must be specified");
99 }
100
101 if request.contents.is_empty() {
102 bail!("Request must contain at least one content item");
103 }
104
105 if let Some(user_content) = request
106 .contents
107 .iter()
108 .find(|content| content.role == Role::User)
109 && user_content.parts.is_empty() {
110 bail!("User content must contain at least one part");
111 }
112
113 Ok(())
114}
115
116#[derive(Debug, Serialize, Deserialize)]
117pub enum Task {
118 #[serde(rename = "generateContent")]
119 GenerateContent,
120 #[serde(rename = "streamGenerateContent")]
121 StreamGenerateContent,
122 #[serde(rename = "countTokens")]
123 CountTokens,
124 #[serde(rename = "embedContent")]
125 EmbedContent,
126 #[serde(rename = "batchEmbedContents")]
127 BatchEmbedContents,
128}
129
130#[derive(Debug, Serialize, Deserialize)]
131#[serde(rename_all = "camelCase")]
132pub struct GenerateContentRequest {
133 #[serde(default, skip_serializing_if = "ModelName::is_empty")]
134 pub model: ModelName,
135 pub contents: Vec<Content>,
136 #[serde(skip_serializing_if = "Option::is_none")]
137 pub system_instruction: Option<SystemInstruction>,
138 #[serde(skip_serializing_if = "Option::is_none")]
139 pub generation_config: Option<GenerationConfig>,
140 #[serde(skip_serializing_if = "Option::is_none")]
141 pub safety_settings: Option<Vec<SafetySetting>>,
142 #[serde(skip_serializing_if = "Option::is_none")]
143 pub tools: Option<Vec<Tool>>,
144 #[serde(skip_serializing_if = "Option::is_none")]
145 pub tool_config: Option<ToolConfig>,
146}
147
148#[derive(Debug, Serialize, Deserialize)]
149#[serde(rename_all = "camelCase")]
150pub struct GenerateContentResponse {
151 #[serde(skip_serializing_if = "Option::is_none")]
152 pub candidates: Option<Vec<GenerateContentCandidate>>,
153 #[serde(skip_serializing_if = "Option::is_none")]
154 pub prompt_feedback: Option<PromptFeedback>,
155 #[serde(skip_serializing_if = "Option::is_none")]
156 pub usage_metadata: Option<UsageMetadata>,
157}
158
159#[derive(Debug, Serialize, Deserialize)]
160#[serde(rename_all = "camelCase")]
161pub struct GenerateContentCandidate {
162 #[serde(skip_serializing_if = "Option::is_none")]
163 pub index: Option<usize>,
164 pub content: Content,
165 #[serde(skip_serializing_if = "Option::is_none")]
166 pub finish_reason: Option<String>,
167 #[serde(skip_serializing_if = "Option::is_none")]
168 pub finish_message: Option<String>,
169 #[serde(skip_serializing_if = "Option::is_none")]
170 pub safety_ratings: Option<Vec<SafetyRating>>,
171 #[serde(skip_serializing_if = "Option::is_none")]
172 pub citation_metadata: Option<CitationMetadata>,
173}
174
175#[derive(Debug, Serialize, Deserialize)]
176#[serde(rename_all = "camelCase")]
177pub struct Content {
178 #[serde(default)]
179 pub parts: Vec<Part>,
180 pub role: Role,
181}
182
183#[derive(Debug, Serialize, Deserialize)]
184#[serde(rename_all = "camelCase")]
185pub struct SystemInstruction {
186 pub parts: Vec<Part>,
187}
188
189#[derive(Debug, PartialEq, Deserialize, Serialize)]
190#[serde(rename_all = "camelCase")]
191pub enum Role {
192 User,
193 Model,
194}
195
196#[derive(Debug, Serialize, Deserialize)]
197#[serde(untagged)]
198pub enum Part {
199 TextPart(TextPart),
200 InlineDataPart(InlineDataPart),
201 FunctionCallPart(FunctionCallPart),
202 FunctionResponsePart(FunctionResponsePart),
203 ThoughtPart(ThoughtPart),
204}
205
206#[derive(Debug, Serialize, Deserialize)]
207#[serde(rename_all = "camelCase")]
208pub struct TextPart {
209 pub text: String,
210}
211
212#[derive(Debug, Serialize, Deserialize)]
213#[serde(rename_all = "camelCase")]
214pub struct InlineDataPart {
215 pub inline_data: GenerativeContentBlob,
216}
217
218#[derive(Debug, Serialize, Deserialize)]
219#[serde(rename_all = "camelCase")]
220pub struct GenerativeContentBlob {
221 pub mime_type: String,
222 pub data: String,
223}
224
225#[derive(Debug, Serialize, Deserialize)]
226#[serde(rename_all = "camelCase")]
227pub struct FunctionCallPart {
228 pub function_call: FunctionCall,
229}
230
231#[derive(Debug, Serialize, Deserialize)]
232#[serde(rename_all = "camelCase")]
233pub struct FunctionResponsePart {
234 pub function_response: FunctionResponse,
235}
236
237#[derive(Debug, Serialize, Deserialize)]
238#[serde(rename_all = "camelCase")]
239pub struct ThoughtPart {
240 pub thought: bool,
241 pub thought_signature: String,
242}
243
244#[derive(Debug, Serialize, Deserialize)]
245#[serde(rename_all = "camelCase")]
246pub struct CitationSource {
247 #[serde(skip_serializing_if = "Option::is_none")]
248 pub start_index: Option<usize>,
249 #[serde(skip_serializing_if = "Option::is_none")]
250 pub end_index: Option<usize>,
251 #[serde(skip_serializing_if = "Option::is_none")]
252 pub uri: Option<String>,
253 #[serde(skip_serializing_if = "Option::is_none")]
254 pub license: Option<String>,
255}
256
257#[derive(Debug, Serialize, Deserialize)]
258#[serde(rename_all = "camelCase")]
259pub struct CitationMetadata {
260 pub citation_sources: Vec<CitationSource>,
261}
262
263#[derive(Debug, Serialize, Deserialize)]
264#[serde(rename_all = "camelCase")]
265pub struct PromptFeedback {
266 #[serde(skip_serializing_if = "Option::is_none")]
267 pub block_reason: Option<String>,
268 pub safety_ratings: Vec<SafetyRating>,
269 #[serde(skip_serializing_if = "Option::is_none")]
270 pub block_reason_message: Option<String>,
271}
272
273#[derive(Debug, Serialize, Deserialize, Default)]
274#[serde(rename_all = "camelCase")]
275pub struct UsageMetadata {
276 #[serde(skip_serializing_if = "Option::is_none")]
277 pub prompt_token_count: Option<u64>,
278 #[serde(skip_serializing_if = "Option::is_none")]
279 pub cached_content_token_count: Option<u64>,
280 #[serde(skip_serializing_if = "Option::is_none")]
281 pub candidates_token_count: Option<u64>,
282 #[serde(skip_serializing_if = "Option::is_none")]
283 pub tool_use_prompt_token_count: Option<u64>,
284 #[serde(skip_serializing_if = "Option::is_none")]
285 pub thoughts_token_count: Option<u64>,
286 #[serde(skip_serializing_if = "Option::is_none")]
287 pub total_token_count: Option<u64>,
288}
289
290#[derive(Debug, Serialize, Deserialize)]
291#[serde(rename_all = "camelCase")]
292pub struct ThinkingConfig {
293 pub thinking_budget: u32,
294}
295
296#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
297#[derive(Copy, Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
298pub enum GoogleModelMode {
299 #[default]
300 Default,
301 Thinking {
302 budget_tokens: Option<u32>,
303 },
304}
305
306#[derive(Debug, Deserialize, Serialize)]
307#[serde(rename_all = "camelCase")]
308pub struct GenerationConfig {
309 #[serde(skip_serializing_if = "Option::is_none")]
310 pub candidate_count: Option<usize>,
311 #[serde(skip_serializing_if = "Option::is_none")]
312 pub stop_sequences: Option<Vec<String>>,
313 #[serde(skip_serializing_if = "Option::is_none")]
314 pub max_output_tokens: Option<usize>,
315 #[serde(skip_serializing_if = "Option::is_none")]
316 pub temperature: Option<f64>,
317 #[serde(skip_serializing_if = "Option::is_none")]
318 pub top_p: Option<f64>,
319 #[serde(skip_serializing_if = "Option::is_none")]
320 pub top_k: Option<usize>,
321 #[serde(skip_serializing_if = "Option::is_none")]
322 pub thinking_config: Option<ThinkingConfig>,
323}
324
325#[derive(Debug, Serialize, Deserialize)]
326#[serde(rename_all = "camelCase")]
327pub struct SafetySetting {
328 pub category: HarmCategory,
329 pub threshold: HarmBlockThreshold,
330}
331
332#[derive(Debug, Serialize, Deserialize)]
333pub enum HarmCategory {
334 #[serde(rename = "HARM_CATEGORY_UNSPECIFIED")]
335 Unspecified,
336 #[serde(rename = "HARM_CATEGORY_DEROGATORY")]
337 Derogatory,
338 #[serde(rename = "HARM_CATEGORY_TOXICITY")]
339 Toxicity,
340 #[serde(rename = "HARM_CATEGORY_VIOLENCE")]
341 Violence,
342 #[serde(rename = "HARM_CATEGORY_SEXUAL")]
343 Sexual,
344 #[serde(rename = "HARM_CATEGORY_MEDICAL")]
345 Medical,
346 #[serde(rename = "HARM_CATEGORY_DANGEROUS")]
347 Dangerous,
348 #[serde(rename = "HARM_CATEGORY_HARASSMENT")]
349 Harassment,
350 #[serde(rename = "HARM_CATEGORY_HATE_SPEECH")]
351 HateSpeech,
352 #[serde(rename = "HARM_CATEGORY_SEXUALLY_EXPLICIT")]
353 SexuallyExplicit,
354 #[serde(rename = "HARM_CATEGORY_DANGEROUS_CONTENT")]
355 DangerousContent,
356}
357
358#[derive(Debug, Serialize, Deserialize)]
359#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
360pub enum HarmBlockThreshold {
361 #[serde(rename = "HARM_BLOCK_THRESHOLD_UNSPECIFIED")]
362 Unspecified,
363 BlockLowAndAbove,
364 BlockMediumAndAbove,
365 BlockOnlyHigh,
366 BlockNone,
367}
368
369#[derive(Debug, Serialize, Deserialize)]
370#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
371pub enum HarmProbability {
372 #[serde(rename = "HARM_PROBABILITY_UNSPECIFIED")]
373 Unspecified,
374 Negligible,
375 Low,
376 Medium,
377 High,
378}
379
380#[derive(Debug, Serialize, Deserialize)]
381#[serde(rename_all = "camelCase")]
382pub struct SafetyRating {
383 pub category: HarmCategory,
384 pub probability: HarmProbability,
385}
386
387#[derive(Debug, Serialize, Deserialize)]
388#[serde(rename_all = "camelCase")]
389pub struct CountTokensRequest {
390 pub generate_content_request: GenerateContentRequest,
391}
392
393#[derive(Debug, Serialize, Deserialize)]
394#[serde(rename_all = "camelCase")]
395pub struct CountTokensResponse {
396 pub total_tokens: u64,
397}
398
399#[derive(Debug, Serialize, Deserialize)]
400pub struct FunctionCall {
401 pub name: String,
402 pub args: serde_json::Value,
403}
404
405#[derive(Debug, Serialize, Deserialize)]
406pub struct FunctionResponse {
407 pub name: String,
408 pub response: serde_json::Value,
409}
410
411#[derive(Debug, Serialize, Deserialize)]
412#[serde(rename_all = "camelCase")]
413pub struct Tool {
414 pub function_declarations: Vec<FunctionDeclaration>,
415}
416
417#[derive(Debug, Serialize, Deserialize)]
418#[serde(rename_all = "camelCase")]
419pub struct ToolConfig {
420 pub function_calling_config: FunctionCallingConfig,
421}
422
423#[derive(Debug, Serialize, Deserialize)]
424#[serde(rename_all = "camelCase")]
425pub struct FunctionCallingConfig {
426 pub mode: FunctionCallingMode,
427 #[serde(skip_serializing_if = "Option::is_none")]
428 pub allowed_function_names: Option<Vec<String>>,
429}
430
431#[derive(Debug, Serialize, Deserialize)]
432#[serde(rename_all = "lowercase")]
433pub enum FunctionCallingMode {
434 Auto,
435 Any,
436 None,
437}
438
439#[derive(Debug, Serialize, Deserialize)]
440pub struct FunctionDeclaration {
441 pub name: String,
442 pub description: String,
443 pub parameters: serde_json::Value,
444}
445
446#[derive(Debug, Default)]
447pub struct ModelName {
448 pub model_id: String,
449}
450
451impl ModelName {
452 pub fn is_empty(&self) -> bool {
453 self.model_id.is_empty()
454 }
455}
456
457const MODEL_NAME_PREFIX: &str = "models/";
458
459impl Serialize for ModelName {
460 fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
461 where
462 S: Serializer,
463 {
464 serializer.serialize_str(&format!("{MODEL_NAME_PREFIX}{}", &self.model_id))
465 }
466}
467
468impl<'de> Deserialize<'de> for ModelName {
469 fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
470 where
471 D: Deserializer<'de>,
472 {
473 let string = String::deserialize(deserializer)?;
474 if let Some(id) = string.strip_prefix(MODEL_NAME_PREFIX) {
475 Ok(Self {
476 model_id: id.to_string(),
477 })
478 } else {
479 return Err(serde::de::Error::custom(format!(
480 "Expected model name to begin with {}, got: {}",
481 MODEL_NAME_PREFIX, string
482 )));
483 }
484 }
485}
486
487#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
488#[derive(Clone, Default, Debug, Deserialize, Serialize, PartialEq, Eq, strum::EnumIter)]
489pub enum Model {
490 #[serde(rename = "gemini-1.5-pro")]
491 Gemini15Pro,
492 #[serde(rename = "gemini-1.5-flash-8b")]
493 Gemini15Flash8b,
494 #[serde(rename = "gemini-1.5-flash")]
495 Gemini15Flash,
496 #[serde(
497 rename = "gemini-2.0-flash-lite",
498 alias = "gemini-2.0-flash-lite-preview"
499 )]
500 Gemini20FlashLite,
501 #[serde(rename = "gemini-2.0-flash")]
502 Gemini20Flash,
503 #[serde(
504 rename = "gemini-2.5-flash-lite-preview",
505 alias = "gemini-2.5-flash-lite-preview-06-17"
506 )]
507 Gemini25FlashLitePreview,
508 #[serde(
509 rename = "gemini-2.5-flash",
510 alias = "gemini-2.0-flash-thinking-exp",
511 alias = "gemini-2.5-flash-preview-04-17",
512 alias = "gemini-2.5-flash-preview-05-20",
513 alias = "gemini-2.5-flash-preview-latest"
514 )]
515 #[default]
516 Gemini25Flash,
517 #[serde(
518 rename = "gemini-2.5-pro",
519 alias = "gemini-2.0-pro-exp",
520 alias = "gemini-2.5-pro-preview-latest",
521 alias = "gemini-2.5-pro-exp-03-25",
522 alias = "gemini-2.5-pro-preview-03-25",
523 alias = "gemini-2.5-pro-preview-05-06",
524 alias = "gemini-2.5-pro-preview-06-05"
525 )]
526 Gemini25Pro,
527 #[serde(rename = "custom")]
528 Custom {
529 name: String,
530 /// The name displayed in the UI, such as in the assistant panel model dropdown menu.
531 display_name: Option<String>,
532 max_tokens: u64,
533 #[serde(default)]
534 mode: GoogleModelMode,
535 },
536}
537
538impl Model {
539 pub fn default_fast() -> Self {
540 Self::Gemini20FlashLite
541 }
542
543 pub fn id(&self) -> &str {
544 match self {
545 Self::Gemini15Pro => "gemini-1.5-pro",
546 Self::Gemini15Flash8b => "gemini-1.5-flash-8b",
547 Self::Gemini15Flash => "gemini-1.5-flash",
548 Self::Gemini20FlashLite => "gemini-2.0-flash-lite",
549 Self::Gemini20Flash => "gemini-2.0-flash",
550 Self::Gemini25FlashLitePreview => "gemini-2.5-flash-lite-preview",
551 Self::Gemini25Flash => "gemini-2.5-flash",
552 Self::Gemini25Pro => "gemini-2.5-pro",
553 Self::Custom { name, .. } => name,
554 }
555 }
556 pub fn request_id(&self) -> &str {
557 match self {
558 Self::Gemini15Pro => "gemini-1.5-pro",
559 Self::Gemini15Flash8b => "gemini-1.5-flash-8b",
560 Self::Gemini15Flash => "gemini-1.5-flash",
561 Self::Gemini20FlashLite => "gemini-2.0-flash-lite",
562 Self::Gemini20Flash => "gemini-2.0-flash",
563 Self::Gemini25FlashLitePreview => "gemini-2.5-flash-lite-preview-06-17",
564 Self::Gemini25Flash => "gemini-2.5-flash",
565 Self::Gemini25Pro => "gemini-2.5-pro",
566 Self::Custom { name, .. } => name,
567 }
568 }
569
570 pub fn display_name(&self) -> &str {
571 match self {
572 Self::Gemini15Pro => "Gemini 1.5 Pro",
573 Self::Gemini15Flash8b => "Gemini 1.5 Flash-8b",
574 Self::Gemini15Flash => "Gemini 1.5 Flash",
575 Self::Gemini20FlashLite => "Gemini 2.0 Flash-Lite",
576 Self::Gemini20Flash => "Gemini 2.0 Flash",
577 Self::Gemini25FlashLitePreview => "Gemini 2.5 Flash-Lite Preview",
578 Self::Gemini25Flash => "Gemini 2.5 Flash",
579 Self::Gemini25Pro => "Gemini 2.5 Pro",
580 Self::Custom {
581 name, display_name, ..
582 } => display_name.as_ref().unwrap_or(name),
583 }
584 }
585
586 pub fn max_token_count(&self) -> u64 {
587 match self {
588 Self::Gemini15Pro => 2_097_152,
589 Self::Gemini15Flash8b => 1_048_576,
590 Self::Gemini15Flash => 1_048_576,
591 Self::Gemini20FlashLite => 1_048_576,
592 Self::Gemini20Flash => 1_048_576,
593 Self::Gemini25FlashLitePreview => 1_000_000,
594 Self::Gemini25Flash => 1_048_576,
595 Self::Gemini25Pro => 1_048_576,
596 Self::Custom { max_tokens, .. } => *max_tokens,
597 }
598 }
599
600 pub fn max_output_tokens(&self) -> Option<u64> {
601 match self {
602 Model::Gemini15Pro => Some(8_192),
603 Model::Gemini15Flash8b => Some(8_192),
604 Model::Gemini15Flash => Some(8_192),
605 Model::Gemini20FlashLite => Some(8_192),
606 Model::Gemini20Flash => Some(8_192),
607 Model::Gemini25FlashLitePreview => Some(64_000),
608 Model::Gemini25Flash => Some(65_536),
609 Model::Gemini25Pro => Some(65_536),
610 Model::Custom { .. } => None,
611 }
612 }
613
614 pub fn supports_tools(&self) -> bool {
615 true
616 }
617
618 pub fn supports_images(&self) -> bool {
619 true
620 }
621
622 pub fn mode(&self) -> GoogleModelMode {
623 match self {
624 Self::Gemini15Pro
625 | Self::Gemini15Flash8b
626 | Self::Gemini15Flash
627 | Self::Gemini20FlashLite
628 | Self::Gemini20Flash => GoogleModelMode::Default,
629 Self::Gemini25FlashLitePreview | Self::Gemini25Flash | Self::Gemini25Pro => {
630 GoogleModelMode::Thinking {
631 // By default these models are set to "auto", so we preserve that behavior
632 // but indicate they are capable of thinking mode
633 budget_tokens: None,
634 }
635 }
636 Self::Custom { mode, .. } => *mode,
637 }
638 }
639}
640
641impl std::fmt::Display for Model {
642 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
643 write!(f, "{}", self.id())
644 }
645}