1use std::mem;
2
3use anyhow::{Result, anyhow, bail};
4use futures::{AsyncBufReadExt, AsyncReadExt, StreamExt, io::BufReader, stream::BoxStream};
5use http_client::{AsyncBody, HttpClient, Method, Request as HttpRequest};
6use serde::{Deserialize, Deserializer, Serialize, Serializer};
7
8pub const API_URL: &str = "https://generativelanguage.googleapis.com";
9
10pub async fn stream_generate_content(
11 client: &dyn HttpClient,
12 api_url: &str,
13 api_key: &str,
14 mut request: GenerateContentRequest,
15) -> Result<BoxStream<'static, Result<GenerateContentResponse>>> {
16 let api_key = api_key.trim();
17 validate_generate_content_request(&request)?;
18
19 // The `model` field is emptied as it is provided as a path parameter.
20 let model_id = mem::take(&mut request.model.model_id);
21
22 let uri =
23 format!("{api_url}/v1beta/models/{model_id}:streamGenerateContent?alt=sse&key={api_key}",);
24
25 let request_builder = HttpRequest::builder()
26 .method(Method::POST)
27 .uri(uri)
28 .header("Content-Type", "application/json");
29
30 let request = request_builder.body(AsyncBody::from(serde_json::to_string(&request)?))?;
31 let mut response = client.send(request).await?;
32 if response.status().is_success() {
33 let reader = BufReader::new(response.into_body());
34 Ok(reader
35 .lines()
36 .filter_map(|line| async move {
37 match line {
38 Ok(line) => {
39 if let Some(line) = line.strip_prefix("data: ") {
40 match serde_json::from_str(line) {
41 Ok(response) => Some(Ok(response)),
42 Err(error) => Some(Err(anyhow!(format!(
43 "Error parsing JSON: {error:?}\n{line:?}"
44 )))),
45 }
46 } else {
47 None
48 }
49 }
50 Err(error) => Some(Err(anyhow!(error))),
51 }
52 })
53 .boxed())
54 } else {
55 let mut text = String::new();
56 response.body_mut().read_to_string(&mut text).await?;
57 Err(anyhow!(
58 "error during streamGenerateContent, status code: {:?}, body: {}",
59 response.status(),
60 text
61 ))
62 }
63}
64
65pub async fn count_tokens(
66 client: &dyn HttpClient,
67 api_url: &str,
68 api_key: &str,
69 request: CountTokensRequest,
70) -> Result<CountTokensResponse> {
71 validate_generate_content_request(&request.generate_content_request)?;
72
73 let uri = format!(
74 "{api_url}/v1beta/models/{model_id}:countTokens?key={api_key}",
75 model_id = &request.generate_content_request.model.model_id,
76 );
77
78 let request = serde_json::to_string(&request)?;
79 let request_builder = HttpRequest::builder()
80 .method(Method::POST)
81 .uri(&uri)
82 .header("Content-Type", "application/json");
83 let http_request = request_builder.body(AsyncBody::from(request))?;
84
85 let mut response = client.send(http_request).await?;
86 let mut text = String::new();
87 response.body_mut().read_to_string(&mut text).await?;
88 anyhow::ensure!(
89 response.status().is_success(),
90 "error during countTokens, status code: {:?}, body: {}",
91 response.status(),
92 text
93 );
94 Ok(serde_json::from_str::<CountTokensResponse>(&text)?)
95}
96
97pub fn validate_generate_content_request(request: &GenerateContentRequest) -> Result<()> {
98 if request.model.is_empty() {
99 bail!("Model must be specified");
100 }
101
102 if request.contents.is_empty() {
103 bail!("Request must contain at least one content item");
104 }
105
106 if let Some(user_content) = request
107 .contents
108 .iter()
109 .find(|content| content.role == Role::User)
110 && user_content.parts.is_empty()
111 {
112 bail!("User content must contain at least one part");
113 }
114
115 Ok(())
116}
117
118#[derive(Debug, Serialize, Deserialize)]
119pub enum Task {
120 #[serde(rename = "generateContent")]
121 GenerateContent,
122 #[serde(rename = "streamGenerateContent")]
123 StreamGenerateContent,
124 #[serde(rename = "countTokens")]
125 CountTokens,
126 #[serde(rename = "embedContent")]
127 EmbedContent,
128 #[serde(rename = "batchEmbedContents")]
129 BatchEmbedContents,
130}
131
132#[derive(Debug, Serialize, Deserialize)]
133#[serde(rename_all = "camelCase")]
134pub struct GenerateContentRequest {
135 #[serde(default, skip_serializing_if = "ModelName::is_empty")]
136 pub model: ModelName,
137 pub contents: Vec<Content>,
138 #[serde(skip_serializing_if = "Option::is_none")]
139 pub system_instruction: Option<SystemInstruction>,
140 #[serde(skip_serializing_if = "Option::is_none")]
141 pub generation_config: Option<GenerationConfig>,
142 #[serde(skip_serializing_if = "Option::is_none")]
143 pub safety_settings: Option<Vec<SafetySetting>>,
144 #[serde(skip_serializing_if = "Option::is_none")]
145 pub tools: Option<Vec<Tool>>,
146 #[serde(skip_serializing_if = "Option::is_none")]
147 pub tool_config: Option<ToolConfig>,
148}
149
150#[derive(Debug, Serialize, Deserialize)]
151#[serde(rename_all = "camelCase")]
152pub struct GenerateContentResponse {
153 #[serde(skip_serializing_if = "Option::is_none")]
154 pub candidates: Option<Vec<GenerateContentCandidate>>,
155 #[serde(skip_serializing_if = "Option::is_none")]
156 pub prompt_feedback: Option<PromptFeedback>,
157 #[serde(skip_serializing_if = "Option::is_none")]
158 pub usage_metadata: Option<UsageMetadata>,
159}
160
161#[derive(Debug, Serialize, Deserialize)]
162#[serde(rename_all = "camelCase")]
163pub struct GenerateContentCandidate {
164 #[serde(skip_serializing_if = "Option::is_none")]
165 pub index: Option<usize>,
166 pub content: Content,
167 #[serde(skip_serializing_if = "Option::is_none")]
168 pub finish_reason: Option<String>,
169 #[serde(skip_serializing_if = "Option::is_none")]
170 pub finish_message: Option<String>,
171 #[serde(skip_serializing_if = "Option::is_none")]
172 pub safety_ratings: Option<Vec<SafetyRating>>,
173 #[serde(skip_serializing_if = "Option::is_none")]
174 pub citation_metadata: Option<CitationMetadata>,
175}
176
177#[derive(Debug, Serialize, Deserialize)]
178#[serde(rename_all = "camelCase")]
179pub struct Content {
180 #[serde(default)]
181 pub parts: Vec<Part>,
182 pub role: Role,
183}
184
185#[derive(Debug, Serialize, Deserialize)]
186#[serde(rename_all = "camelCase")]
187pub struct SystemInstruction {
188 pub parts: Vec<Part>,
189}
190
191#[derive(Debug, PartialEq, Deserialize, Serialize)]
192#[serde(rename_all = "camelCase")]
193pub enum Role {
194 User,
195 Model,
196}
197
198#[derive(Debug, Serialize, Deserialize)]
199#[serde(untagged)]
200pub enum Part {
201 TextPart(TextPart),
202 InlineDataPart(InlineDataPart),
203 FunctionCallPart(FunctionCallPart),
204 FunctionResponsePart(FunctionResponsePart),
205 ThoughtPart(ThoughtPart),
206}
207
208#[derive(Debug, Serialize, Deserialize)]
209#[serde(rename_all = "camelCase")]
210pub struct TextPart {
211 pub text: String,
212}
213
214#[derive(Debug, Serialize, Deserialize)]
215#[serde(rename_all = "camelCase")]
216pub struct InlineDataPart {
217 pub inline_data: GenerativeContentBlob,
218}
219
220#[derive(Debug, Serialize, Deserialize)]
221#[serde(rename_all = "camelCase")]
222pub struct GenerativeContentBlob {
223 pub mime_type: String,
224 pub data: String,
225}
226
227#[derive(Debug, Serialize, Deserialize)]
228#[serde(rename_all = "camelCase")]
229pub struct FunctionCallPart {
230 pub function_call: FunctionCall,
231}
232
233#[derive(Debug, Serialize, Deserialize)]
234#[serde(rename_all = "camelCase")]
235pub struct FunctionResponsePart {
236 pub function_response: FunctionResponse,
237}
238
239#[derive(Debug, Serialize, Deserialize)]
240#[serde(rename_all = "camelCase")]
241pub struct ThoughtPart {
242 pub thought: bool,
243 pub thought_signature: String,
244}
245
246#[derive(Debug, Serialize, Deserialize)]
247#[serde(rename_all = "camelCase")]
248pub struct CitationSource {
249 #[serde(skip_serializing_if = "Option::is_none")]
250 pub start_index: Option<usize>,
251 #[serde(skip_serializing_if = "Option::is_none")]
252 pub end_index: Option<usize>,
253 #[serde(skip_serializing_if = "Option::is_none")]
254 pub uri: Option<String>,
255 #[serde(skip_serializing_if = "Option::is_none")]
256 pub license: Option<String>,
257}
258
259#[derive(Debug, Serialize, Deserialize)]
260#[serde(rename_all = "camelCase")]
261pub struct CitationMetadata {
262 pub citation_sources: Vec<CitationSource>,
263}
264
265#[derive(Debug, Serialize, Deserialize)]
266#[serde(rename_all = "camelCase")]
267pub struct PromptFeedback {
268 #[serde(skip_serializing_if = "Option::is_none")]
269 pub block_reason: Option<String>,
270 pub safety_ratings: Option<Vec<SafetyRating>>,
271 #[serde(skip_serializing_if = "Option::is_none")]
272 pub block_reason_message: Option<String>,
273}
274
275#[derive(Debug, Serialize, Deserialize, Default)]
276#[serde(rename_all = "camelCase")]
277pub struct UsageMetadata {
278 #[serde(skip_serializing_if = "Option::is_none")]
279 pub prompt_token_count: Option<u64>,
280 #[serde(skip_serializing_if = "Option::is_none")]
281 pub cached_content_token_count: Option<u64>,
282 #[serde(skip_serializing_if = "Option::is_none")]
283 pub candidates_token_count: Option<u64>,
284 #[serde(skip_serializing_if = "Option::is_none")]
285 pub tool_use_prompt_token_count: Option<u64>,
286 #[serde(skip_serializing_if = "Option::is_none")]
287 pub thoughts_token_count: Option<u64>,
288 #[serde(skip_serializing_if = "Option::is_none")]
289 pub total_token_count: Option<u64>,
290}
291
292#[derive(Debug, Serialize, Deserialize)]
293#[serde(rename_all = "camelCase")]
294pub struct ThinkingConfig {
295 pub thinking_budget: u32,
296}
297
298#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
299#[derive(Copy, Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
300pub enum GoogleModelMode {
301 #[default]
302 Default,
303 Thinking {
304 budget_tokens: Option<u32>,
305 },
306}
307
308#[derive(Debug, Deserialize, Serialize)]
309#[serde(rename_all = "camelCase")]
310pub struct GenerationConfig {
311 #[serde(skip_serializing_if = "Option::is_none")]
312 pub candidate_count: Option<usize>,
313 #[serde(skip_serializing_if = "Option::is_none")]
314 pub stop_sequences: Option<Vec<String>>,
315 #[serde(skip_serializing_if = "Option::is_none")]
316 pub max_output_tokens: Option<usize>,
317 #[serde(skip_serializing_if = "Option::is_none")]
318 pub temperature: Option<f64>,
319 #[serde(skip_serializing_if = "Option::is_none")]
320 pub top_p: Option<f64>,
321 #[serde(skip_serializing_if = "Option::is_none")]
322 pub top_k: Option<usize>,
323 #[serde(skip_serializing_if = "Option::is_none")]
324 pub thinking_config: Option<ThinkingConfig>,
325}
326
327#[derive(Debug, Serialize, Deserialize)]
328#[serde(rename_all = "camelCase")]
329pub struct SafetySetting {
330 pub category: HarmCategory,
331 pub threshold: HarmBlockThreshold,
332}
333
334#[derive(Debug, Serialize, Deserialize)]
335pub enum HarmCategory {
336 #[serde(rename = "HARM_CATEGORY_UNSPECIFIED")]
337 Unspecified,
338 #[serde(rename = "HARM_CATEGORY_DEROGATORY")]
339 Derogatory,
340 #[serde(rename = "HARM_CATEGORY_TOXICITY")]
341 Toxicity,
342 #[serde(rename = "HARM_CATEGORY_VIOLENCE")]
343 Violence,
344 #[serde(rename = "HARM_CATEGORY_SEXUAL")]
345 Sexual,
346 #[serde(rename = "HARM_CATEGORY_MEDICAL")]
347 Medical,
348 #[serde(rename = "HARM_CATEGORY_DANGEROUS")]
349 Dangerous,
350 #[serde(rename = "HARM_CATEGORY_HARASSMENT")]
351 Harassment,
352 #[serde(rename = "HARM_CATEGORY_HATE_SPEECH")]
353 HateSpeech,
354 #[serde(rename = "HARM_CATEGORY_SEXUALLY_EXPLICIT")]
355 SexuallyExplicit,
356 #[serde(rename = "HARM_CATEGORY_DANGEROUS_CONTENT")]
357 DangerousContent,
358}
359
360#[derive(Debug, Serialize, Deserialize)]
361#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
362pub enum HarmBlockThreshold {
363 #[serde(rename = "HARM_BLOCK_THRESHOLD_UNSPECIFIED")]
364 Unspecified,
365 BlockLowAndAbove,
366 BlockMediumAndAbove,
367 BlockOnlyHigh,
368 BlockNone,
369}
370
371#[derive(Debug, Serialize, Deserialize)]
372#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
373pub enum HarmProbability {
374 #[serde(rename = "HARM_PROBABILITY_UNSPECIFIED")]
375 Unspecified,
376 Negligible,
377 Low,
378 Medium,
379 High,
380}
381
382#[derive(Debug, Serialize, Deserialize)]
383#[serde(rename_all = "camelCase")]
384pub struct SafetyRating {
385 pub category: HarmCategory,
386 pub probability: HarmProbability,
387}
388
389#[derive(Debug, Serialize, Deserialize)]
390#[serde(rename_all = "camelCase")]
391pub struct CountTokensRequest {
392 pub generate_content_request: GenerateContentRequest,
393}
394
395#[derive(Debug, Serialize, Deserialize)]
396#[serde(rename_all = "camelCase")]
397pub struct CountTokensResponse {
398 pub total_tokens: u64,
399}
400
401#[derive(Debug, Serialize, Deserialize)]
402pub struct FunctionCall {
403 pub name: String,
404 pub args: serde_json::Value,
405}
406
407#[derive(Debug, Serialize, Deserialize)]
408pub struct FunctionResponse {
409 pub name: String,
410 pub response: serde_json::Value,
411}
412
413#[derive(Debug, Serialize, Deserialize)]
414#[serde(rename_all = "camelCase")]
415pub struct Tool {
416 pub function_declarations: Vec<FunctionDeclaration>,
417}
418
419#[derive(Debug, Serialize, Deserialize)]
420#[serde(rename_all = "camelCase")]
421pub struct ToolConfig {
422 pub function_calling_config: FunctionCallingConfig,
423}
424
425#[derive(Debug, Serialize, Deserialize)]
426#[serde(rename_all = "camelCase")]
427pub struct FunctionCallingConfig {
428 pub mode: FunctionCallingMode,
429 #[serde(skip_serializing_if = "Option::is_none")]
430 pub allowed_function_names: Option<Vec<String>>,
431}
432
433#[derive(Debug, Serialize, Deserialize)]
434#[serde(rename_all = "lowercase")]
435pub enum FunctionCallingMode {
436 Auto,
437 Any,
438 None,
439}
440
441#[derive(Debug, Serialize, Deserialize)]
442pub struct FunctionDeclaration {
443 pub name: String,
444 pub description: String,
445 pub parameters: serde_json::Value,
446}
447
448#[derive(Debug, Default)]
449pub struct ModelName {
450 pub model_id: String,
451}
452
453impl ModelName {
454 pub fn is_empty(&self) -> bool {
455 self.model_id.is_empty()
456 }
457}
458
459const MODEL_NAME_PREFIX: &str = "models/";
460
461impl Serialize for ModelName {
462 fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
463 where
464 S: Serializer,
465 {
466 serializer.serialize_str(&format!("{MODEL_NAME_PREFIX}{}", &self.model_id))
467 }
468}
469
470impl<'de> Deserialize<'de> for ModelName {
471 fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
472 where
473 D: Deserializer<'de>,
474 {
475 let string = String::deserialize(deserializer)?;
476 if let Some(id) = string.strip_prefix(MODEL_NAME_PREFIX) {
477 Ok(Self {
478 model_id: id.to_string(),
479 })
480 } else {
481 Err(serde::de::Error::custom(format!(
482 "Expected model name to begin with {}, got: {}",
483 MODEL_NAME_PREFIX, string
484 )))
485 }
486 }
487}
488
489#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
490#[derive(Clone, Default, Debug, Deserialize, Serialize, PartialEq, Eq, strum::EnumIter)]
491pub enum Model {
492 #[serde(rename = "gemini-1.5-pro")]
493 Gemini15Pro,
494 #[serde(rename = "gemini-1.5-flash-8b")]
495 Gemini15Flash8b,
496 #[serde(rename = "gemini-1.5-flash")]
497 Gemini15Flash,
498 #[serde(
499 rename = "gemini-2.0-flash-lite",
500 alias = "gemini-2.0-flash-lite-preview"
501 )]
502 Gemini20FlashLite,
503 #[serde(rename = "gemini-2.0-flash")]
504 Gemini20Flash,
505 #[serde(
506 rename = "gemini-2.5-flash-lite-preview",
507 alias = "gemini-2.5-flash-lite-preview-06-17"
508 )]
509 Gemini25FlashLitePreview,
510 #[serde(
511 rename = "gemini-2.5-flash",
512 alias = "gemini-2.0-flash-thinking-exp",
513 alias = "gemini-2.5-flash-preview-04-17",
514 alias = "gemini-2.5-flash-preview-05-20",
515 alias = "gemini-2.5-flash-preview-latest"
516 )]
517 #[default]
518 Gemini25Flash,
519 #[serde(
520 rename = "gemini-2.5-pro",
521 alias = "gemini-2.0-pro-exp",
522 alias = "gemini-2.5-pro-preview-latest",
523 alias = "gemini-2.5-pro-exp-03-25",
524 alias = "gemini-2.5-pro-preview-03-25",
525 alias = "gemini-2.5-pro-preview-05-06",
526 alias = "gemini-2.5-pro-preview-06-05"
527 )]
528 Gemini25Pro,
529 #[serde(rename = "custom")]
530 Custom {
531 name: String,
532 /// The name displayed in the UI, such as in the assistant panel model dropdown menu.
533 display_name: Option<String>,
534 max_tokens: u64,
535 #[serde(default)]
536 mode: GoogleModelMode,
537 },
538}
539
540impl Model {
541 pub fn default_fast() -> Self {
542 Self::Gemini20FlashLite
543 }
544
545 pub fn id(&self) -> &str {
546 match self {
547 Self::Gemini15Pro => "gemini-1.5-pro",
548 Self::Gemini15Flash8b => "gemini-1.5-flash-8b",
549 Self::Gemini15Flash => "gemini-1.5-flash",
550 Self::Gemini20FlashLite => "gemini-2.0-flash-lite",
551 Self::Gemini20Flash => "gemini-2.0-flash",
552 Self::Gemini25FlashLitePreview => "gemini-2.5-flash-lite-preview",
553 Self::Gemini25Flash => "gemini-2.5-flash",
554 Self::Gemini25Pro => "gemini-2.5-pro",
555 Self::Custom { name, .. } => name,
556 }
557 }
558 pub fn request_id(&self) -> &str {
559 match self {
560 Self::Gemini15Pro => "gemini-1.5-pro",
561 Self::Gemini15Flash8b => "gemini-1.5-flash-8b",
562 Self::Gemini15Flash => "gemini-1.5-flash",
563 Self::Gemini20FlashLite => "gemini-2.0-flash-lite",
564 Self::Gemini20Flash => "gemini-2.0-flash",
565 Self::Gemini25FlashLitePreview => "gemini-2.5-flash-lite-preview-06-17",
566 Self::Gemini25Flash => "gemini-2.5-flash",
567 Self::Gemini25Pro => "gemini-2.5-pro",
568 Self::Custom { name, .. } => name,
569 }
570 }
571
572 pub fn display_name(&self) -> &str {
573 match self {
574 Self::Gemini15Pro => "Gemini 1.5 Pro",
575 Self::Gemini15Flash8b => "Gemini 1.5 Flash-8b",
576 Self::Gemini15Flash => "Gemini 1.5 Flash",
577 Self::Gemini20FlashLite => "Gemini 2.0 Flash-Lite",
578 Self::Gemini20Flash => "Gemini 2.0 Flash",
579 Self::Gemini25FlashLitePreview => "Gemini 2.5 Flash-Lite Preview",
580 Self::Gemini25Flash => "Gemini 2.5 Flash",
581 Self::Gemini25Pro => "Gemini 2.5 Pro",
582 Self::Custom {
583 name, display_name, ..
584 } => display_name.as_ref().unwrap_or(name),
585 }
586 }
587
588 pub fn max_token_count(&self) -> u64 {
589 match self {
590 Self::Gemini15Pro => 2_097_152,
591 Self::Gemini15Flash8b => 1_048_576,
592 Self::Gemini15Flash => 1_048_576,
593 Self::Gemini20FlashLite => 1_048_576,
594 Self::Gemini20Flash => 1_048_576,
595 Self::Gemini25FlashLitePreview => 1_000_000,
596 Self::Gemini25Flash => 1_048_576,
597 Self::Gemini25Pro => 1_048_576,
598 Self::Custom { max_tokens, .. } => *max_tokens,
599 }
600 }
601
602 pub fn max_output_tokens(&self) -> Option<u64> {
603 match self {
604 Model::Gemini15Pro => Some(8_192),
605 Model::Gemini15Flash8b => Some(8_192),
606 Model::Gemini15Flash => Some(8_192),
607 Model::Gemini20FlashLite => Some(8_192),
608 Model::Gemini20Flash => Some(8_192),
609 Model::Gemini25FlashLitePreview => Some(64_000),
610 Model::Gemini25Flash => Some(65_536),
611 Model::Gemini25Pro => Some(65_536),
612 Model::Custom { .. } => None,
613 }
614 }
615
616 pub fn supports_tools(&self) -> bool {
617 true
618 }
619
620 pub fn supports_images(&self) -> bool {
621 true
622 }
623
624 pub fn mode(&self) -> GoogleModelMode {
625 match self {
626 Self::Gemini15Pro
627 | Self::Gemini15Flash8b
628 | Self::Gemini15Flash
629 | Self::Gemini20FlashLite
630 | Self::Gemini20Flash => GoogleModelMode::Default,
631 Self::Gemini25FlashLitePreview | Self::Gemini25Flash | Self::Gemini25Pro => {
632 GoogleModelMode::Thinking {
633 // By default these models are set to "auto", so we preserve that behavior
634 // but indicate they are capable of thinking mode
635 budget_tokens: None,
636 }
637 }
638 Self::Custom { mode, .. } => *mode,
639 }
640 }
641}
642
643impl std::fmt::Display for Model {
644 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
645 write!(f, "{}", self.id())
646 }
647}