1use std::mem;
2
3use anyhow::{Result, anyhow, bail};
4use futures::{AsyncBufReadExt, AsyncReadExt, StreamExt, io::BufReader, stream::BoxStream};
5use http_client::{AsyncBody, HttpClient, Method, Request as HttpRequest};
6use serde::{Deserialize, Deserializer, Serialize, Serializer};
7
8pub const API_URL: &str = "https://generativelanguage.googleapis.com";
9
10pub async fn stream_generate_content(
11 client: &dyn HttpClient,
12 api_url: &str,
13 api_key: &str,
14 mut request: GenerateContentRequest,
15) -> Result<BoxStream<'static, Result<GenerateContentResponse>>> {
16 validate_generate_content_request(&request)?;
17
18 // The `model` field is emptied as it is provided as a path parameter.
19 let model_id = mem::take(&mut request.model.model_id);
20
21 let uri =
22 format!("{api_url}/v1beta/models/{model_id}:streamGenerateContent?alt=sse&key={api_key}",);
23
24 let request_builder = HttpRequest::builder()
25 .method(Method::POST)
26 .uri(uri)
27 .header("Content-Type", "application/json");
28
29 let request = request_builder.body(AsyncBody::from(serde_json::to_string(&request)?))?;
30 let mut response = client.send(request).await?;
31 if response.status().is_success() {
32 let reader = BufReader::new(response.into_body());
33 Ok(reader
34 .lines()
35 .filter_map(|line| async move {
36 match line {
37 Ok(line) => {
38 if let Some(line) = line.strip_prefix("data: ") {
39 match serde_json::from_str(line) {
40 Ok(response) => Some(Ok(response)),
41 Err(error) => Some(Err(anyhow!(format!(
42 "Error parsing JSON: {error:?}\n{line:?}"
43 )))),
44 }
45 } else {
46 None
47 }
48 }
49 Err(error) => Some(Err(anyhow!(error))),
50 }
51 })
52 .boxed())
53 } else {
54 let mut text = String::new();
55 response.body_mut().read_to_string(&mut text).await?;
56 Err(anyhow!(
57 "error during streamGenerateContent, status code: {:?}, body: {}",
58 response.status(),
59 text
60 ))
61 }
62}
63
64pub async fn count_tokens(
65 client: &dyn HttpClient,
66 api_url: &str,
67 api_key: &str,
68 request: CountTokensRequest,
69) -> Result<CountTokensResponse> {
70 validate_generate_content_request(&request.generate_content_request)?;
71
72 let uri = format!(
73 "{api_url}/v1beta/models/{model_id}:countTokens?key={api_key}",
74 model_id = &request.generate_content_request.model.model_id,
75 );
76
77 let request = serde_json::to_string(&request)?;
78 let request_builder = HttpRequest::builder()
79 .method(Method::POST)
80 .uri(&uri)
81 .header("Content-Type", "application/json");
82 let http_request = request_builder.body(AsyncBody::from(request))?;
83
84 let mut response = client.send(http_request).await?;
85 let mut text = String::new();
86 response.body_mut().read_to_string(&mut text).await?;
87 anyhow::ensure!(
88 response.status().is_success(),
89 "error during countTokens, status code: {:?}, body: {}",
90 response.status(),
91 text
92 );
93 Ok(serde_json::from_str::<CountTokensResponse>(&text)?)
94}
95
96pub fn validate_generate_content_request(request: &GenerateContentRequest) -> Result<()> {
97 if request.model.is_empty() {
98 bail!("Model must be specified");
99 }
100
101 if request.contents.is_empty() {
102 bail!("Request must contain at least one content item");
103 }
104
105 if let Some(user_content) = request
106 .contents
107 .iter()
108 .find(|content| content.role == Role::User)
109 {
110 if user_content.parts.is_empty() {
111 bail!("User content must contain at least one part");
112 }
113 }
114
115 Ok(())
116}
117
118#[derive(Debug, Serialize, Deserialize)]
119pub enum Task {
120 #[serde(rename = "generateContent")]
121 GenerateContent,
122 #[serde(rename = "streamGenerateContent")]
123 StreamGenerateContent,
124 #[serde(rename = "countTokens")]
125 CountTokens,
126 #[serde(rename = "embedContent")]
127 EmbedContent,
128 #[serde(rename = "batchEmbedContents")]
129 BatchEmbedContents,
130}
131
132#[derive(Debug, Serialize, Deserialize)]
133#[serde(rename_all = "camelCase")]
134pub struct GenerateContentRequest {
135 #[serde(default, skip_serializing_if = "ModelName::is_empty")]
136 pub model: ModelName,
137 pub contents: Vec<Content>,
138 #[serde(skip_serializing_if = "Option::is_none")]
139 pub system_instruction: Option<SystemInstruction>,
140 #[serde(skip_serializing_if = "Option::is_none")]
141 pub generation_config: Option<GenerationConfig>,
142 #[serde(skip_serializing_if = "Option::is_none")]
143 pub safety_settings: Option<Vec<SafetySetting>>,
144 #[serde(skip_serializing_if = "Option::is_none")]
145 pub tools: Option<Vec<Tool>>,
146 #[serde(skip_serializing_if = "Option::is_none")]
147 pub tool_config: Option<ToolConfig>,
148}
149
150#[derive(Debug, Serialize, Deserialize)]
151#[serde(rename_all = "camelCase")]
152pub struct GenerateContentResponse {
153 #[serde(skip_serializing_if = "Option::is_none")]
154 pub candidates: Option<Vec<GenerateContentCandidate>>,
155 #[serde(skip_serializing_if = "Option::is_none")]
156 pub prompt_feedback: Option<PromptFeedback>,
157 #[serde(skip_serializing_if = "Option::is_none")]
158 pub usage_metadata: Option<UsageMetadata>,
159}
160
161#[derive(Debug, Serialize, Deserialize)]
162#[serde(rename_all = "camelCase")]
163pub struct GenerateContentCandidate {
164 #[serde(skip_serializing_if = "Option::is_none")]
165 pub index: Option<usize>,
166 pub content: Content,
167 #[serde(skip_serializing_if = "Option::is_none")]
168 pub finish_reason: Option<String>,
169 #[serde(skip_serializing_if = "Option::is_none")]
170 pub finish_message: Option<String>,
171 #[serde(skip_serializing_if = "Option::is_none")]
172 pub safety_ratings: Option<Vec<SafetyRating>>,
173 #[serde(skip_serializing_if = "Option::is_none")]
174 pub citation_metadata: Option<CitationMetadata>,
175}
176
177#[derive(Debug, Serialize, Deserialize)]
178#[serde(rename_all = "camelCase")]
179pub struct Content {
180 #[serde(default)]
181 pub parts: Vec<Part>,
182 pub role: Role,
183}
184
185#[derive(Debug, Serialize, Deserialize)]
186#[serde(rename_all = "camelCase")]
187pub struct SystemInstruction {
188 pub parts: Vec<Part>,
189}
190
191#[derive(Debug, PartialEq, Deserialize, Serialize)]
192#[serde(rename_all = "camelCase")]
193pub enum Role {
194 User,
195 Model,
196}
197
198#[derive(Debug, Serialize, Deserialize)]
199#[serde(untagged)]
200pub enum Part {
201 TextPart(TextPart),
202 InlineDataPart(InlineDataPart),
203 FunctionCallPart(FunctionCallPart),
204 FunctionResponsePart(FunctionResponsePart),
205 ThoughtPart(ThoughtPart),
206}
207
208#[derive(Debug, Serialize, Deserialize)]
209#[serde(rename_all = "camelCase")]
210pub struct TextPart {
211 pub text: String,
212}
213
214#[derive(Debug, Serialize, Deserialize)]
215#[serde(rename_all = "camelCase")]
216pub struct InlineDataPart {
217 pub inline_data: GenerativeContentBlob,
218}
219
220#[derive(Debug, Serialize, Deserialize)]
221#[serde(rename_all = "camelCase")]
222pub struct GenerativeContentBlob {
223 pub mime_type: String,
224 pub data: String,
225}
226
227#[derive(Debug, Serialize, Deserialize)]
228#[serde(rename_all = "camelCase")]
229pub struct FunctionCallPart {
230 pub function_call: FunctionCall,
231}
232
233#[derive(Debug, Serialize, Deserialize)]
234#[serde(rename_all = "camelCase")]
235pub struct FunctionResponsePart {
236 pub function_response: FunctionResponse,
237}
238
239#[derive(Debug, Serialize, Deserialize)]
240#[serde(rename_all = "camelCase")]
241pub struct ThoughtPart {
242 pub thought: bool,
243 pub thought_signature: String,
244}
245
246#[derive(Debug, Serialize, Deserialize)]
247#[serde(rename_all = "camelCase")]
248pub struct CitationSource {
249 #[serde(skip_serializing_if = "Option::is_none")]
250 pub start_index: Option<usize>,
251 #[serde(skip_serializing_if = "Option::is_none")]
252 pub end_index: Option<usize>,
253 #[serde(skip_serializing_if = "Option::is_none")]
254 pub uri: Option<String>,
255 #[serde(skip_serializing_if = "Option::is_none")]
256 pub license: Option<String>,
257}
258
259#[derive(Debug, Serialize, Deserialize)]
260#[serde(rename_all = "camelCase")]
261pub struct CitationMetadata {
262 pub citation_sources: Vec<CitationSource>,
263}
264
265#[derive(Debug, Serialize, Deserialize)]
266#[serde(rename_all = "camelCase")]
267pub struct PromptFeedback {
268 #[serde(skip_serializing_if = "Option::is_none")]
269 pub block_reason: Option<String>,
270 pub safety_ratings: Vec<SafetyRating>,
271 #[serde(skip_serializing_if = "Option::is_none")]
272 pub block_reason_message: Option<String>,
273}
274
275#[derive(Debug, Serialize, Deserialize, Default)]
276#[serde(rename_all = "camelCase")]
277pub struct UsageMetadata {
278 #[serde(skip_serializing_if = "Option::is_none")]
279 pub prompt_token_count: Option<usize>,
280 #[serde(skip_serializing_if = "Option::is_none")]
281 pub cached_content_token_count: Option<usize>,
282 #[serde(skip_serializing_if = "Option::is_none")]
283 pub candidates_token_count: Option<usize>,
284 #[serde(skip_serializing_if = "Option::is_none")]
285 pub tool_use_prompt_token_count: Option<usize>,
286 #[serde(skip_serializing_if = "Option::is_none")]
287 pub thoughts_token_count: Option<usize>,
288 #[serde(skip_serializing_if = "Option::is_none")]
289 pub total_token_count: Option<usize>,
290}
291
292#[derive(Debug, Deserialize, Serialize)]
293#[serde(rename_all = "camelCase")]
294pub struct GenerationConfig {
295 #[serde(skip_serializing_if = "Option::is_none")]
296 pub candidate_count: Option<usize>,
297 #[serde(skip_serializing_if = "Option::is_none")]
298 pub stop_sequences: Option<Vec<String>>,
299 #[serde(skip_serializing_if = "Option::is_none")]
300 pub max_output_tokens: Option<usize>,
301 #[serde(skip_serializing_if = "Option::is_none")]
302 pub temperature: Option<f64>,
303 #[serde(skip_serializing_if = "Option::is_none")]
304 pub top_p: Option<f64>,
305 #[serde(skip_serializing_if = "Option::is_none")]
306 pub top_k: Option<usize>,
307}
308
309#[derive(Debug, Serialize, Deserialize)]
310#[serde(rename_all = "camelCase")]
311pub struct SafetySetting {
312 pub category: HarmCategory,
313 pub threshold: HarmBlockThreshold,
314}
315
316#[derive(Debug, Serialize, Deserialize)]
317pub enum HarmCategory {
318 #[serde(rename = "HARM_CATEGORY_UNSPECIFIED")]
319 Unspecified,
320 #[serde(rename = "HARM_CATEGORY_DEROGATORY")]
321 Derogatory,
322 #[serde(rename = "HARM_CATEGORY_TOXICITY")]
323 Toxicity,
324 #[serde(rename = "HARM_CATEGORY_VIOLENCE")]
325 Violence,
326 #[serde(rename = "HARM_CATEGORY_SEXUAL")]
327 Sexual,
328 #[serde(rename = "HARM_CATEGORY_MEDICAL")]
329 Medical,
330 #[serde(rename = "HARM_CATEGORY_DANGEROUS")]
331 Dangerous,
332 #[serde(rename = "HARM_CATEGORY_HARASSMENT")]
333 Harassment,
334 #[serde(rename = "HARM_CATEGORY_HATE_SPEECH")]
335 HateSpeech,
336 #[serde(rename = "HARM_CATEGORY_SEXUALLY_EXPLICIT")]
337 SexuallyExplicit,
338 #[serde(rename = "HARM_CATEGORY_DANGEROUS_CONTENT")]
339 DangerousContent,
340}
341
342#[derive(Debug, Serialize, Deserialize)]
343#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
344pub enum HarmBlockThreshold {
345 #[serde(rename = "HARM_BLOCK_THRESHOLD_UNSPECIFIED")]
346 Unspecified,
347 BlockLowAndAbove,
348 BlockMediumAndAbove,
349 BlockOnlyHigh,
350 BlockNone,
351}
352
353#[derive(Debug, Serialize, Deserialize)]
354#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
355pub enum HarmProbability {
356 #[serde(rename = "HARM_PROBABILITY_UNSPECIFIED")]
357 Unspecified,
358 Negligible,
359 Low,
360 Medium,
361 High,
362}
363
364#[derive(Debug, Serialize, Deserialize)]
365#[serde(rename_all = "camelCase")]
366pub struct SafetyRating {
367 pub category: HarmCategory,
368 pub probability: HarmProbability,
369}
370
371#[derive(Debug, Serialize, Deserialize)]
372#[serde(rename_all = "camelCase")]
373pub struct CountTokensRequest {
374 pub generate_content_request: GenerateContentRequest,
375}
376
377#[derive(Debug, Serialize, Deserialize)]
378#[serde(rename_all = "camelCase")]
379pub struct CountTokensResponse {
380 pub total_tokens: usize,
381}
382
383#[derive(Debug, Serialize, Deserialize)]
384pub struct FunctionCall {
385 pub name: String,
386 pub args: serde_json::Value,
387}
388
389#[derive(Debug, Serialize, Deserialize)]
390pub struct FunctionResponse {
391 pub name: String,
392 pub response: serde_json::Value,
393}
394
395#[derive(Debug, Serialize, Deserialize)]
396#[serde(rename_all = "camelCase")]
397pub struct Tool {
398 pub function_declarations: Vec<FunctionDeclaration>,
399}
400
401#[derive(Debug, Serialize, Deserialize)]
402#[serde(rename_all = "camelCase")]
403pub struct ToolConfig {
404 pub function_calling_config: FunctionCallingConfig,
405}
406
407#[derive(Debug, Serialize, Deserialize)]
408#[serde(rename_all = "camelCase")]
409pub struct FunctionCallingConfig {
410 pub mode: FunctionCallingMode,
411 #[serde(skip_serializing_if = "Option::is_none")]
412 pub allowed_function_names: Option<Vec<String>>,
413}
414
415#[derive(Debug, Serialize, Deserialize)]
416#[serde(rename_all = "lowercase")]
417pub enum FunctionCallingMode {
418 Auto,
419 Any,
420 None,
421}
422
423#[derive(Debug, Serialize, Deserialize)]
424pub struct FunctionDeclaration {
425 pub name: String,
426 pub description: String,
427 pub parameters: serde_json::Value,
428}
429
430#[derive(Debug, Default)]
431pub struct ModelName {
432 pub model_id: String,
433}
434
435impl ModelName {
436 pub fn is_empty(&self) -> bool {
437 self.model_id.is_empty()
438 }
439}
440
441const MODEL_NAME_PREFIX: &str = "models/";
442
443impl Serialize for ModelName {
444 fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
445 where
446 S: Serializer,
447 {
448 serializer.serialize_str(&format!("{MODEL_NAME_PREFIX}{}", &self.model_id))
449 }
450}
451
452impl<'de> Deserialize<'de> for ModelName {
453 fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
454 where
455 D: Deserializer<'de>,
456 {
457 let string = String::deserialize(deserializer)?;
458 if let Some(id) = string.strip_prefix(MODEL_NAME_PREFIX) {
459 Ok(Self {
460 model_id: id.to_string(),
461 })
462 } else {
463 return Err(serde::de::Error::custom(format!(
464 "Expected model name to begin with {}, got: {}",
465 MODEL_NAME_PREFIX, string
466 )));
467 }
468 }
469}
470
471#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
472#[derive(Clone, Default, Debug, Deserialize, Serialize, PartialEq, Eq, strum::EnumIter)]
473pub enum Model {
474 #[serde(rename = "gemini-1.5-pro")]
475 Gemini15Pro,
476 #[serde(rename = "gemini-1.5-flash")]
477 Gemini15Flash,
478 #[serde(rename = "gemini-2.0-pro-exp")]
479 Gemini20Pro,
480 #[serde(rename = "gemini-2.0-flash")]
481 #[default]
482 Gemini20Flash,
483 #[serde(rename = "gemini-2.0-flash-thinking-exp")]
484 Gemini20FlashThinking,
485 #[serde(rename = "gemini-2.0-flash-lite-preview")]
486 Gemini20FlashLite,
487 #[serde(rename = "gemini-2.5-pro-exp-03-25")]
488 Gemini25ProExp0325,
489 #[serde(rename = "gemini-2.5-pro-preview-03-25")]
490 Gemini25ProPreview0325,
491 #[serde(rename = "gemini-2.5-flash-preview-04-17")]
492 Gemini25FlashPreview0417,
493 #[serde(rename = "custom")]
494 Custom {
495 name: String,
496 /// The name displayed in the UI, such as in the assistant panel model dropdown menu.
497 display_name: Option<String>,
498 max_tokens: usize,
499 },
500}
501
502impl Model {
503 pub fn default_fast() -> Model {
504 Model::Gemini20Flash
505 }
506
507 pub fn id(&self) -> &str {
508 match self {
509 Model::Gemini15Pro => "gemini-1.5-pro",
510 Model::Gemini15Flash => "gemini-1.5-flash",
511 Model::Gemini20Pro => "gemini-2.0-pro-exp",
512 Model::Gemini20Flash => "gemini-2.0-flash",
513 Model::Gemini20FlashThinking => "gemini-2.0-flash-thinking-exp",
514 Model::Gemini20FlashLite => "gemini-2.0-flash-lite-preview",
515 Model::Gemini25ProExp0325 => "gemini-2.5-pro-exp-03-25",
516 Model::Gemini25ProPreview0325 => "gemini-2.5-pro-preview-03-25",
517 Model::Gemini25FlashPreview0417 => "gemini-2.5-flash-preview-04-17",
518 Model::Custom { name, .. } => name,
519 }
520 }
521
522 pub fn display_name(&self) -> &str {
523 match self {
524 Model::Gemini15Pro => "Gemini 1.5 Pro",
525 Model::Gemini15Flash => "Gemini 1.5 Flash",
526 Model::Gemini20Pro => "Gemini 2.0 Pro",
527 Model::Gemini20Flash => "Gemini 2.0 Flash",
528 Model::Gemini20FlashThinking => "Gemini 2.0 Flash Thinking",
529 Model::Gemini20FlashLite => "Gemini 2.0 Flash Lite",
530 Model::Gemini25ProExp0325 => "Gemini 2.5 Pro Exp",
531 Model::Gemini25ProPreview0325 => "Gemini 2.5 Pro Preview",
532 Model::Gemini25FlashPreview0417 => "Gemini 2.5 Flash Preview",
533 Self::Custom {
534 name, display_name, ..
535 } => display_name.as_ref().unwrap_or(name),
536 }
537 }
538
539 pub fn max_token_count(&self) -> usize {
540 const ONE_MILLION: usize = 1_048_576;
541 const TWO_MILLION: usize = 2_097_152;
542 match self {
543 Model::Gemini15Pro => TWO_MILLION,
544 Model::Gemini15Flash => ONE_MILLION,
545 Model::Gemini20Pro => TWO_MILLION,
546 Model::Gemini20Flash => ONE_MILLION,
547 Model::Gemini20FlashThinking => ONE_MILLION,
548 Model::Gemini20FlashLite => ONE_MILLION,
549 Model::Gemini25ProExp0325 => ONE_MILLION,
550 Model::Gemini25ProPreview0325 => ONE_MILLION,
551 Model::Gemini25FlashPreview0417 => ONE_MILLION,
552 Model::Custom { max_tokens, .. } => *max_tokens,
553 }
554 }
555}
556
557impl std::fmt::Display for Model {
558 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
559 write!(f, "{}", self.id())
560 }
561}