1mod eval;
2mod headless_assistant;
3mod judge;
4
5use clap::Parser;
6use eval::{Eval, EvalOutput};
7use futures::{stream, StreamExt};
8use gpui::{Application, AsyncApp};
9use headless_assistant::{authenticate_model_provider, find_model, HeadlessAppState};
10use itertools::Itertools;
11use judge::Judge;
12use language_model::{LanguageModel, LanguageModelRegistry};
13use regex::Regex;
14use reqwest_client::ReqwestClient;
15use std::{cmp, path::PathBuf, sync::Arc};
16
17#[derive(Parser, Debug)]
18#[command(
19 name = "assistant_eval",
20 disable_version_flag = true,
21 before_help = "Tool eval runner"
22)]
23struct Args {
24 /// Regexes to match the names of evals to run.
25 eval_name_regexes: Vec<String>,
26 /// Runs all evals in `evaluation_data`, causes the regex to be ignored.
27 #[arg(long)]
28 all: bool,
29 /// Name of the model (default: "claude-3-7-sonnet-latest")
30 #[arg(long, default_value = "claude-3-7-sonnet-latest")]
31 model_name: String,
32 /// Name of the editor model (default: value of `--model_name`).
33 #[arg(long)]
34 editor_model_name: Option<String>,
35 /// Name of the judge model (default: value of `--model_name`).
36 #[arg(long)]
37 judge_model_name: Option<String>,
38 /// Number of evaluations to run concurrently (default: 10)
39 #[arg(short, long, default_value = "10")]
40 concurrency: usize,
41}
42
43fn main() {
44 env_logger::init();
45 let args = Args::parse();
46 let http_client = Arc::new(ReqwestClient::new());
47 let app = Application::headless().with_http_client(http_client.clone());
48
49 let crate_dir = PathBuf::from("../zed-agent-bench");
50 let evaluation_data_dir = crate_dir.join("evaluation_data").canonicalize().unwrap();
51 let repos_dir = crate_dir.join("repos").canonicalize().unwrap();
52
53 let all_evals = std::fs::read_dir(&evaluation_data_dir)
54 .unwrap()
55 .map(|path| path.unwrap().file_name().to_string_lossy().to_string())
56 .collect::<Vec<_>>();
57
58 let evals_to_run = if args.all {
59 all_evals
60 } else {
61 args.eval_name_regexes
62 .into_iter()
63 .map(|regex_string| Regex::new(®ex_string).unwrap())
64 .flat_map(|regex| {
65 all_evals
66 .iter()
67 .filter(|eval_name| regex.is_match(eval_name))
68 .cloned()
69 .collect::<Vec<_>>()
70 })
71 .collect::<Vec<_>>()
72 };
73
74 if evals_to_run.is_empty() {
75 panic!("Names of evals to run must be provided or `--all` specified");
76 }
77
78 println!("Will run the following evals: {evals_to_run:?}");
79 println!("Running up to {} evals concurrently", args.concurrency);
80
81 let editor_model_name = if let Some(model_name) = args.editor_model_name {
82 model_name
83 } else {
84 args.model_name.clone()
85 };
86
87 let judge_model_name = if let Some(model_name) = args.judge_model_name {
88 model_name
89 } else {
90 args.model_name.clone()
91 };
92
93 app.run(move |cx| {
94 let app_state = headless_assistant::init(cx);
95
96 let model = find_model(&args.model_name, cx).unwrap();
97 let editor_model = find_model(&editor_model_name, cx).unwrap();
98 let judge_model = find_model(&judge_model_name, cx).unwrap();
99
100 LanguageModelRegistry::global(cx).update(cx, |registry, cx| {
101 registry.set_active_model(Some(model.clone()), cx);
102 registry.set_editor_model(Some(editor_model.clone()), cx);
103 });
104
105 let model_provider_id = model.provider_id();
106 let editor_model_provider_id = editor_model.provider_id();
107 let judge_model_provider_id = judge_model.provider_id();
108
109 cx.spawn(move |cx| async move {
110 // Authenticate all model providers first
111 cx.update(|cx| authenticate_model_provider(model_provider_id.clone(), cx))
112 .unwrap()
113 .await
114 .unwrap();
115 cx.update(|cx| authenticate_model_provider(editor_model_provider_id.clone(), cx))
116 .unwrap()
117 .await
118 .unwrap();
119 cx.update(|cx| authenticate_model_provider(judge_model_provider_id.clone(), cx))
120 .unwrap()
121 .await
122 .unwrap();
123
124 let loaded_evals = stream::iter(evals_to_run)
125 .map(|eval_name| {
126 let eval_path = evaluation_data_dir.join(&eval_name);
127 let repos_dir = repos_dir.clone();
128 async move {
129 match Eval::load(eval_name.clone(), eval_path, &repos_dir).await {
130 Ok(eval) => Some(eval),
131 Err(err) => {
132 // TODO: Persist errors / surface errors at the end.
133 println!("Error loading {eval_name}: {err}");
134 None
135 }
136 }
137 }
138 })
139 .buffer_unordered(args.concurrency)
140 .collect::<Vec<_>>()
141 .await
142 .into_iter()
143 .flatten()
144 .collect::<Vec<_>>();
145
146 // The evals need to be loaded and grouped by URL before concurrently running, since
147 // evals that use the same remote URL will use the same working directory.
148 let mut evals_grouped_by_url: Vec<Vec<Eval>> = loaded_evals
149 .into_iter()
150 .map(|eval| (eval.eval_setup.url.clone(), eval))
151 .into_group_map()
152 .into_values()
153 .collect::<Vec<_>>();
154
155 // Sort groups in descending order, so that bigger groups start first.
156 evals_grouped_by_url.sort_by_key(|evals| cmp::Reverse(evals.len()));
157
158 let results = stream::iter(evals_grouped_by_url)
159 .map(|evals| {
160 let model = model.clone();
161 let judge_model = judge_model.clone();
162 let app_state = app_state.clone();
163 let cx = cx.clone();
164
165 async move {
166 let mut results = Vec::new();
167 for eval in evals {
168 let name = eval.name.clone();
169 println!("Starting eval named {}", name);
170 let result = run_eval(
171 eval,
172 model.clone(),
173 judge_model.clone(),
174 app_state.clone(),
175 cx.clone(),
176 )
177 .await;
178 results.push((name, result));
179 }
180 results
181 }
182 })
183 .buffer_unordered(args.concurrency)
184 .collect::<Vec<_>>()
185 .await
186 .into_iter()
187 .flatten()
188 .collect::<Vec<_>>();
189
190 // Process results in order of completion
191 for (eval_name, result) in results {
192 match result {
193 Ok((eval_output, judge_output)) => {
194 println!("Generated diff for {eval_name}:\n");
195 println!("{}\n", eval_output.diff);
196 println!("Last message for {eval_name}:\n");
197 println!("{}\n", eval_output.last_message);
198 println!("Elapsed time: {:?}", eval_output.elapsed_time);
199 println!(
200 "Assistant response count: {}",
201 eval_output.assistant_response_count
202 );
203 println!("Tool use counts: {:?}", eval_output.tool_use_counts);
204 println!("Judge output for {eval_name}: {judge_output}");
205 }
206 Err(err) => {
207 // TODO: Persist errors / surface errors at the end.
208 println!("Error running {eval_name}: {err}");
209 }
210 }
211 }
212
213 cx.update(|cx| cx.quit()).unwrap();
214 })
215 .detach();
216 });
217
218 println!("Done running evals");
219}
220
221async fn run_eval(
222 eval: Eval,
223 model: Arc<dyn LanguageModel>,
224 judge_model: Arc<dyn LanguageModel>,
225 app_state: Arc<HeadlessAppState>,
226 cx: AsyncApp,
227) -> anyhow::Result<(EvalOutput, String)> {
228 let path = eval.path.clone();
229 let judge = Judge::load(&path, judge_model).await?;
230 let eval_output = cx.update(|cx| eval.run(app_state, model, cx))?.await?;
231 let judge_output = cx.update(|cx| judge.run(&eval_output, cx))?.await?;
232 eval_output.save_to_directory(&path, judge_output.to_string())?;
233 Ok((eval_output, judge_output))
234}