1mod eval;
2mod headless_assistant;
3mod judge;
4
5use clap::Parser;
6use eval::{Eval, EvalOutput};
7use futures::{stream, StreamExt};
8use gpui::{Application, AsyncApp};
9use headless_assistant::{authenticate_model_provider, find_model, HeadlessAppState};
10use itertools::Itertools;
11use judge::Judge;
12use language_model::{LanguageModel, LanguageModelRegistry};
13use regex::Regex;
14use reqwest_client::ReqwestClient;
15use std::{cmp, path::PathBuf, sync::Arc};
16
17#[derive(Parser, Debug)]
18#[command(
19 name = "assistant_eval",
20 disable_version_flag = true,
21 before_help = "Tool eval runner"
22)]
23struct Args {
24 /// Regexes to match the names of evals to run.
25 eval_name_regexes: Vec<String>,
26 /// Runs all evals in `evaluation_data`, causes the regex to be ignored.
27 #[arg(long)]
28 all: bool,
29 /// Name of the model (default: "claude-3-7-sonnet-latest")
30 #[arg(long, default_value = "claude-3-7-sonnet-latest")]
31 model_name: String,
32 /// Name of the editor model (default: value of `--model_name`).
33 #[arg(long)]
34 editor_model_name: Option<String>,
35 /// Name of the judge model (default: value of `--model_name`).
36 #[arg(long)]
37 judge_model_name: Option<String>,
38 /// Number of evaluations to run concurrently (default: 10)
39 #[arg(short, long, default_value = "10")]
40 concurrency: usize,
41}
42
43fn main() {
44 env_logger::init();
45 let args = Args::parse();
46 let http_client = Arc::new(ReqwestClient::new());
47 let app = Application::headless().with_http_client(http_client.clone());
48
49 let crate_dir = PathBuf::from("../zed-agent-bench");
50 let evaluation_data_dir = crate_dir.join("evaluation_data").canonicalize().unwrap();
51
52 let repos_dir = crate_dir.join("repos");
53 if !repos_dir.exists() {
54 std::fs::create_dir_all(&repos_dir).unwrap();
55 }
56 let repos_dir = repos_dir.canonicalize().unwrap();
57
58 let all_evals = std::fs::read_dir(&evaluation_data_dir)
59 .unwrap()
60 .map(|path| path.unwrap().file_name().to_string_lossy().to_string())
61 .collect::<Vec<_>>();
62
63 let evals_to_run = if args.all {
64 all_evals
65 } else {
66 args.eval_name_regexes
67 .into_iter()
68 .map(|regex_string| Regex::new(®ex_string).unwrap())
69 .flat_map(|regex| {
70 all_evals
71 .iter()
72 .filter(|eval_name| regex.is_match(eval_name))
73 .cloned()
74 .collect::<Vec<_>>()
75 })
76 .collect::<Vec<_>>()
77 };
78
79 if evals_to_run.is_empty() {
80 panic!("Names of evals to run must be provided or `--all` specified");
81 }
82
83 println!("Will run the following evals: {evals_to_run:?}");
84 println!("Running up to {} evals concurrently", args.concurrency);
85
86 let editor_model_name = if let Some(model_name) = args.editor_model_name {
87 model_name
88 } else {
89 args.model_name.clone()
90 };
91
92 let judge_model_name = if let Some(model_name) = args.judge_model_name {
93 model_name
94 } else {
95 args.model_name.clone()
96 };
97
98 app.run(move |cx| {
99 let app_state = headless_assistant::init(cx);
100
101 let model = find_model(&args.model_name, cx).unwrap();
102 let editor_model = find_model(&editor_model_name, cx).unwrap();
103 let judge_model = find_model(&judge_model_name, cx).unwrap();
104
105 LanguageModelRegistry::global(cx).update(cx, |registry, cx| {
106 registry.set_active_model(Some(model.clone()), cx);
107 registry.set_editor_model(Some(editor_model.clone()), cx);
108 });
109
110 let model_provider_id = model.provider_id();
111 let editor_model_provider_id = editor_model.provider_id();
112 let judge_model_provider_id = judge_model.provider_id();
113
114 cx.spawn(move |cx| async move {
115 // Authenticate all model providers first
116 cx.update(|cx| authenticate_model_provider(model_provider_id.clone(), cx))
117 .unwrap()
118 .await
119 .unwrap();
120 cx.update(|cx| authenticate_model_provider(editor_model_provider_id.clone(), cx))
121 .unwrap()
122 .await
123 .unwrap();
124 cx.update(|cx| authenticate_model_provider(judge_model_provider_id.clone(), cx))
125 .unwrap()
126 .await
127 .unwrap();
128
129 let loaded_evals = stream::iter(evals_to_run)
130 .map(|eval_name| {
131 let eval_path = evaluation_data_dir.join(&eval_name);
132 let repos_dir = repos_dir.clone();
133 async move {
134 match Eval::load(eval_name.clone(), eval_path, &repos_dir).await {
135 Ok(eval) => Some(eval),
136 Err(err) => {
137 // TODO: Persist errors / surface errors at the end.
138 println!("Error loading {eval_name}: {err}");
139 None
140 }
141 }
142 }
143 })
144 .buffer_unordered(args.concurrency)
145 .collect::<Vec<_>>()
146 .await
147 .into_iter()
148 .flatten()
149 .collect::<Vec<_>>();
150
151 // The evals need to be loaded and grouped by URL before concurrently running, since
152 // evals that use the same remote URL will use the same working directory.
153 let mut evals_grouped_by_url: Vec<Vec<Eval>> = loaded_evals
154 .into_iter()
155 .map(|eval| (eval.eval_setup.url.clone(), eval))
156 .into_group_map()
157 .into_values()
158 .collect::<Vec<_>>();
159
160 // Sort groups in descending order, so that bigger groups start first.
161 evals_grouped_by_url.sort_by_key(|evals| cmp::Reverse(evals.len()));
162
163 let results = stream::iter(evals_grouped_by_url)
164 .map(|evals| {
165 let model = model.clone();
166 let judge_model = judge_model.clone();
167 let app_state = app_state.clone();
168 let cx = cx.clone();
169
170 async move {
171 let mut results = Vec::new();
172 for eval in evals {
173 let name = eval.name.clone();
174 println!("Starting eval named {}", name);
175 let result = run_eval(
176 eval,
177 model.clone(),
178 judge_model.clone(),
179 app_state.clone(),
180 cx.clone(),
181 )
182 .await;
183 results.push((name, result));
184 }
185 results
186 }
187 })
188 .buffer_unordered(args.concurrency)
189 .collect::<Vec<_>>()
190 .await
191 .into_iter()
192 .flatten()
193 .collect::<Vec<_>>();
194
195 // Process results in order of completion
196 for (eval_name, result) in results {
197 match result {
198 Ok((eval_output, judge_output)) => {
199 println!("Generated diff for {eval_name}:\n");
200 println!("{}\n", eval_output.diff);
201 println!("Last message for {eval_name}:\n");
202 println!("{}\n", eval_output.last_message);
203 println!("Elapsed time: {:?}", eval_output.elapsed_time);
204 println!(
205 "Assistant response count: {}",
206 eval_output.assistant_response_count
207 );
208 println!("Tool use counts: {:?}", eval_output.tool_use_counts);
209 println!("Judge output for {eval_name}: {judge_output}");
210 }
211 Err(err) => {
212 // TODO: Persist errors / surface errors at the end.
213 println!("Error running {eval_name}: {err}");
214 }
215 }
216 }
217
218 cx.update(|cx| cx.quit()).unwrap();
219 })
220 .detach();
221 });
222
223 println!("Done running evals");
224}
225
226async fn run_eval(
227 eval: Eval,
228 model: Arc<dyn LanguageModel>,
229 judge_model: Arc<dyn LanguageModel>,
230 app_state: Arc<HeadlessAppState>,
231 cx: AsyncApp,
232) -> anyhow::Result<(EvalOutput, String)> {
233 let path = eval.path.clone();
234 let judge = Judge::load(&path, judge_model).await?;
235 let eval_output = cx.update(|cx| eval.run(app_state, model, cx))?.await?;
236 let judge_output = cx.update(|cx| judge.run(&eval_output, cx))?.await?;
237 eval_output.save_to_directory(&path, judge_output.to_string())?;
238 Ok((eval_output, judge_output))
239}