@@ -26,6 +26,7 @@ use settings::{Settings, SettingsStore};
use std::collections::HashSet;
use std::path::{Path, PathBuf};
use std::sync::Arc;
+use std::usize;
use util::ResultExt as _;
pub const RUNS_DIR: &str = "./crates/eval/runs";
@@ -97,8 +98,27 @@ fn main() {
std::fs::create_dir_all(&run_dir)?;
let mut examples = Vec::new();
- for example_path in example_paths {
- let example = Example::load_from_directory(&example_path, &run_dir)?;
+
+ const COLORS: [&str; 12] = [
+ "\x1b[31m", // Red
+ "\x1b[32m", // Green
+ "\x1b[33m", // Yellow
+ "\x1b[34m", // Blue
+ "\x1b[35m", // Magenta
+ "\x1b[36m", // Cyan
+ "\x1b[91m", // Bright Red
+ "\x1b[92m", // Bright Green
+ "\x1b[93m", // Bright Yellow
+ "\x1b[94m", // Bright Blue
+ "\x1b[95m", // Bright Magenta
+ "\x1b[96m", // Bright Cyan
+ ];
+
+ let mut max_name_width = 0;
+ let mut skipped = Vec::new();
+
+ for example_path in &example_paths {
+ let example = Example::load_from_directory(example_path, &run_dir)?;
if !example
.base
@@ -106,25 +126,49 @@ fn main() {
.as_ref()
.map_or(false, |lang| languages.contains(lang))
{
- println!("Skipping {}", example.name);
+ skipped.push(example.name);
continue;
}
- println!("{}> Logging to {:?}", example.name, example.log_file_path);
+ let name_len = example.name.len();
+ if name_len > max_name_width {
+ max_name_width = example.name.len();
+ }
examples.push(example);
}
- let mut repo_urls = HashSet::new();
+ println!("Skipped examples: {}\n", skipped.join(", "));
+
+ if examples.is_empty() {
+ eprintln!("Filter matched no examples");
+ return cx.update(|cx| cx.quit());
+ }
+
+ let mut repo_urls = HashSet::new();
let mut clone_tasks = Vec::new();
- for example in examples.iter() {
+ for (i, example) in examples.iter_mut().enumerate() {
+ let color = COLORS[i % COLORS.len()].to_string();
+ example.set_log_prefix_style(&color, max_name_width);
+
+ println!(
+ "{}Logging to: {}",
+ example.log_prefix,
+ example.output_file_path.display()
+ );
+
let repo_url = example.base.url.clone();
if repo_urls.insert(repo_url.clone()) {
let repo_path = repo_path_for_url(&repo_url);
if !repo_path.join(".git").is_dir() {
- println!("Cloning: {}", repo_url);
+ println!(
+ "{:<width$} < {}",
+ "↓ Cloning",
+ repo_url,
+ width = max_name_width
+ );
let git_task = cx.spawn(async move |_cx| {
std::fs::create_dir_all(&repo_path)?;
@@ -134,7 +178,12 @@ fn main() {
clone_tasks.push(git_task);
} else {
- println!("Already cloned: {}", repo_url);
+ println!(
+ "{:<width$} < {}",
+ "✔︎ Already cloned",
+ repo_url,
+ width = max_name_width
+ );
let actual_origin =
run_git(&repo_path, &["remote", "get-url", "origin"]).await?;
@@ -177,23 +226,27 @@ fn main() {
let mut judge_scores = Vec::new();
for (result, example) in results {
- println!("📜 {:<30}: {:?}", example.name, example.log_file_path);
match result {
Err(err) => {
- println!("💥 {:<30}: {:?}", example.name, err);
+ println!("💥 {}{:?}", example.log_prefix, err);
}
Ok(judge_output) => {
const SCORES: [&str; 6] = ["💀", "😭", "😔", "😐", "🙂", "🤩"];
println!(
- "{} {:<30}: {}",
+ "{} {}{}",
SCORES[judge_output.score.min(5) as usize],
- example.name,
+ example.log_prefix,
judge_output.score,
);
judge_scores.push(judge_output.score);
}
}
+ println!(
+ "{} > {}",
+ " ".repeat(max_name_width),
+ example.output_file_path.display()
+ );
}
let score_count = judge_scores.len();
@@ -56,10 +56,12 @@ pub struct Example {
pub prompt: String,
/// Content of `criteria.md`
pub criteria: String,
- /// Markdown log file to append to
- pub log_file: Arc<Mutex<File>>,
- /// Path to markdown log file
- pub log_file_path: PathBuf,
+ /// Markdown output file to append to
+ pub output_file: Arc<Mutex<File>>,
+ /// Path to markdown output file
+ pub output_file_path: PathBuf,
+ /// Prefix used for logging that identifies this example
+ pub log_prefix: String,
}
#[derive(Debug, Serialize, Deserialize, Clone)]
@@ -86,28 +88,41 @@ pub struct JudgeOutput {
impl Example {
/// Load an example from a directory containing base.toml, prompt.md, and criteria.md
pub fn load_from_directory(dir_path: &Path, run_dir: &Path) -> Result<Self> {
- let name = dir_path.file_name().unwrap().to_string_lossy().to_string();
+ let name = Self::name_from_path(dir_path);
let base_path = dir_path.join("base.toml");
let prompt_path = dir_path.join("prompt.md");
let criteria_path = dir_path.join("criteria.md");
- let log_file_path = run_dir.join(format!(
+ let output_file_path = run_dir.join(format!(
"{}.md",
dir_path.file_name().unwrap().to_str().unwrap()
));
- let log_file = Arc::new(Mutex::new(File::create(&log_file_path).unwrap()));
- println!("{}> Logging to {:?}", name, log_file_path);
+ let output_file = Arc::new(Mutex::new(File::create(&output_file_path).unwrap()));
Ok(Example {
- name,
+ name: name.clone(),
base: toml::from_str(&fs::read_to_string(&base_path)?)?,
prompt: fs::read_to_string(prompt_path.clone())?,
criteria: fs::read_to_string(criteria_path.clone())?,
- log_file,
- log_file_path,
+ output_file,
+ output_file_path,
+ log_prefix: name,
})
}
+ pub fn set_log_prefix_style(&mut self, color: &str, name_width: usize) {
+ self.log_prefix = format!(
+ "{}{:<width$}\x1b[0m | ",
+ color,
+ self.name,
+ width = name_width
+ );
+ }
+
+ pub fn name_from_path(path: &Path) -> String {
+ path.file_name().unwrap().to_string_lossy().to_string()
+ }
+
pub fn worktree_path(&self) -> PathBuf {
Path::new(WORKTREES_DIR)
.canonicalize()
@@ -120,7 +135,7 @@ impl Example {
pub async fn setup(&self) -> Result<()> {
let repo_path = repo_path_for_url(&self.base.url);
- println!("{}> Fetching", self.name);
+ println!("{}Fetching", self.log_prefix);
run_git(
&repo_path,
@@ -131,7 +146,7 @@ impl Example {
let worktree_path = self.worktree_path();
if worktree_path.is_dir() {
- println!("{}> Resetting existing worktree", self.name);
+ println!("{}Resetting existing worktree", self.log_prefix);
// TODO: consider including "-x" to remove ignored files. The downside of this is that
// it will also remove build artifacts, and so prevent incremental reuse there.
@@ -139,7 +154,7 @@ impl Example {
run_git(&worktree_path, &["reset", "--hard", "HEAD"]).await?;
run_git(&worktree_path, &["checkout", &self.base.revision]).await?;
} else {
- println!("{}> Creating worktree", self.name);
+ println!("{}Creating worktree", self.log_prefix);
let worktree_path_string = worktree_path.to_string_lossy().to_string();
@@ -235,7 +250,7 @@ impl Example {
// TODO: remove this once the diagnostics tool waits for new diagnostics
cx.background_executor().timer(Duration::new(5, 0)).await;
- wait_for_lang_server(&lsp_store, this.name.clone(), cx).await?;
+ wait_for_lang_server(&lsp_store, this.log_prefix.clone(), cx).await?;
lsp_store.update(cx, |lsp_store, cx| {
lsp_open_handle.update(cx, |buffer, cx| {
@@ -272,11 +287,11 @@ impl Example {
thread_store.update(cx, |thread_store, cx| thread_store.create_thread(cx))?;
{
- let mut log_file = this.log_file.lock().unwrap();
- writeln!(&mut log_file, "👤 USER:").log_err();
- writeln!(&mut log_file, "{}", this.prompt).log_err();
- writeln!(&mut log_file, "🤖 ASSISTANT:").log_err();
- log_file.flush().log_err();
+ let mut output_file = this.output_file.lock().unwrap();
+ writeln!(&mut output_file, "👤 USER:").log_err();
+ writeln!(&mut output_file, "{}", this.prompt).log_err();
+ writeln!(&mut output_file, "🤖 ASSISTANT:").log_err();
+ output_file.flush().log_err();
}
let tool_use_counts: Arc<Mutex<HashMap<Arc<str>, u32>>> =
@@ -289,8 +304,8 @@ impl Example {
});
let event_handler_task = cx.spawn({
- let log_file = this.log_file.clone();
- let name = this.name.clone();
+ let output_file = this.output_file.clone();
+ let log_prefix = this.log_prefix.clone();
let tool_use_counts = tool_use_counts.clone();
let thread = thread.downgrade();
async move |cx| {
@@ -305,7 +320,7 @@ impl Example {
return Err(anyhow!("ThreadEvent channel ended early"));
};
- let mut log_file = log_file.lock().unwrap();
+ let mut output_file = output_file.lock().unwrap();
match event {
ThreadEvent::Stopped(reason) => match reason {
@@ -324,15 +339,15 @@ impl Example {
break Err(anyhow!(thread_error.clone()));
}
ThreadEvent::StreamedAssistantText(_, chunk) => {
- write!(&mut log_file, "{}", chunk).log_err();
+ write!(&mut output_file, "{}", chunk).log_err();
}
ThreadEvent::StreamedAssistantThinking(_, chunk) => {
- write!(&mut log_file, "{}", chunk).log_err();
+ write!(&mut output_file, "{}", chunk).log_err();
}
ThreadEvent::UsePendingTools { tool_uses } => {
- writeln!(&mut log_file, "\n\nUSING TOOLS:").log_err();
+ writeln!(&mut output_file, "\n\nUSING TOOLS:").log_err();
for tool_use in tool_uses {
- writeln!(&mut log_file, "{}: {}", tool_use.name, tool_use.input)
+ writeln!(&mut output_file, "{}: {}", tool_use.name, tool_use.input)
.log_err();
}
}
@@ -343,12 +358,12 @@ impl Example {
} => {
if let Some(tool_use) = pending_tool_use {
let message = format!("TOOL FINISHED: {}", tool_use.name);
- println!("{name}> {message}");
- writeln!(&mut log_file, "\n{}", message).log_err();
+ println!("{}{message}", log_prefix);
+ writeln!(&mut output_file, "\n{}", message).log_err();
}
thread.update(cx, |thread, _cx| {
if let Some(tool_result) = thread.tool_result(&tool_use_id) {
- writeln!(&mut log_file, "\n{}\n", tool_result.content).log_err();
+ writeln!(&mut output_file, "\n{}\n", tool_result.content).log_err();
let mut tool_use_counts = tool_use_counts.lock().unwrap();
*tool_use_counts
.entry(tool_result.tool_name.clone())
@@ -359,7 +374,7 @@ impl Example {
_ => {}
}
- log_file.flush().log_err();
+ output_file.flush().log_err();
}
}
});
@@ -373,7 +388,7 @@ impl Example {
event_handler_task.await?;
if let Some((_, lsp_store)) = lsp_open_handle_and_store.as_ref() {
- wait_for_lang_server(lsp_store, this.name.clone(), cx).await?;
+ wait_for_lang_server(lsp_store, this.log_prefix.clone(), cx).await?;
}
let repository_diff = this.repository_diff().await?;
@@ -433,13 +448,13 @@ impl Example {
let response = send_language_model_request(model, request, cx).await?;
- let mut log_file = self.log_file.lock().unwrap();
+ let mut output_file = self.output_file.lock().unwrap();
- writeln!(&mut log_file, "\n\n").log_err();
- writeln!(&mut log_file, "========================================").log_err();
- writeln!(&mut log_file, " JUDGE OUTPUT ").log_err();
- writeln!(&mut log_file, "========================================").log_err();
- writeln!(&mut log_file, "\n{}", &response).log_err();
+ writeln!(&mut output_file, "\n\n").log_err();
+ writeln!(&mut output_file, "========================================").log_err();
+ writeln!(&mut output_file, " JUDGE OUTPUT ").log_err();
+ writeln!(&mut output_file, "========================================").log_err();
+ writeln!(&mut output_file, "\n{}", &response).log_err();
parse_judge_output(&response)
}
@@ -453,7 +468,7 @@ impl Example {
fn wait_for_lang_server(
lsp_store: &Entity<LspStore>,
- name: String,
+ log_prefix: String,
cx: &mut AsyncApp,
) -> Task<Result<()>> {
if cx
@@ -464,13 +479,13 @@ fn wait_for_lang_server(
return Task::ready(anyhow::Ok(()));
}
- println!("{}> ⏵ Waiting for language server", name);
+ println!("{}⏵ Waiting for language server", log_prefix);
let (mut tx, mut rx) = mpsc::channel(1);
let subscription =
cx.subscribe(&lsp_store, {
- let name = name.clone();
+ let log_prefix = log_prefix.clone();
move |lsp_store, event, cx| {
match event {
project::LspStoreEvent::LanguageServerUpdate {
@@ -482,7 +497,7 @@ fn wait_for_lang_server(
},
),
..
- } => println!("{name}> ⟲ {message}"),
+ } => println!("{}⟲ {message}", log_prefix),
_ => {}
}
@@ -496,7 +511,7 @@ fn wait_for_lang_server(
let timeout = cx.background_executor().timer(Duration::new(60 * 5, 0));
let result = futures::select! {
_ = rx.next() => {
- println!("{}> ⚑ Language server idle", name);
+ println!("{}⚑ Language server idle", log_prefix);
anyhow::Ok(())
},
_ = timeout.fuse() => {
@@ -623,7 +638,6 @@ pub async fn send_language_model_request(
while let Some(chunk_result) = stream.stream.next().await {
match chunk_result {
Ok(chunk_str) => {
- print!("{}", &chunk_str);
full_response.push_str(&chunk_str);
}
Err(err) => {