Use longer timeout on evals (#42465)

Richard Feldman and Bennet Bo Fenner created 3 days ago

The GPT-5 ones in particular can take a long time!

Release Notes:

- N/A

---------

Co-authored-by: Bennet Bo Fenner <[e-mail]>

Change summary

.github/workflows/run_agent_evals.yml                | 2 +-
.github/workflows/run_unit_evals.yml                 | 4 ++--
crates/gpui/src/executor.rs                          | 6 +++++-
script/run-unit-evals                                | 2 +-
tooling/xtask/src/tasks/workflows/run_agent_evals.rs | 3 ++-
tooling/xtask/src/tasks/workflows/vars.rs            | 2 +-
6 files changed, 12 insertions(+), 7 deletions(-)

Detailed changes

.github/workflows/run_agent_evals.yml

@@ -7,7 +7,7 @@ env:
   RUST_BACKTRACE: '1'
   ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
   OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-  GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
+  GOOGLE_AI_API_KEY: ${{ secrets.GOOGLE_AI_API_KEY }}
   GOOGLE_CLOUD_PROJECT: ${{ secrets.GOOGLE_CLOUD_PROJECT }}
   ZED_CLIENT_CHECKSUM_SEED: ${{ secrets.ZED_CLIENT_CHECKSUM_SEED }}
   ZED_EVAL_TELEMETRY: '1'

.github/workflows/run_unit_evals.yml

@@ -1,6 +1,6 @@
-# Generated from xtask::workflows::run_agent_evals
+# Generated from xtask::workflows::run_unit_evals
 # Rebuild with `cargo xtask workflows`.
-name: run_agent_evals
+name: run_unit_evals
 env:
   CARGO_TERM_COLOR: always
   CARGO_INCREMENTAL: '0'

crates/gpui/src/executor.rs

@@ -281,7 +281,11 @@ impl BackgroundExecutor {
         });
         let mut cx = std::task::Context::from_waker(&waker);
 
-        let duration = Duration::from_secs(180);
+        let duration = Duration::from_secs(
+            option_env!("GPUI_TEST_TIMEOUT")
+                .and_then(|s| s.parse::<u64>().ok())
+                .unwrap_or(180),
+        );
         let mut test_should_end_by = Instant::now() + duration;
 
         loop {

script/run-unit-evals

@@ -2,4 +2,4 @@
 
 set -euxo pipefail
 
-cargo nextest run --workspace --no-fail-fast --features unit-eval --no-capture -E 'test(::eval_)'
+GPUI_TEST_TIMEOUT=1500 cargo nextest run --workspace --no-fail-fast --features unit-eval --no-capture -E 'test(::eval_)'

tooling/xtask/src/tasks/workflows/run_agent_evals.rs

@@ -20,7 +20,7 @@ pub(crate) fn run_agent_evals() -> Workflow {
         .add_env(("RUST_BACKTRACE", 1))
         .add_env(("ANTHROPIC_API_KEY", vars::ANTHROPIC_API_KEY))
         .add_env(("OPENAI_API_KEY", vars::OPENAI_API_KEY))
-        .add_env(("GOOGLE_API_KEY", vars::GOOGLE_API_KEY))
+        .add_env(("GOOGLE_AI_API_KEY", vars::GOOGLE_AI_API_KEY))
         .add_env(("GOOGLE_CLOUD_PROJECT", vars::GOOGLE_CLOUD_PROJECT))
         .add_env(("ZED_CLIENT_CHECKSUM_SEED", vars::ZED_CLIENT_CHECKSUM_SEED))
         .add_env(("ZED_EVAL_TELEMETRY", 1))
@@ -53,6 +53,7 @@ pub(crate) fn run_unit_evals() -> Workflow {
     let unit_evals = unit_evals();
 
     named::workflow()
+        .name("run_unit_evals")
         .on(Event::default()
             .schedule([
                 // GitHub might drop jobs at busy times, so we choose a random time in the middle of the night.

tooling/xtask/src/tasks/workflows/vars.rs

@@ -18,7 +18,7 @@ macro_rules! var {
 
 secret!(ANTHROPIC_API_KEY);
 secret!(OPENAI_API_KEY);
-secret!(GOOGLE_API_KEY);
+secret!(GOOGLE_AI_API_KEY);
 secret!(GOOGLE_CLOUD_PROJECT);
 secret!(APPLE_NOTARIZATION_ISSUER_ID);
 secret!(APPLE_NOTARIZATION_KEY);