1name: Run Unit Evals
 2
 3on:
 4  schedule:
 5    # GitHub might drop jobs at busy times, so we choose a random time in the middle of the night.
 6    - cron: "47 1 * * 2"
 7  workflow_dispatch:
 8
 9concurrency:
10  # Allow only one workflow per any non-`main` branch.
11  group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.ref_name == 'main' && github.sha || 'anysha' }}
12  cancel-in-progress: true
13
14env:
15  CARGO_TERM_COLOR: always
16  CARGO_INCREMENTAL: 0
17  RUST_BACKTRACE: 1
18  ZED_CLIENT_CHECKSUM_SEED: ${{ secrets.ZED_CLIENT_CHECKSUM_SEED }}
19
20jobs:
21  unit_evals:
22    if: github.repository_owner == 'zed-industries'
23    timeout-minutes: 60
24    name: Run unit evals
25    runs-on:
26      - namespace-profile-16x32-ubuntu-2204
27    steps:
28      - name: Add Rust to the PATH
29        run: echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
30
31      - name: Checkout repo
32        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
33        with:
34          clean: false
35
36      - name: Cache dependencies
37        uses: swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2
38        with:
39          save-if: ${{ github.ref == 'refs/heads/main' }}
40          # cache-provider: "buildjet"
41
42      - name: Install Linux dependencies
43        run: ./script/linux
44
45      - name: Configure CI
46        run: |
47          mkdir -p ./../.cargo
48          cp ./.cargo/ci-config.toml ./../.cargo/config.toml
49
50      - name: Install Rust
51        shell: bash -euxo pipefail {0}
52        run: |
53          cargo install cargo-nextest --locked
54
55      - name: Install Node
56        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
57        with:
58          node-version: "18"
59
60      - name: Limit target directory size
61        shell: bash -euxo pipefail {0}
62        run: script/clear-target-dir-if-larger-than 100
63
64      - name: Run unit evals
65        shell: bash -euxo pipefail {0}
66        run: cargo nextest run --workspace --no-fail-fast --features eval --no-capture -E 'test(::eval_)'
67        env:
68          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
69
70      - name: Send failure message to Slack channel if needed
71        if: ${{ failure() }}
72        uses: slackapi/slack-github-action@b0fa283ad8fea605de13dc3f449259339835fc52
73        with:
74          method: chat.postMessage
75          token: ${{ secrets.SLACK_APP_ZED_UNIT_EVALS_BOT_TOKEN }}
76          payload: |
77            channel: C04UDRNNJFQ
78            text: "Unit Evals Failed: https://github.com/zed-industries/zed/actions/runs/${{ github.run_id }}"
79
80      # Even the Linux runner is not stateful, in theory there is no need to do this cleanup.
81      # But, to avoid potential issues in the future if we choose to use a stateful Linux runner and forget to add code
82      # to clean up the config file, I’ve included the cleanup code here as a precaution.
83      # While it’s not strictly necessary at this moment, I believe it’s better to err on the side of caution.
84      - name: Clean CI config file
85        if: always()
86        run: rm -rf ./../.cargo