1use gh_workflow::{
2 Concurrency, Event, Expression, Job, PullRequest, Push, Run, Step, Use, Workflow,
3};
4use indexmap::IndexMap;
5
6use crate::tasks::workflows::{
7 nix_build::build_nix,
8 runners::Arch,
9 steps::{BASH_SHELL, CommonJobConditions, repository_owner_guard_expression},
10 vars::{self, PathCondition},
11};
12
13use super::{
14 runners::{self, Platform},
15 steps::{self, FluentBuilder, NamedJob, named, release_job},
16};
17
18pub(crate) fn run_tests() -> Workflow {
19 // Specify anything which should potentially skip full test suite in this regex:
20 // - docs/
21 // - script/update_top_ranking_issues/
22 // - .github/ISSUE_TEMPLATE/
23 // - .github/workflows/ (except .github/workflows/ci.yml)
24 let should_run_tests = PathCondition::inverted(
25 "run_tests",
26 r"^(docs/|script/update_top_ranking_issues/|\.github/(ISSUE_TEMPLATE|workflows/(?!run_tests)))",
27 );
28 let should_check_docs = PathCondition::new("run_docs", r"^(docs/|crates/.*\.rs)");
29 let should_check_scripts = PathCondition::new(
30 "run_action_checks",
31 r"^\.github/(workflows/|actions/|actionlint.yml)|tooling/xtask|script/",
32 );
33 let should_check_licences =
34 PathCondition::new("run_licenses", r"^(Cargo.lock|script/.*licenses)");
35 let should_build_nix = PathCondition::new(
36 "run_nix",
37 r"^(nix/|flake\.|Cargo\.|rust-toolchain.toml|\.cargo/config.toml)",
38 );
39
40 let orchestrate = orchestrate(&[
41 &should_check_scripts,
42 &should_check_docs,
43 &should_check_licences,
44 &should_build_nix,
45 &should_run_tests,
46 ]);
47
48 let check_style = check_style();
49 let run_tests_linux = run_platform_tests(Platform::Linux);
50 let call_autofix = call_autofix(&check_style, &run_tests_linux);
51
52 let mut jobs = vec![
53 orchestrate,
54 check_style,
55 should_run_tests.guard(run_platform_tests(Platform::Windows)),
56 should_run_tests.guard(run_tests_linux),
57 should_run_tests.guard(run_platform_tests(Platform::Mac)),
58 should_run_tests.guard(doctests()),
59 should_run_tests.guard(check_workspace_binaries()),
60 should_run_tests.guard(check_dependencies()), // could be more specific here?
61 should_check_docs.guard(check_docs()),
62 should_check_licences.guard(check_licenses()),
63 should_check_scripts.guard(check_scripts()),
64 should_build_nix.guard(build_nix(
65 Platform::Linux,
66 Arch::X86_64,
67 "debug",
68 // *don't* cache the built output
69 Some("-zed-editor-[0-9.]*-nightly"),
70 &[],
71 )),
72 should_build_nix.guard(build_nix(
73 Platform::Mac,
74 Arch::AARCH64,
75 "debug",
76 // *don't* cache the built output
77 Some("-zed-editor-[0-9.]*-nightly"),
78 &[],
79 )),
80 ];
81 let tests_pass = tests_pass(&jobs);
82
83 jobs.push(should_run_tests.guard(check_postgres_and_protobuf_migrations())); // could be more specific here?
84
85 named::workflow()
86 .add_event(
87 Event::default()
88 .push(
89 Push::default()
90 .add_branch("main")
91 .add_branch("v[0-9]+.[0-9]+.x"),
92 )
93 .pull_request(PullRequest::default().add_branch("**")),
94 )
95 .concurrency(
96 Concurrency::default()
97 .group(concat!(
98 "${{ github.workflow }}-${{ github.ref_name }}-",
99 "${{ github.ref_name == 'main' && github.sha || 'anysha' }}"
100 ))
101 .cancel_in_progress(true),
102 )
103 .add_env(("CARGO_TERM_COLOR", "always"))
104 .add_env(("RUST_BACKTRACE", 1))
105 .add_env(("CARGO_INCREMENTAL", 0))
106 .map(|mut workflow| {
107 for job in jobs {
108 workflow = workflow.add_job(job.name, job.job)
109 }
110 workflow
111 })
112 .add_job(tests_pass.name, tests_pass.job)
113 .add_job(call_autofix.name, call_autofix.job)
114}
115
116// Generates a bash script that checks changed files against regex patterns
117// and sets GitHub output variables accordingly
118pub fn orchestrate(rules: &[&PathCondition]) -> NamedJob {
119 let name = "orchestrate".to_owned();
120 let step_name = "filter".to_owned();
121 let mut script = String::new();
122
123 script.push_str(indoc::indoc! {r#"
124 if [ -z "$GITHUB_BASE_REF" ]; then
125 echo "Not in a PR context (i.e., push to main/stable/preview)"
126 COMPARE_REV="$(git rev-parse HEAD~1)"
127 else
128 echo "In a PR context comparing to pull_request.base.ref"
129 git fetch origin "$GITHUB_BASE_REF" --depth=350
130 COMPARE_REV="$(git merge-base "origin/${GITHUB_BASE_REF}" HEAD)"
131 fi
132 CHANGED_FILES="$(git diff --name-only "$COMPARE_REV" ${{ github.sha }})"
133
134 check_pattern() {
135 local output_name="$1"
136 local pattern="$2"
137 local grep_arg="$3"
138
139 echo "$CHANGED_FILES" | grep "$grep_arg" "$pattern" && \
140 echo "${output_name}=true" >> "$GITHUB_OUTPUT" || \
141 echo "${output_name}=false" >> "$GITHUB_OUTPUT"
142 }
143
144 "#});
145
146 let mut outputs = IndexMap::new();
147
148 for rule in rules {
149 assert!(
150 rule.set_by_step
151 .borrow_mut()
152 .replace(name.clone())
153 .is_none()
154 );
155 assert!(
156 outputs
157 .insert(
158 rule.name.to_owned(),
159 format!("${{{{ steps.{}.outputs.{} }}}}", step_name, rule.name)
160 )
161 .is_none()
162 );
163
164 let grep_arg = if rule.invert { "-qvP" } else { "-qP" };
165 script.push_str(&format!(
166 "check_pattern \"{}\" '{}' {}\n",
167 rule.name, rule.pattern, grep_arg
168 ));
169 }
170
171 let job = Job::default()
172 .runs_on(runners::LINUX_SMALL)
173 .with_repository_owner_guard()
174 .outputs(outputs)
175 .add_step(steps::checkout_repo().add_with((
176 "fetch-depth",
177 "${{ github.ref == 'refs/heads/main' && 2 || 350 }}",
178 )))
179 .add_step(
180 Step::new(step_name.clone())
181 .run(script)
182 .id(step_name)
183 .shell(BASH_SHELL),
184 );
185
186 NamedJob { name, job }
187}
188
189pub fn tests_pass(jobs: &[NamedJob]) -> NamedJob {
190 let mut script = String::from(indoc::indoc! {r#"
191 set +x
192 EXIT_CODE=0
193
194 check_result() {
195 echo "* $1: $2"
196 if [[ "$2" != "skipped" && "$2" != "success" ]]; then EXIT_CODE=1; fi
197 }
198
199 "#});
200
201 script.push_str(
202 &jobs
203 .iter()
204 .map(|job| {
205 format!(
206 "check_result \"{}\" \"${{{{ needs.{}.result }}}}\"",
207 job.name, job.name
208 )
209 })
210 .collect::<Vec<_>>()
211 .join("\n"),
212 );
213
214 script.push_str("\n\nexit $EXIT_CODE\n");
215
216 let job = Job::default()
217 .runs_on(runners::LINUX_SMALL)
218 .needs(
219 jobs.iter()
220 .map(|j| j.name.to_string())
221 .collect::<Vec<String>>(),
222 )
223 .cond(repository_owner_guard_expression(true))
224 .add_step(named::bash(&script));
225
226 named::job(job)
227}
228
229pub const STYLE_FAILED_OUTPUT: &str = "style_failed";
230
231fn check_style() -> NamedJob {
232 fn check_for_typos() -> Step<Use> {
233 named::uses(
234 "crate-ci",
235 "typos",
236 "2d0ce569feab1f8752f1dde43cc2f2aa53236e06",
237 ) // v1.40.0
238 .with(("config", "./typos.toml"))
239 }
240 named::job(
241 release_job(&[])
242 .runs_on(runners::LINUX_MEDIUM)
243 .add_step(steps::checkout_repo())
244 .add_step(steps::cache_rust_dependencies_namespace())
245 .add_step(steps::setup_pnpm())
246 .add_step(steps::prettier())
247 .add_step(steps::cargo_fmt())
248 .add_step(steps::record_style_failure())
249 .add_step(steps::script("./script/check-todos"))
250 .add_step(steps::script("./script/check-keymaps"))
251 .add_step(check_for_typos())
252 .outputs([(
253 STYLE_FAILED_OUTPUT.to_owned(),
254 format!(
255 "${{{{ steps.{}.outputs.failed == 'true' }}}}",
256 steps::RECORD_STYLE_FAILURE_STEP_ID
257 ),
258 )]),
259 )
260}
261
262fn call_autofix(check_style: &NamedJob, run_tests_linux: &NamedJob) -> NamedJob {
263 fn dispatch_autofix(run_tests_linux_name: &str) -> Step<Run> {
264 let clippy_failed_expr = format!(
265 "needs.{}.outputs.{} == 'true'",
266 run_tests_linux_name, CLIPPY_FAILED_OUTPUT
267 );
268 named::bash(format!(
269 "gh workflow run autofix_pr.yml -f pr_number=${{{{ github.event.pull_request.number }}}} -f run_clippy=${{{{ {} }}}}",
270 clippy_failed_expr
271 ))
272 .add_env(("GITHUB_TOKEN", "${{ steps.get-app-token.outputs.token }}"))
273 }
274
275 let style_failed_expr = format!(
276 "needs.{}.outputs.{} == 'true'",
277 check_style.name, STYLE_FAILED_OUTPUT
278 );
279 let clippy_failed_expr = format!(
280 "needs.{}.outputs.{} == 'true'",
281 run_tests_linux.name, CLIPPY_FAILED_OUTPUT
282 );
283 let (authenticate, _token) = steps::authenticate_as_zippy();
284
285 let job = Job::default()
286 .runs_on(runners::LINUX_SMALL)
287 .cond(Expression::new(format!(
288 "always() && ({} || {}) && github.event_name == 'pull_request' && github.actor != 'zed-zippy[bot]'",
289 style_failed_expr, clippy_failed_expr
290 )))
291 .needs(vec![check_style.name.clone(), run_tests_linux.name.clone()])
292 .add_step(authenticate)
293 .add_step(dispatch_autofix(&run_tests_linux.name));
294
295 named::job(job)
296}
297
298fn check_dependencies() -> NamedJob {
299 fn install_cargo_machete() -> Step<Use> {
300 named::uses(
301 "clechasseur",
302 "rs-cargo",
303 "8435b10f6e71c2e3d4d3b7573003a8ce4bfc6386", // v2
304 )
305 .add_with(("command", "install"))
306 .add_with(("args", "cargo-machete@0.7.0"))
307 }
308
309 fn run_cargo_machete() -> Step<Use> {
310 named::uses(
311 "clechasseur",
312 "rs-cargo",
313 "8435b10f6e71c2e3d4d3b7573003a8ce4bfc6386", // v2
314 )
315 .add_with(("command", "machete"))
316 }
317
318 fn check_cargo_lock() -> Step<Run> {
319 named::bash("cargo update --locked --workspace")
320 }
321
322 fn check_vulnerable_dependencies() -> Step<Use> {
323 named::uses(
324 "actions",
325 "dependency-review-action",
326 "67d4f4bd7a9b17a0db54d2a7519187c65e339de8", // v4
327 )
328 .if_condition(Expression::new("github.event_name == 'pull_request'"))
329 .with(("license-check", false))
330 }
331
332 named::job(
333 release_job(&[])
334 .runs_on(runners::LINUX_SMALL)
335 .add_step(steps::checkout_repo())
336 .add_step(steps::cache_rust_dependencies_namespace())
337 .add_step(install_cargo_machete())
338 .add_step(run_cargo_machete())
339 .add_step(check_cargo_lock())
340 .add_step(check_vulnerable_dependencies()),
341 )
342}
343
344fn check_workspace_binaries() -> NamedJob {
345 named::job(
346 release_job(&[])
347 .runs_on(runners::LINUX_LARGE)
348 .add_step(steps::checkout_repo())
349 .add_step(steps::setup_cargo_config(Platform::Linux))
350 .add_step(steps::cache_rust_dependencies_namespace())
351 .map(steps::install_linux_dependencies)
352 .add_step(steps::script("cargo build -p collab"))
353 .add_step(steps::script("cargo build --workspace --bins --examples"))
354 .add_step(steps::cleanup_cargo_config(Platform::Linux)),
355 )
356}
357
358pub const CLIPPY_FAILED_OUTPUT: &str = "clippy_failed";
359
360pub(crate) fn run_platform_tests(platform: Platform) -> NamedJob {
361 let runner = match platform {
362 Platform::Windows => runners::WINDOWS_DEFAULT,
363 Platform::Linux => runners::LINUX_DEFAULT,
364 Platform::Mac => runners::MAC_DEFAULT,
365 };
366 NamedJob {
367 name: format!("run_tests_{platform}"),
368 job: release_job(&[])
369 .runs_on(runner)
370 .add_step(steps::checkout_repo())
371 .add_step(steps::setup_cargo_config(platform))
372 .when(platform == Platform::Linux, |this| {
373 this.add_step(steps::cache_rust_dependencies_namespace())
374 })
375 .when(
376 platform == Platform::Linux,
377 steps::install_linux_dependencies,
378 )
379 .add_step(steps::setup_node())
380 .add_step(steps::clippy(platform))
381 .when(platform == Platform::Linux, |job| {
382 job.add_step(steps::record_clippy_failure())
383 })
384 .when(platform == Platform::Linux, |job| {
385 job.add_step(steps::cargo_install_nextest())
386 })
387 .add_step(steps::clear_target_dir_if_large(platform))
388 .add_step(steps::cargo_nextest(platform))
389 .add_step(steps::cleanup_cargo_config(platform))
390 .when(platform == Platform::Linux, |job| {
391 job.outputs([(
392 CLIPPY_FAILED_OUTPUT.to_owned(),
393 format!(
394 "${{{{ steps.{}.outputs.failed == 'true' }}}}",
395 steps::RECORD_CLIPPY_FAILURE_STEP_ID
396 ),
397 )])
398 }),
399 }
400}
401
402pub(crate) fn check_postgres_and_protobuf_migrations() -> NamedJob {
403 fn remove_untracked_files() -> Step<Run> {
404 named::bash("git clean -df")
405 }
406
407 fn ensure_fresh_merge() -> Step<Run> {
408 named::bash(indoc::indoc! {r#"
409 if [ -z "$GITHUB_BASE_REF" ];
410 then
411 echo "BUF_BASE_BRANCH=$(git merge-base origin/main HEAD)" >> "$GITHUB_ENV"
412 else
413 git checkout -B temp
414 git merge -q "origin/$GITHUB_BASE_REF" -m "merge main into temp"
415 echo "BUF_BASE_BRANCH=$GITHUB_BASE_REF" >> "$GITHUB_ENV"
416 fi
417 "#})
418 }
419
420 fn bufbuild_setup_action() -> Step<Use> {
421 named::uses("bufbuild", "buf-setup-action", "v1")
422 .add_with(("version", "v1.29.0"))
423 .add_with(("github_token", vars::GITHUB_TOKEN))
424 }
425
426 fn bufbuild_breaking_action() -> Step<Use> {
427 named::uses("bufbuild", "buf-breaking-action", "v1").add_with(("input", "crates/proto/proto/"))
428 .add_with(("against", "https://github.com/${GITHUB_REPOSITORY}.git#branch=${BUF_BASE_BRANCH},subdir=crates/proto/proto/"))
429 }
430
431 named::job(
432 release_job(&[])
433 .runs_on(runners::LINUX_DEFAULT)
434 .add_env(("GIT_AUTHOR_NAME", "Protobuf Action"))
435 .add_env(("GIT_AUTHOR_EMAIL", "ci@zed.dev"))
436 .add_env(("GIT_COMMITTER_NAME", "Protobuf Action"))
437 .add_env(("GIT_COMMITTER_EMAIL", "ci@zed.dev"))
438 .add_step(steps::checkout_repo().with(("fetch-depth", 0))) // fetch full history
439 .add_step(remove_untracked_files())
440 .add_step(ensure_fresh_merge())
441 .add_step(bufbuild_setup_action())
442 .add_step(bufbuild_breaking_action()),
443 )
444}
445
446fn doctests() -> NamedJob {
447 fn run_doctests() -> Step<Run> {
448 named::bash(indoc::indoc! {r#"
449 cargo test --workspace --doc --no-fail-fast
450 "#})
451 .id("run_doctests")
452 }
453
454 named::job(
455 release_job(&[])
456 .runs_on(runners::LINUX_DEFAULT)
457 .add_step(steps::checkout_repo())
458 .add_step(steps::cache_rust_dependencies_namespace())
459 .map(steps::install_linux_dependencies)
460 .add_step(steps::setup_cargo_config(Platform::Linux))
461 .add_step(run_doctests())
462 .add_step(steps::cleanup_cargo_config(Platform::Linux)),
463 )
464}
465
466fn check_licenses() -> NamedJob {
467 named::job(
468 Job::default()
469 .runs_on(runners::LINUX_SMALL)
470 .add_step(steps::checkout_repo())
471 .add_step(steps::cache_rust_dependencies_namespace())
472 .add_step(steps::script("./script/check-licenses"))
473 .add_step(steps::script("./script/generate-licenses")),
474 )
475}
476
477fn check_docs() -> NamedJob {
478 fn lychee_link_check(dir: &str) -> Step<Use> {
479 named::uses(
480 "lycheeverse",
481 "lychee-action",
482 "82202e5e9c2f4ef1a55a3d02563e1cb6041e5332",
483 ) // v2.4.1
484 .add_with(("args", format!("--no-progress --exclude '^http' '{dir}'")))
485 .add_with(("fail", true))
486 .add_with(("jobSummary", false))
487 }
488
489 fn install_mdbook() -> Step<Use> {
490 named::uses(
491 "peaceiris",
492 "actions-mdbook",
493 "ee69d230fe19748b7abf22df32acaa93833fad08", // v2
494 )
495 .with(("mdbook-version", "0.4.37"))
496 }
497
498 fn build_docs() -> Step<Run> {
499 named::bash(indoc::indoc! {r#"
500 mkdir -p target/deploy
501 mdbook build ./docs --dest-dir=../target/deploy/docs/
502 "#})
503 }
504
505 named::job(
506 release_job(&[])
507 .runs_on(runners::LINUX_LARGE)
508 .add_step(steps::checkout_repo())
509 .add_step(steps::setup_cargo_config(Platform::Linux))
510 // todo(ci): un-inline build_docs/action.yml here
511 .add_step(steps::cache_rust_dependencies_namespace())
512 .add_step(
513 lychee_link_check("./docs/src/**/*"), // check markdown links
514 )
515 .map(steps::install_linux_dependencies)
516 .add_step(install_mdbook())
517 .add_step(build_docs())
518 .add_step(
519 lychee_link_check("target/deploy/docs"), // check links in generated html
520 ),
521 )
522}
523
524pub(crate) fn check_scripts() -> NamedJob {
525 fn download_actionlint() -> Step<Run> {
526 named::bash(
527 "bash <(curl https://raw.githubusercontent.com/rhysd/actionlint/main/scripts/download-actionlint.bash)",
528 )
529 }
530
531 fn run_actionlint() -> Step<Run> {
532 named::bash(indoc::indoc! {r#"
533 ${{ steps.get_actionlint.outputs.executable }} -color
534 "#})
535 }
536
537 fn run_shellcheck() -> Step<Run> {
538 named::bash("./script/shellcheck-scripts error")
539 }
540
541 fn check_xtask_workflows() -> Step<Run> {
542 named::bash(indoc::indoc! {r#"
543 cargo xtask workflows
544 if ! git diff --exit-code .github; then
545 echo "Error: .github directory has uncommitted changes after running 'cargo xtask workflows'"
546 echo "Please run 'cargo xtask workflows' locally and commit the changes"
547 exit 1
548 fi
549 "#})
550 }
551
552 named::job(
553 release_job(&[])
554 .runs_on(runners::LINUX_SMALL)
555 .add_step(steps::checkout_repo())
556 .add_step(run_shellcheck())
557 .add_step(download_actionlint().id("get_actionlint"))
558 .add_step(run_actionlint())
559 .add_step(check_xtask_workflows()),
560 )
561}