From ccb2674a77a73169c9d5d41d4ef993dc5ef5b2cc Mon Sep 17 00:00:00 2001 From: Finn Evers Date: Fri, 13 Mar 2026 18:17:29 +0100 Subject: [PATCH] extension_ci: Add infrastructure for this repository (#51493) This will allow us to also use the workflows for this repository, which will especially come in handy once we revisit provider extensions. Not perfect, as we will trigger some failed workflows for extensions that were just added Release Notes: - N/A --- .github/workflows/extension_auto_bump.yml | 72 +++++++++++ .github/workflows/extension_bump.yml | 2 +- .github/workflows/extension_tests.yml | 6 +- .github/workflows/run_tests.yml | 28 ++++- Cargo.lock | 55 +++++++-- Cargo.toml | 2 +- extensions/html/languages/html/brackets.scm | 4 +- tooling/xtask/src/tasks/workflows.rs | 2 + .../tasks/workflows/extension_auto_bump.rs | 113 ++++++++++++++++++ .../src/tasks/workflows/extension_bump.rs | 5 +- .../src/tasks/workflows/extension_tests.rs | 8 +- .../xtask/src/tasks/workflows/run_tests.rs | 108 +++++++++++++---- 12 files changed, 358 insertions(+), 47 deletions(-) create mode 100644 .github/workflows/extension_auto_bump.yml create mode 100644 tooling/xtask/src/tasks/workflows/extension_auto_bump.rs diff --git a/.github/workflows/extension_auto_bump.yml b/.github/workflows/extension_auto_bump.yml new file mode 100644 index 0000000000000000000000000000000000000000..215cdbe5eec30b1e9212616bcd1e1d89ecf9e564 --- /dev/null +++ b/.github/workflows/extension_auto_bump.yml @@ -0,0 +1,72 @@ +# Generated from xtask::workflows::extension_auto_bump +# Rebuild with `cargo xtask workflows`. +name: extension_auto_bump +on: + push: + branches: + - main + paths: + - extensions/** + - '!extensions/workflows/**' + - '!extensions/*.md' +jobs: + detect_changed_extensions: + if: (github.repository_owner == 'zed-industries' || github.repository_owner == 'zed-extensions') + runs-on: namespace-profile-2x4-ubuntu-2404 + steps: + - name: steps::checkout_repo + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + clean: false + fetch-depth: 2 + - id: detect + name: extension_auto_bump::detect_changed_extensions + run: | + COMPARE_REV="$(git rev-parse HEAD~1)" + CHANGED_FILES="$(git diff --name-only "$COMPARE_REV" "$GITHUB_SHA")" + # Detect changed extension directories (excluding extensions/workflows) + CHANGED_EXTENSIONS=$(echo "$CHANGED_FILES" | grep -oP '^extensions/[^/]+(?=/)' | sort -u | grep -v '^extensions/workflows$' || true) + if [ -n "$CHANGED_EXTENSIONS" ]; then + EXTENSIONS_JSON=$(echo "$CHANGED_EXTENSIONS" | jq -R -s -c 'split("\n") | map(select(length > 0))') + else + EXTENSIONS_JSON="[]" + fi + # Filter out newly added or entirely removed extensions + FILTERED="[]" + for ext in $(echo "$EXTENSIONS_JSON" | jq -r '.[]'); do + if git show HEAD~1:"$ext/extension.toml" >/dev/null 2>&1 && \ + [ -f "$ext/extension.toml" ]; then + FILTERED=$(echo "$FILTERED" | jq --arg e "$ext" '. + [$e]') + fi + done + echo "changed_extensions=$FILTERED" >> "$GITHUB_OUTPUT" + outputs: + changed_extensions: ${{ steps.detect.outputs.changed_extensions }} + timeout-minutes: 5 + bump_extension_versions: + needs: + - detect_changed_extensions + if: needs.detect_changed_extensions.outputs.changed_extensions != '[]' + permissions: + actions: write + contents: write + issues: write + pull-requests: write + strategy: + matrix: + extension: ${{ fromJson(needs.detect_changed_extensions.outputs.changed_extensions) }} + fail-fast: false + max-parallel: 1 + uses: ./.github/workflows/extension_bump.yml + secrets: + app-id: ${{ secrets.ZED_ZIPPY_APP_ID }} + app-secret: ${{ secrets.ZED_ZIPPY_APP_PRIVATE_KEY }} + with: + working-directory: ${{ matrix.extension }} + force-bump: false +concurrency: + group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.ref_name == 'main' && github.sha || 'anysha' }} + cancel-in-progress: true +defaults: + run: + shell: bash -euxo pipefail {0} diff --git a/.github/workflows/extension_bump.yml b/.github/workflows/extension_bump.yml index e61e98f4042826858e54c6f5565c5fd62f280553..31f34c9299cee8b464162d501aecaa2bb70035d6 100644 --- a/.github/workflows/extension_bump.yml +++ b/.github/workflows/extension_bump.yml @@ -214,7 +214,7 @@ jobs: shell: bash -euxo pipefail {0} working-directory: ${{ inputs.working-directory }} concurrency: - group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.ref_name == 'main' && github.sha || 'anysha' }} + group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.ref_name == 'main' && github.sha || 'anysha' }}extension-bump cancel-in-progress: true defaults: run: diff --git a/.github/workflows/extension_tests.yml b/.github/workflows/extension_tests.yml index de9b4dc047a039c0f6af063c2a95fdecd70e8cba..89668c028a6d1fa4baddd417687226dd55a52426 100644 --- a/.github/workflows/extension_tests.yml +++ b/.github/workflows/extension_tests.yml @@ -216,12 +216,8 @@ jobs: RESULT_ORCHESTRATE: ${{ needs.orchestrate.result }} RESULT_CHECK_RUST: ${{ needs.check_rust.result }} RESULT_CHECK_EXTENSION: ${{ needs.check_extension.result }} - defaults: - run: - shell: bash -euxo pipefail {0} - working-directory: ${{ inputs.working-directory }} concurrency: - group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.ref_name == 'main' && github.sha || 'anysha' }} + group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.ref_name == 'main' && github.sha || 'anysha' }}extension-tests cancel-in-progress: true defaults: run: diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index b1d8c1fff3c9f48e62f42fab05473d5f38aad2ce..fed05e00459b3c688c4244ddb9ea29ec1dbfd564 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -103,13 +103,22 @@ jobs: check_pattern "run_action_checks" '^\.github/(workflows/|actions/|actionlint.yml)|tooling/xtask|script/' -qP check_pattern "run_docs" '^(docs/|crates/.*\.rs)' -qP check_pattern "run_licenses" '^(Cargo.lock|script/.*licenses)' -qP - check_pattern "run_tests" '^(docs/|script/update_top_ranking_issues/|\.github/(ISSUE_TEMPLATE|workflows/(?!run_tests)))' -qvP + check_pattern "run_tests" '^(docs/|script/update_top_ranking_issues/|\.github/(ISSUE_TEMPLATE|workflows/(?!run_tests))|extensions/)' -qvP + # Detect changed extension directories (excluding extensions/workflows) + CHANGED_EXTENSIONS=$(echo "$CHANGED_FILES" | grep -oP '^extensions/[^/]+(?=/)' | sort -u | grep -v '^extensions/workflows$' || true) + if [ -n "$CHANGED_EXTENSIONS" ]; then + EXTENSIONS_JSON=$(echo "$CHANGED_EXTENSIONS" | jq -R -s -c 'split("\n") | map(select(length > 0))') + else + EXTENSIONS_JSON="[]" + fi + echo "changed_extensions=$EXTENSIONS_JSON" >> "$GITHUB_OUTPUT" outputs: changed_packages: ${{ steps.filter.outputs.changed_packages }} run_action_checks: ${{ steps.filter.outputs.run_action_checks }} run_docs: ${{ steps.filter.outputs.run_docs }} run_licenses: ${{ steps.filter.outputs.run_licenses }} run_tests: ${{ steps.filter.outputs.run_tests }} + changed_extensions: ${{ steps.filter.outputs.changed_extensions }} check_style: if: (github.repository_owner == 'zed-industries' || github.repository_owner == 'zed-extensions') runs-on: namespace-profile-4x8-ubuntu-2204 @@ -711,6 +720,20 @@ jobs: - name: run_tests::check_postgres_and_protobuf_migrations::check_protobuf_formatting run: buf format --diff --exit-code crates/proto/proto timeout-minutes: 60 + extension_tests: + needs: + - orchestrate + if: needs.orchestrate.outputs.changed_extensions != '[]' + permissions: + contents: read + strategy: + matrix: + extension: ${{ fromJson(needs.orchestrate.outputs.changed_extensions) }} + fail-fast: false + max-parallel: 1 + uses: ./.github/workflows/extension_tests.yml + with: + working-directory: ${{ matrix.extension }} tests_pass: needs: - orchestrate @@ -728,6 +751,7 @@ jobs: - check_docs - check_licenses - check_scripts + - extension_tests if: (github.repository_owner == 'zed-industries' || github.repository_owner == 'zed-extensions') && always() runs-on: namespace-profile-2x4-ubuntu-2404 steps: @@ -756,6 +780,7 @@ jobs: check_result "check_docs" "$RESULT_CHECK_DOCS" check_result "check_licenses" "$RESULT_CHECK_LICENSES" check_result "check_scripts" "$RESULT_CHECK_SCRIPTS" + check_result "extension_tests" "$RESULT_EXTENSION_TESTS" exit $EXIT_CODE env: @@ -774,6 +799,7 @@ jobs: RESULT_CHECK_DOCS: ${{ needs.check_docs.result }} RESULT_CHECK_LICENSES: ${{ needs.check_licenses.result }} RESULT_CHECK_SCRIPTS: ${{ needs.check_scripts.result }} + RESULT_EXTENSION_TESTS: ${{ needs.extension_tests.result }} concurrency: group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.ref_name == 'main' && github.sha || 'anysha' }} cancel-in-progress: true diff --git a/Cargo.lock b/Cargo.lock index 4e347d40f3f0e0f23f48770537e7df92d8bd862a..65d7f7ccb5ae148e337257d52f71ac2cc4aeebc0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2193,7 +2193,7 @@ version = "3.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89ec27229c38ed0eb3c0feee3d2c1d6a4379ae44f418a29a658890e062d8f365" dependencies = [ - "darling", + "darling 0.20.11", "ident_case", "prettyplease", "proc-macro2", @@ -2459,7 +2459,7 @@ version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9225bdcf4e4a9a4c08bf16607908eb2fbf746828d5e0b5e019726dbf6571f201" dependencies = [ - "darling", + "darling 0.20.11", "proc-macro2", "quote", "syn 2.0.117", @@ -4513,8 +4513,18 @@ version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" dependencies = [ - "darling_core", - "darling_macro", + "darling_core 0.20.11", + "darling_macro 0.20.11", +] + +[[package]] +name = "darling" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cdf337090841a411e2a7f3deb9187445851f91b309c0c0a29e05f74a00a48c0" +dependencies = [ + "darling_core 0.21.3", + "darling_macro 0.21.3", ] [[package]] @@ -4531,13 +4541,38 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "darling_core" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1247195ecd7e3c85f83c8d2a366e4210d588e802133e1e355180a9870b517ea4" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.117", +] + [[package]] name = "darling_macro" version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ - "darling_core", + "darling_core 0.20.11", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "darling_macro" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" +dependencies = [ + "darling_core 0.21.3", "quote", "syn 2.0.117", ] @@ -4808,11 +4843,11 @@ dependencies = [ [[package]] name = "derive_setters" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae5c625eda104c228c06ecaf988d1c60e542176bd7a490e60eeda3493244c0c9" +checksum = "b7e6f6fa1f03c14ae082120b84b3c7fbd7b8588d924cf2d7c3daf9afd49df8b9" dependencies = [ - "darling", + "darling 0.21.3", "proc-macro2", "quote", "syn 2.0.117", @@ -7143,7 +7178,7 @@ dependencies = [ [[package]] name = "gh-workflow" version = "0.8.0" -source = "git+https://github.com/zed-industries/gh-workflow?rev=c9eac0ed361583e1072860d96776fa52775b82ac#c9eac0ed361583e1072860d96776fa52775b82ac" +source = "git+https://github.com/zed-industries/gh-workflow?rev=37f3c0575d379c218a9c455ee67585184e40d43f#37f3c0575d379c218a9c455ee67585184e40d43f" dependencies = [ "async-trait", "derive_more", @@ -7160,7 +7195,7 @@ dependencies = [ [[package]] name = "gh-workflow-macros" version = "0.8.0" -source = "git+https://github.com/zed-industries/gh-workflow?rev=c9eac0ed361583e1072860d96776fa52775b82ac#c9eac0ed361583e1072860d96776fa52775b82ac" +source = "git+https://github.com/zed-industries/gh-workflow?rev=37f3c0575d379c218a9c455ee67585184e40d43f#37f3c0575d379c218a9c455ee67585184e40d43f" dependencies = [ "heck 0.5.0", "quote", diff --git a/Cargo.toml b/Cargo.toml index 36e7ca8cc7129af0ed7ab29dc5db338cdf33f7d4..754860cc43f5b841e45316a0434b37886e901a0f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -558,7 +558,7 @@ fork = "0.4.0" futures = "0.3" futures-concurrency = "7.7.1" futures-lite = "1.13" -gh-workflow = { git = "https://github.com/zed-industries/gh-workflow", rev = "c9eac0ed361583e1072860d96776fa52775b82ac" } +gh-workflow = { git = "https://github.com/zed-industries/gh-workflow", rev = "37f3c0575d379c218a9c455ee67585184e40d43f" } git2 = { version = "0.20.1", default-features = false, features = ["vendored-libgit2"] } globset = "0.4" handlebars = "4.3" diff --git a/extensions/html/languages/html/brackets.scm b/extensions/html/languages/html/brackets.scm index adc11a1d7408ae33b80f0daa78a03d8f3352b745..02619c109f3ff2d830948e8e8c4889e1e733fae9 100644 --- a/extensions/html/languages/html/brackets.scm +++ b/extensions/html/languages/html/brackets.scm @@ -2,11 +2,11 @@ "/>" @close) (#set! rainbow.exclude)) -(("" @close) (#set! rainbow.exclude)) -(("<" @open +(("" @close) (#set! rainbow.exclude)) diff --git a/tooling/xtask/src/tasks/workflows.rs b/tooling/xtask/src/tasks/workflows.rs index 26596c9401c1d3c500a8c1cb18083d525c934e20..35f053f46666a4d5e81bffe27bc80490c20c166d 100644 --- a/tooling/xtask/src/tasks/workflows.rs +++ b/tooling/xtask/src/tasks/workflows.rs @@ -13,6 +13,7 @@ mod cherry_pick; mod compare_perf; mod danger; mod deploy_collab; +mod extension_auto_bump; mod extension_bump; mod extension_tests; mod extension_workflow_rollout; @@ -199,6 +200,7 @@ pub fn run_workflows(args: GenerateWorkflowArgs) -> Result<()> { WorkflowFile::zed(danger::danger), WorkflowFile::zed(deploy_collab::deploy_collab), WorkflowFile::zed(extension_bump::extension_bump), + WorkflowFile::zed(extension_auto_bump::extension_auto_bump), WorkflowFile::zed(extension_tests::extension_tests), WorkflowFile::zed(extension_workflow_rollout::extension_workflow_rollout), WorkflowFile::zed(publish_extension_cli::publish_extension_cli), diff --git a/tooling/xtask/src/tasks/workflows/extension_auto_bump.rs b/tooling/xtask/src/tasks/workflows/extension_auto_bump.rs new file mode 100644 index 0000000000000000000000000000000000000000..3201fdb1f65233c096738670e48d1b7def1a8975 --- /dev/null +++ b/tooling/xtask/src/tasks/workflows/extension_auto_bump.rs @@ -0,0 +1,113 @@ +use gh_workflow::{ + Event, Expression, Input, Job, Level, Permissions, Push, Strategy, UsesJob, Workflow, +}; +use indoc::indoc; +use serde_json::json; + +use crate::tasks::workflows::{ + extensions::WithAppSecrets, + run_tests::DETECT_CHANGED_EXTENSIONS_SCRIPT, + runners, + steps::{self, CommonJobConditions, NamedJob, named}, + vars::{StepOutput, one_workflow_per_non_main_branch}, +}; + +/// Generates a workflow that triggers on push to main, detects changed extensions +/// in the `extensions/` directory, and invokes the `extension_bump` reusable workflow +/// for each changed extension via a matrix strategy. +pub(crate) fn extension_auto_bump() -> Workflow { + let detect = detect_changed_extensions(); + let bump = bump_extension_versions(&detect); + + named::workflow() + .add_event( + Event::default().push( + Push::default() + .add_branch("main") + .add_path("extensions/**") + .add_path("!extensions/workflows/**") + .add_path("!extensions/*.md"), + ), + ) + .concurrency(one_workflow_per_non_main_branch()) + .add_job(detect.name, detect.job) + .add_job(bump.name, bump.job) +} + +fn detect_changed_extensions() -> NamedJob { + let preamble = indoc! {r#" + COMPARE_REV="$(git rev-parse HEAD~1)" + CHANGED_FILES="$(git diff --name-only "$COMPARE_REV" "$GITHUB_SHA")" + "#}; + + let filter_new_and_removed = indoc! {r#" + # Filter out newly added or entirely removed extensions + FILTERED="[]" + for ext in $(echo "$EXTENSIONS_JSON" | jq -r '.[]'); do + if git show HEAD~1:"$ext/extension.toml" >/dev/null 2>&1 && \ + [ -f "$ext/extension.toml" ]; then + FILTERED=$(echo "$FILTERED" | jq --arg e "$ext" '. + [$e]') + fi + done + echo "changed_extensions=$FILTERED" >> "$GITHUB_OUTPUT" + "#}; + + let script = format!( + "{preamble}{detect}{filter}", + preamble = preamble, + detect = DETECT_CHANGED_EXTENSIONS_SCRIPT, + filter = filter_new_and_removed, + ); + + let step = named::bash(script).id("detect"); + + let output = StepOutput::new(&step, "changed_extensions"); + + let job = Job::default() + .with_repository_owner_guard() + .runs_on(runners::LINUX_SMALL) + .timeout_minutes(5u32) + .add_step(steps::checkout_repo().with_custom_fetch_depth(2)) + .add_step(step) + .outputs([("changed_extensions".to_owned(), output.to_string())]); + + named::job(job) +} + +fn bump_extension_versions(detect_job: &NamedJob) -> NamedJob { + let job = Job::default() + .needs(vec![detect_job.name.clone()]) + .cond(Expression::new(format!( + "needs.{}.outputs.changed_extensions != '[]'", + detect_job.name + ))) + .permissions( + Permissions::default() + .contents(Level::Write) + .issues(Level::Write) + .pull_requests(Level::Write) + .actions(Level::Write), + ) + .strategy( + Strategy::default() + .fail_fast(false) + // TODO: Remove the limit. We currently need this to workaround the concurrency group issue + // where different matrix jobs would be placed in the same concurrency group and thus cancelled. + .max_parallel(1u32) + .matrix(json!({ + "extension": format!( + "${{{{ fromJson(needs.{}.outputs.changed_extensions) }}}}", + detect_job.name + ) + })), + ) + .uses_local(".github/workflows/extension_bump.yml") + .with( + Input::default() + .add("working-directory", "${{ matrix.extension }}") + .add("force-bump", false), + ) + .with_app_secrets(); + + named::job(job) +} diff --git a/tooling/xtask/src/tasks/workflows/extension_bump.rs b/tooling/xtask/src/tasks/workflows/extension_bump.rs index e31800e3ecd4a1039e7a1a191fffa735f64f84f2..91d2e5645f9f5e9fd24dbceaf5e2ad6886e41cb6 100644 --- a/tooling/xtask/src/tasks/workflows/extension_bump.rs +++ b/tooling/xtask/src/tasks/workflows/extension_bump.rs @@ -9,7 +9,8 @@ use crate::tasks::workflows::{ NamedJob, checkout_repo, dependant_job, named, }, vars::{ - JobOutput, StepOutput, WorkflowInput, WorkflowSecret, one_workflow_per_non_main_branch, + JobOutput, StepOutput, WorkflowInput, WorkflowSecret, + one_workflow_per_non_main_branch_and_token, }, }; @@ -70,7 +71,7 @@ pub(crate) fn extension_bump() -> Workflow { ]), ), ) - .concurrency(one_workflow_per_non_main_branch()) + .concurrency(one_workflow_per_non_main_branch_and_token("extension-bump")) .add_env(("CARGO_TERM_COLOR", "always")) .add_env(("RUST_BACKTRACE", 1)) .add_env(("CARGO_INCREMENTAL", 0)) diff --git a/tooling/xtask/src/tasks/workflows/extension_tests.rs b/tooling/xtask/src/tasks/workflows/extension_tests.rs index a50db3f98bf7bec887ea69f841f547ad717976f9..caf57ce130f7d7e9f0018ef20d4cf4892823f4ab 100644 --- a/tooling/xtask/src/tasks/workflows/extension_tests.rs +++ b/tooling/xtask/src/tasks/workflows/extension_tests.rs @@ -9,7 +9,7 @@ use crate::tasks::workflows::{ self, BASH_SHELL, CommonJobConditions, FluentBuilder, NamedJob, cache_rust_dependencies_namespace, named, }, - vars::{PathCondition, StepOutput, WorkflowInput, one_workflow_per_non_main_branch}, + vars::{PathCondition, StepOutput, WorkflowInput, one_workflow_per_non_main_branch_and_token}, }; pub(crate) const ZED_EXTENSION_CLI_SHA: &str = "03d8e9aee95ea6117d75a48bcac2e19241f6e667"; @@ -34,7 +34,7 @@ pub(crate) fn extension_tests() -> Workflow { should_check_extension.guard(check_extension()), ]; - let tests_pass = with_extension_defaults(tests_pass(&jobs)); + let tests_pass = tests_pass(&jobs, &[]); let working_directory = WorkflowInput::string("working-directory", Some(".".to_owned())); @@ -45,7 +45,9 @@ pub(crate) fn extension_tests() -> Workflow { .add_input(working_directory.name, working_directory.call_input()), ), ) - .concurrency(one_workflow_per_non_main_branch()) + .concurrency(one_workflow_per_non_main_branch_and_token( + "extension-tests", + )) .add_env(("CARGO_TERM_COLOR", "always")) .add_env(("RUST_BACKTRACE", 1)) .add_env(("CARGO_INCREMENTAL", 0)) diff --git a/tooling/xtask/src/tasks/workflows/run_tests.rs b/tooling/xtask/src/tasks/workflows/run_tests.rs index f134fa166d6dfe2ef00e47516e33d658a71badd9..3ca8e456346dc5b1bbea89ca40993456e4f1354c 100644 --- a/tooling/xtask/src/tasks/workflows/run_tests.rs +++ b/tooling/xtask/src/tasks/workflows/run_tests.rs @@ -1,9 +1,10 @@ use gh_workflow::{ - Concurrency, Container, Event, Expression, Job, Port, PullRequest, Push, Run, Step, Use, - Workflow, + Concurrency, Container, Event, Expression, Input, Job, Level, Permissions, Port, PullRequest, + Push, Run, Step, Strategy, Use, UsesJob, Workflow, }; use indexmap::IndexMap; use indoc::formatdoc; +use serde_json::json; use crate::tasks::workflows::{ steps::{ @@ -24,9 +25,10 @@ pub(crate) fn run_tests() -> Workflow { // - script/update_top_ranking_issues/ // - .github/ISSUE_TEMPLATE/ // - .github/workflows/ (except .github/workflows/ci.yml) + // - extensions/ (these have their own test workflow) let should_run_tests = PathCondition::inverted( "run_tests", - r"^(docs/|script/update_top_ranking_issues/|\.github/(ISSUE_TEMPLATE|workflows/(?!run_tests)))", + r"^(docs/|script/update_top_ranking_issues/|\.github/(ISSUE_TEMPLATE|workflows/(?!run_tests))|extensions/)", ); let should_check_docs = PathCondition::new("run_docs", r"^(docs/|crates/.*\.rs)"); let should_check_scripts = PathCondition::new( @@ -60,7 +62,8 @@ pub(crate) fn run_tests() -> Workflow { should_check_licences.guard(check_licenses()), should_check_scripts.guard(check_scripts()), ]; - let tests_pass = tests_pass(&jobs); + let ext_tests = extension_tests(); + let tests_pass = tests_pass(&jobs, &[&ext_tests.name]); jobs.push(should_run_tests.guard(check_postgres_and_protobuf_migrations())); // could be more specific here? @@ -91,24 +94,32 @@ pub(crate) fn run_tests() -> Workflow { } workflow }) + .add_job(ext_tests.name, ext_tests.job) .add_job(tests_pass.name, tests_pass.job) } +/// Controls which features `orchestrate_impl` includes in the generated script. +#[derive(PartialEq, Eq)] +enum OrchestrateTarget { + /// For the main Zed repo: includes the cargo package filter and extension + /// change detection, but no working-directory scoping. + ZedRepo, + /// For individual extension repos: scopes changed-file detection to the + /// working directory, with no package filter or extension detection. + Extension, +} + // Generates a bash script that checks changed files against regex patterns // and sets GitHub output variables accordingly pub fn orchestrate(rules: &[&PathCondition]) -> NamedJob { - orchestrate_impl(rules, true, false) + orchestrate_impl(rules, OrchestrateTarget::ZedRepo) } pub fn orchestrate_for_extension(rules: &[&PathCondition]) -> NamedJob { - orchestrate_impl(rules, false, true) + orchestrate_impl(rules, OrchestrateTarget::Extension) } -fn orchestrate_impl( - rules: &[&PathCondition], - include_package_filter: bool, - filter_by_working_directory: bool, -) -> NamedJob { +fn orchestrate_impl(rules: &[&PathCondition], target: OrchestrateTarget) -> NamedJob { let name = "orchestrate".to_owned(); let step_name = "filter".to_owned(); let mut script = String::new(); @@ -127,7 +138,7 @@ fn orchestrate_impl( "#}); - if filter_by_working_directory { + if target == OrchestrateTarget::Extension { script.push_str(indoc::indoc! {r#" # When running from a subdirectory, git diff returns repo-root-relative paths. # Filter to only files within the current working directory and strip the prefix. @@ -155,7 +166,7 @@ fn orchestrate_impl( let mut outputs = IndexMap::new(); - if include_package_filter { + if target == OrchestrateTarget::ZedRepo { script.push_str(indoc::indoc! {r#" # Check for changes that require full rebuild (no filter) # Direct pushes to main/stable/preview always run full suite @@ -241,6 +252,16 @@ fn orchestrate_impl( )); } + if target == OrchestrateTarget::ZedRepo { + script.push_str(DETECT_CHANGED_EXTENSIONS_SCRIPT); + script.push_str("echo \"changed_extensions=$EXTENSIONS_JSON\" >> \"$GITHUB_OUTPUT\"\n"); + + outputs.insert( + "changed_extensions".to_owned(), + format!("${{{{ steps.{}.outputs.changed_extensions }}}}", step_name), + ); + } + let job = Job::default() .runs_on(runners::LINUX_SMALL) .with_repository_owner_guard() @@ -251,7 +272,7 @@ fn orchestrate_impl( NamedJob { name, job } } -pub fn tests_pass(jobs: &[NamedJob]) -> NamedJob { +pub fn tests_pass(jobs: &[NamedJob], extra_job_names: &[&str]) -> NamedJob { let mut script = String::from(indoc::indoc! {r#" set +x EXIT_CODE=0 @@ -263,20 +284,26 @@ pub fn tests_pass(jobs: &[NamedJob]) -> NamedJob { "#}); - let env_entries: Vec<_> = jobs + let all_names: Vec<&str> = jobs + .iter() + .map(|job| job.name.as_str()) + .chain(extra_job_names.iter().copied()) + .collect(); + + let env_entries: Vec<_> = all_names .iter() - .map(|job| { - let env_name = format!("RESULT_{}", job.name.to_uppercase()); - let env_value = format!("${{{{ needs.{}.result }}}}", job.name); + .map(|name| { + let env_name = format!("RESULT_{}", name.to_uppercase()); + let env_value = format!("${{{{ needs.{}.result }}}}", name); (env_name, env_value) }) .collect(); script.push_str( - &jobs + &all_names .iter() .zip(env_entries.iter()) - .map(|(job, (env_name, _))| format!("check_result \"{}\" \"${}\"", job.name, env_name)) + .map(|(name, (env_name, _))| format!("check_result \"{}\" \"${}\"", name, env_name)) .collect::>() .join("\n"), ); @@ -286,8 +313,9 @@ pub fn tests_pass(jobs: &[NamedJob]) -> NamedJob { let job = Job::default() .runs_on(runners::LINUX_SMALL) .needs( - jobs.iter() - .map(|j| j.name.to_string()) + all_names + .iter() + .map(|name| name.to_string()) .collect::>(), ) .cond(repository_owner_guard_expression(true)) @@ -302,6 +330,19 @@ pub fn tests_pass(jobs: &[NamedJob]) -> NamedJob { named::job(job) } +/// Bash script snippet that detects changed extension directories from `$CHANGED_FILES`. +/// Assumes `$CHANGED_FILES` is already set. Sets `$EXTENSIONS_JSON` to a JSON array of +/// changed extension paths. Callers are responsible for writing the result to `$GITHUB_OUTPUT`. +pub(crate) const DETECT_CHANGED_EXTENSIONS_SCRIPT: &str = indoc::indoc! {r#" + # Detect changed extension directories (excluding extensions/workflows) + CHANGED_EXTENSIONS=$(echo "$CHANGED_FILES" | grep -oP '^extensions/[^/]+(?=/)' | sort -u | grep -v '^extensions/workflows$' || true) + if [ -n "$CHANGED_EXTENSIONS" ]; then + EXTENSIONS_JSON=$(echo "$CHANGED_EXTENSIONS" | jq -R -s -c 'split("\n") | map(select(length > 0))') + else + EXTENSIONS_JSON="[]" + fi +"#}; + const TS_QUERY_LS_FILE: &str = "ts_query_ls-x86_64-unknown-linux-gnu.tar.gz"; const CI_TS_QUERY_RELEASE: &str = "tags/v3.15.1"; @@ -712,3 +753,26 @@ pub(crate) fn check_scripts() -> NamedJob { .add_step(check_xtask_workflows()), ) } + +fn extension_tests() -> NamedJob { + let job = Job::default() + .needs(vec!["orchestrate".to_owned()]) + .cond(Expression::new( + "needs.orchestrate.outputs.changed_extensions != '[]'", + )) + .permissions(Permissions::default().contents(Level::Read)) + .strategy( + Strategy::default() + .fail_fast(false) + // TODO: Remove the limit. We currently need this to workaround the concurrency group issue + // where different matrix jobs would be placed in the same concurrency group and thus cancelled. + .max_parallel(1u32) + .matrix(json!({ + "extension": "${{ fromJson(needs.orchestrate.outputs.changed_extensions) }}" + })), + ) + .uses_local(".github/workflows/extension_tests.yml") + .with(Input::default().add("working-directory", "${{ matrix.extension }}")); + + named::job(job) +}