From 958798d20e6ed6a94de74164f7cdf34dbb199e37 Mon Sep 17 00:00:00 2001 From: Marshall Bowers Date: Mon, 16 Feb 2026 17:40:10 -0500 Subject: [PATCH] ci: Generate the Collab deployment workflow (#49306) This PR updates the Collab deployment workflow to be generated with `cargo xtask workflows`. Release Notes: - N/A --- .github/workflows/deploy_collab.yml | 266 +++++++++--------- tooling/xtask/src/tasks/workflows.rs | 2 + .../src/tasks/workflows/deploy_collab.rs | 171 +++++++++++ tooling/xtask/src/tasks/workflows/vars.rs | 2 + 4 files changed, 311 insertions(+), 130 deletions(-) create mode 100644 tooling/xtask/src/tasks/workflows/deploy_collab.rs diff --git a/.github/workflows/deploy_collab.yml b/.github/workflows/deploy_collab.yml index a3bacd7f1e90c02d0f3ad4f9781c4251c56d42d7..6eb1c20b43e0563fbb011ba83213411e4a1dd4d5 100644 --- a/.github/workflows/deploy_collab.yml +++ b/.github/workflows/deploy_collab.yml @@ -1,146 +1,152 @@ -name: Publish Collab Server Image - +# Generated from xtask::workflows::deploy_collab +# Rebuild with `cargo xtask workflows`. +name: deploy_collab +env: + DOCKER_BUILDKIT: '1' on: push: tags: - - collab-production - - collab-staging - -env: - DOCKER_BUILDKIT: 1 - + - collab-production jobs: style: + if: (github.repository_owner == 'zed-industries' || github.repository_owner == 'zed-extensions') name: Check formatting and Clippy lints - if: github.repository_owner == 'zed-industries' - runs-on: - - namespace-profile-16x32-ubuntu-2204 + runs-on: namespace-profile-16x32-ubuntu-2204 steps: - - name: Checkout repo - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - with: - clean: false - fetch-depth: 0 - - - name: Run style checks - uses: ./.github/actions/check_style - - - name: Run clippy - run: ./script/clippy - + - name: steps::checkout_repo + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + clean: false + fetch-depth: 0 + - name: steps::setup_cargo_config + run: | + mkdir -p ./../.cargo + cp ./.cargo/ci-config.toml ./../.cargo/config.toml + - name: steps::cache_rust_dependencies_namespace + uses: namespacelabs/nscloud-cache-action@v1 + with: + cache: rust + path: ~/.rustup + - name: steps::cargo_fmt + run: cargo fmt --all -- --check + - name: steps::clippy + run: ./script/clippy tests: + needs: + - style name: Run tests - runs-on: - - namespace-profile-16x32-ubuntu-2204 - needs: style + runs-on: namespace-profile-16x32-ubuntu-2204 steps: - - name: Checkout repo - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - with: - clean: false - fetch-depth: 0 - - - name: Install cargo nextest - uses: taiki-e/install-action@nextest - - - name: Limit target directory size - shell: bash -euxo pipefail {0} - run: script/clear-target-dir-if-larger-than 300 - - - name: Run tests - shell: bash -euxo pipefail {0} - run: cargo nextest run --package collab --no-fail-fast - + - name: steps::checkout_repo + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + clean: false + fetch-depth: 0 + - name: steps::setup_cargo_config + run: | + mkdir -p ./../.cargo + cp ./.cargo/ci-config.toml ./../.cargo/config.toml + - name: steps::cache_rust_dependencies_namespace + uses: namespacelabs/nscloud-cache-action@v1 + with: + cache: rust + path: ~/.rustup + - name: steps::cargo_install_nextest + uses: taiki-e/install-action@nextest + - name: steps::clear_target_dir_if_large + run: ./script/clear-target-dir-if-larger-than 250 + - name: deploy_collab::tests::run_collab_tests + run: cargo nextest run --package collab --no-fail-fast + services: + postgres: + image: postgres:15 + env: + POSTGRES_HOST_AUTH_METHOD: trust + ports: + - 5432:5432 + options: --health-cmd pg_isready --health-interval 500ms --health-timeout 5s --health-retries 10 publish: - name: Publish collab server image needs: - - style - - tests - runs-on: - - namespace-profile-16x32-ubuntu-2204 + - style + - tests + name: Publish collab server image + runs-on: namespace-profile-16x32-ubuntu-2204 steps: - - name: Install doctl - uses: digitalocean/action-doctl@v2 - with: - token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }} - - - name: Sign into DigitalOcean docker registry - run: doctl registry login - - - name: Checkout repo - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - with: - clean: false - - - name: Build docker image - run: | - docker build -f Dockerfile-collab \ - --build-arg "GITHUB_SHA=$GITHUB_SHA" \ - --tag "registry.digitalocean.com/zed/collab:$GITHUB_SHA" \ - . - - - name: Publish docker image - run: docker push "registry.digitalocean.com/zed/collab:${GITHUB_SHA}" - - - name: Prune Docker system - run: docker system prune --filter 'until=72h' -f - + - name: deploy_collab::publish::install_doctl + uses: digitalocean/action-doctl@v2 + with: + token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }} + - name: deploy_collab::publish::sign_into_registry + run: doctl registry login + - name: steps::checkout_repo + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + clean: false + - name: deploy_collab::publish::build_docker_image + run: | + docker build -f Dockerfile-collab \ + --build-arg "GITHUB_SHA=$GITHUB_SHA" \ + --tag "registry.digitalocean.com/zed/collab:$GITHUB_SHA" \ + . + - name: deploy_collab::publish::publish_docker_image + run: docker push "registry.digitalocean.com/zed/collab:${GITHUB_SHA}" + - name: deploy_collab::publish::prune_docker_system + run: docker system prune --filter 'until=72h' -f deploy: - name: Deploy new server image needs: - - publish - runs-on: - - namespace-profile-16x32-ubuntu-2204 - + - publish + name: Deploy new server image + runs-on: namespace-profile-16x32-ubuntu-2204 steps: - - name: Checkout repo - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - with: - clean: false - - - name: Install doctl - uses: digitalocean/action-doctl@v2 - with: - token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }} - - - name: Sign into Kubernetes - run: doctl kubernetes cluster kubeconfig save --expiry-seconds 600 ${{ secrets.CLUSTER_NAME }} - - - name: Start rollout - run: | - set -eu - if [[ $GITHUB_REF_NAME = "collab-production" ]]; then - export ZED_KUBE_NAMESPACE=production - export ZED_COLLAB_LOAD_BALANCER_SIZE_UNIT=10 - export ZED_API_LOAD_BALANCER_SIZE_UNIT=2 - elif [[ $GITHUB_REF_NAME = "collab-staging" ]]; then - export ZED_KUBE_NAMESPACE=staging - export ZED_COLLAB_LOAD_BALANCER_SIZE_UNIT=1 - export ZED_API_LOAD_BALANCER_SIZE_UNIT=1 - else - echo "cowardly refusing to deploy from an unknown branch" - exit 1 - fi - - echo "Deploying collab:$GITHUB_SHA to $ZED_KUBE_NAMESPACE" - - source script/lib/deploy-helpers.sh - export_vars_for_environment $ZED_KUBE_NAMESPACE - - ZED_DO_CERTIFICATE_ID="$(doctl compute certificate list --format ID --no-header)" - export ZED_DO_CERTIFICATE_ID - export ZED_IMAGE_ID="registry.digitalocean.com/zed/collab:${GITHUB_SHA}" - - export ZED_SERVICE_NAME=collab - export ZED_LOAD_BALANCER_SIZE_UNIT=$ZED_COLLAB_LOAD_BALANCER_SIZE_UNIT - export DATABASE_MAX_CONNECTIONS=850 - envsubst < crates/collab/k8s/collab.template.yml | kubectl apply -f - - kubectl -n "$ZED_KUBE_NAMESPACE" rollout status deployment/$ZED_SERVICE_NAME --watch - echo "deployed ${ZED_SERVICE_NAME} to ${ZED_KUBE_NAMESPACE}" - - export ZED_SERVICE_NAME=api - export ZED_LOAD_BALANCER_SIZE_UNIT=$ZED_API_LOAD_BALANCER_SIZE_UNIT - export DATABASE_MAX_CONNECTIONS=60 - envsubst < crates/collab/k8s/collab.template.yml | kubectl apply -f - - kubectl -n "$ZED_KUBE_NAMESPACE" rollout status deployment/$ZED_SERVICE_NAME --watch - echo "deployed ${ZED_SERVICE_NAME} to ${ZED_KUBE_NAMESPACE}" + - name: steps::checkout_repo + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + clean: false + - name: deploy_collab::deploy::install_doctl + uses: digitalocean/action-doctl@v2 + with: + token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }} + - name: deploy_collab::deploy::sign_into_kubernetes + run: | + doctl kubernetes cluster kubeconfig save --expiry-seconds 600 ${{ secrets.CLUSTER_NAME }} + - name: deploy_collab::deploy::start_rollout + run: | + set -eu + if [[ $GITHUB_REF_NAME = "collab-production" ]]; then + export ZED_KUBE_NAMESPACE=production + export ZED_COLLAB_LOAD_BALANCER_SIZE_UNIT=10 + export ZED_API_LOAD_BALANCER_SIZE_UNIT=2 + elif [[ $GITHUB_REF_NAME = "collab-staging" ]]; then + export ZED_KUBE_NAMESPACE=staging + export ZED_COLLAB_LOAD_BALANCER_SIZE_UNIT=1 + export ZED_API_LOAD_BALANCER_SIZE_UNIT=1 + else + echo "cowardly refusing to deploy from an unknown branch" + exit 1 + fi + + echo "Deploying collab:$GITHUB_SHA to $ZED_KUBE_NAMESPACE" + + source script/lib/deploy-helpers.sh + export_vars_for_environment $ZED_KUBE_NAMESPACE + + ZED_DO_CERTIFICATE_ID="$(doctl compute certificate list --format ID --no-header)" + export ZED_DO_CERTIFICATE_ID + export ZED_IMAGE_ID="registry.digitalocean.com/zed/collab:${GITHUB_SHA}" + + export ZED_SERVICE_NAME=collab + export ZED_LOAD_BALANCER_SIZE_UNIT=$ZED_COLLAB_LOAD_BALANCER_SIZE_UNIT + export DATABASE_MAX_CONNECTIONS=850 + envsubst < crates/collab/k8s/collab.template.yml | kubectl apply -f - + kubectl -n "$ZED_KUBE_NAMESPACE" rollout status deployment/$ZED_SERVICE_NAME --watch + echo "deployed ${ZED_SERVICE_NAME} to ${ZED_KUBE_NAMESPACE}" + + export ZED_SERVICE_NAME=api + export ZED_LOAD_BALANCER_SIZE_UNIT=$ZED_API_LOAD_BALANCER_SIZE_UNIT + export DATABASE_MAX_CONNECTIONS=60 + envsubst < crates/collab/k8s/collab.template.yml | kubectl apply -f - + kubectl -n "$ZED_KUBE_NAMESPACE" rollout status deployment/$ZED_SERVICE_NAME --watch + echo "deployed ${ZED_SERVICE_NAME} to ${ZED_KUBE_NAMESPACE}" +defaults: + run: + shell: bash -euxo pipefail {0} diff --git a/tooling/xtask/src/tasks/workflows.rs b/tooling/xtask/src/tasks/workflows.rs index 077282760a4cc141cd1210c58f38cdf0669a551b..5663ebec247c4025f7cfbae8e9467733e2c7be2d 100644 --- a/tooling/xtask/src/tasks/workflows.rs +++ b/tooling/xtask/src/tasks/workflows.rs @@ -10,6 +10,7 @@ mod bump_patch_version; mod cherry_pick; mod compare_perf; mod danger; +mod deploy_collab; mod extension_bump; mod extension_tests; mod extension_workflow_rollout; @@ -133,6 +134,7 @@ pub fn run_workflows(_: GenerateWorkflowArgs) -> Result<()> { WorkflowFile::zed(cherry_pick::cherry_pick), WorkflowFile::zed(compare_perf::compare_perf), WorkflowFile::zed(danger::danger), + WorkflowFile::zed(deploy_collab::deploy_collab), WorkflowFile::zed(extension_bump::extension_bump), WorkflowFile::zed(extension_tests::extension_tests), WorkflowFile::zed(extension_workflow_rollout::extension_workflow_rollout), diff --git a/tooling/xtask/src/tasks/workflows/deploy_collab.rs b/tooling/xtask/src/tasks/workflows/deploy_collab.rs new file mode 100644 index 0000000000000000000000000000000000000000..5c18da45e58f48ca281f800533a1b6b27f1433e7 --- /dev/null +++ b/tooling/xtask/src/tasks/workflows/deploy_collab.rs @@ -0,0 +1,171 @@ +use gh_workflow::{Container, Event, Port, Push, Run, Step, Use, Workflow}; +use indoc::{formatdoc, indoc}; + +use crate::tasks::workflows::{ + runners::{self, Platform}, + steps::{self, CommonJobConditions, NamedJob, dependant_job, named}, + vars, +}; + +pub(crate) fn deploy_collab() -> Workflow { + let style = style(); + let tests = tests(&[&style]); + let publish = publish(&[&style, &tests]); + let deploy = deploy(&[&publish]); + + named::workflow() + .on(Event::default().push(Push::default().add_tag("collab-production"))) + .add_env(("DOCKER_BUILDKIT", "1")) + .add_job(style.name, style.job) + .add_job(tests.name, tests.job) + .add_job(publish.name, publish.job) + .add_job(deploy.name, deploy.job) +} + +fn style() -> NamedJob { + named::job( + dependant_job(&[]) + .name("Check formatting and Clippy lints") + .with_repository_owner_guard() + .runs_on(runners::LINUX_XL) + .add_step(steps::checkout_repo().add_with(("fetch-depth", 0))) + .add_step(steps::setup_cargo_config(Platform::Linux)) + .add_step(steps::cache_rust_dependencies_namespace()) + .add_step(steps::cargo_fmt()) + .add_step(steps::clippy(Platform::Linux)), + ) +} + +fn tests(deps: &[&NamedJob]) -> NamedJob { + fn run_collab_tests() -> Step { + named::bash("cargo nextest run --package collab --no-fail-fast") + } + + named::job( + dependant_job(deps) + .name("Run tests") + .runs_on(runners::LINUX_XL) + .add_service( + "postgres", + Container::new("postgres:15") + .add_env(("POSTGRES_HOST_AUTH_METHOD", "trust")) + .ports(vec![Port::Name("5432:5432".into())]) + .options( + "--health-cmd pg_isready \ + --health-interval 500ms \ + --health-timeout 5s \ + --health-retries 10", + ), + ) + .add_step(steps::checkout_repo().add_with(("fetch-depth", 0))) + .add_step(steps::setup_cargo_config(Platform::Linux)) + .add_step(steps::cache_rust_dependencies_namespace()) + .add_step(steps::cargo_install_nextest()) + .add_step(steps::clear_target_dir_if_large(Platform::Linux)) + .add_step(run_collab_tests()), + ) +} + +fn publish(deps: &[&NamedJob]) -> NamedJob { + fn install_doctl() -> Step { + named::uses("digitalocean", "action-doctl", "v2") + .add_with(("token", vars::DIGITALOCEAN_ACCESS_TOKEN)) + } + + fn sign_into_registry() -> Step { + named::bash("doctl registry login") + } + + fn build_docker_image() -> Step { + named::bash(indoc! {r#" + docker build -f Dockerfile-collab \ + --build-arg "GITHUB_SHA=$GITHUB_SHA" \ + --tag "registry.digitalocean.com/zed/collab:$GITHUB_SHA" \ + . + "#}) + } + + fn publish_docker_image() -> Step { + named::bash(r#"docker push "registry.digitalocean.com/zed/collab:${GITHUB_SHA}""#) + } + + fn prune_docker_system() -> Step { + named::bash("docker system prune --filter 'until=72h' -f") + } + + named::job( + dependant_job(deps) + .name("Publish collab server image") + .runs_on(runners::LINUX_XL) + .add_step(install_doctl()) + .add_step(sign_into_registry()) + .add_step(steps::checkout_repo()) + .add_step(build_docker_image()) + .add_step(publish_docker_image()) + .add_step(prune_docker_system()), + ) +} + +fn deploy(deps: &[&NamedJob]) -> NamedJob { + fn install_doctl() -> Step { + named::uses("digitalocean", "action-doctl", "v2") + .add_with(("token", vars::DIGITALOCEAN_ACCESS_TOKEN)) + } + + fn sign_into_kubernetes() -> Step { + named::bash(formatdoc! {r#" + doctl kubernetes cluster kubeconfig save --expiry-seconds 600 {cluster_name} + "#, cluster_name = vars::CLUSTER_NAME}) + } + + fn start_rollout() -> Step { + named::bash(indoc! {r#" + set -eu + if [[ $GITHUB_REF_NAME = "collab-production" ]]; then + export ZED_KUBE_NAMESPACE=production + export ZED_COLLAB_LOAD_BALANCER_SIZE_UNIT=10 + export ZED_API_LOAD_BALANCER_SIZE_UNIT=2 + elif [[ $GITHUB_REF_NAME = "collab-staging" ]]; then + export ZED_KUBE_NAMESPACE=staging + export ZED_COLLAB_LOAD_BALANCER_SIZE_UNIT=1 + export ZED_API_LOAD_BALANCER_SIZE_UNIT=1 + else + echo "cowardly refusing to deploy from an unknown branch" + exit 1 + fi + + echo "Deploying collab:$GITHUB_SHA to $ZED_KUBE_NAMESPACE" + + source script/lib/deploy-helpers.sh + export_vars_for_environment $ZED_KUBE_NAMESPACE + + ZED_DO_CERTIFICATE_ID="$(doctl compute certificate list --format ID --no-header)" + export ZED_DO_CERTIFICATE_ID + export ZED_IMAGE_ID="registry.digitalocean.com/zed/collab:${GITHUB_SHA}" + + export ZED_SERVICE_NAME=collab + export ZED_LOAD_BALANCER_SIZE_UNIT=$ZED_COLLAB_LOAD_BALANCER_SIZE_UNIT + export DATABASE_MAX_CONNECTIONS=850 + envsubst < crates/collab/k8s/collab.template.yml | kubectl apply -f - + kubectl -n "$ZED_KUBE_NAMESPACE" rollout status deployment/$ZED_SERVICE_NAME --watch + echo "deployed ${ZED_SERVICE_NAME} to ${ZED_KUBE_NAMESPACE}" + + export ZED_SERVICE_NAME=api + export ZED_LOAD_BALANCER_SIZE_UNIT=$ZED_API_LOAD_BALANCER_SIZE_UNIT + export DATABASE_MAX_CONNECTIONS=60 + envsubst < crates/collab/k8s/collab.template.yml | kubectl apply -f - + kubectl -n "$ZED_KUBE_NAMESPACE" rollout status deployment/$ZED_SERVICE_NAME --watch + echo "deployed ${ZED_SERVICE_NAME} to ${ZED_KUBE_NAMESPACE}" + "#}) + } + + named::job( + dependant_job(deps) + .name("Deploy new server image") + .runs_on(runners::LINUX_XL) + .add_step(steps::checkout_repo()) + .add_step(install_doctl()) + .add_step(sign_into_kubernetes()) + .add_step(start_rollout()), + ) +} diff --git a/tooling/xtask/src/tasks/workflows/vars.rs b/tooling/xtask/src/tasks/workflows/vars.rs index 5939ddbffe6e3cc28cf14e424070480e0ffc60c9..aa8fb0a4056a53807cd4b2f12f331cb9d4d0a235 100644 --- a/tooling/xtask/src/tasks/workflows/vars.rs +++ b/tooling/xtask/src/tasks/workflows/vars.rs @@ -30,6 +30,8 @@ secret!(AZURE_SIGNING_CLIENT_ID); secret!(AZURE_SIGNING_CLIENT_SECRET); secret!(AZURE_SIGNING_TENANT_ID); secret!(CACHIX_AUTH_TOKEN); +secret!(CLUSTER_NAME); +secret!(DIGITALOCEAN_ACCESS_TOKEN); secret!(DIGITALOCEAN_SPACES_ACCESS_KEY); secret!(DIGITALOCEAN_SPACES_SECRET_KEY); secret!(GITHUB_TOKEN);