From 9367f719f267f7981684cdbdb32b114ca51d9e4d Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Wed, 17 Jan 2024 12:45:18 -0800 Subject: [PATCH 1/2] Rework db-seeding, so that it doesn't depend on a github auth token Instead, admins are specified using a JSON file, 'admins.json'. This file is gitignored. If it is not present, there is a default list of admins in 'admins.default.json'. --- .gitignore | 1 + crates/collab/.admins.default.json | 1 + crates/collab/src/bin/seed.rs | 121 ++++++++++++-------------- crates/collab/src/db/queries/users.rs | 6 +- script/seed-db | 4 - script/zed-local | 39 +++++---- 6 files changed, 84 insertions(+), 88 deletions(-) create mode 100644 crates/collab/.admins.default.json diff --git a/.gitignore b/.gitignore index 2d8807a4b0559751ff341eacf7dfaf51c84c405c..6923b060f6ac1f1fc50b423100edbfd5006d5909 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ /styles/src/types/zed.ts /crates/theme/schemas/theme.json /crates/collab/static/styles.css +/crates/collab/.admins.json /vendor/bin /assets/themes/*.json /assets/*licenses.md diff --git a/crates/collab/.admins.default.json b/crates/collab/.admins.default.json new file mode 100644 index 0000000000000000000000000000000000000000..6ee4d8726a303be4457078be9353402cbd712f20 --- /dev/null +++ b/crates/collab/.admins.default.json @@ -0,0 +1 @@ +["nathansobo", "as-cii", "maxbrunsfeld", "iamnbutler"] diff --git a/crates/collab/src/bin/seed.rs b/crates/collab/src/bin/seed.rs index 88fe0a647b8924b2df1312aa8a9a3bd68b5d99f1..ed24ccef75dce446eb54a431d1371139d67b7140 100644 --- a/crates/collab/src/bin/seed.rs +++ b/crates/collab/src/bin/seed.rs @@ -1,7 +1,11 @@ -use collab::{db, executor::Executor}; +use collab::{ + db::{self, NewUserParams}, + env::load_dotenv, + executor::Executor, +}; use db::{ConnectOptions, Database}; use serde::{de::DeserializeOwned, Deserialize}; -use std::fmt::Write; +use std::{fmt::Write, fs}; #[derive(Debug, Deserialize)] struct GitHubUser { @@ -12,90 +16,75 @@ struct GitHubUser { #[tokio::main] async fn main() { + load_dotenv().expect("failed to load .env.toml file"); + + let mut admin_logins = + load_admins("./.admins.default.json").expect("failed to load default admins file"); + if let Ok(other_admins) = load_admins("./.admins.json") { + admin_logins.extend(other_admins); + } + let database_url = std::env::var("DATABASE_URL").expect("missing DATABASE_URL env var"); let db = Database::new(ConnectOptions::new(database_url), Executor::Production) .await .expect("failed to connect to postgres database"); - let github_token = std::env::var("GITHUB_TOKEN").expect("missing GITHUB_TOKEN env var"); let client = reqwest::Client::new(); - let mut current_user = - fetch_github::(&client, &github_token, "https://api.github.com/user").await; - current_user - .email - .get_or_insert_with(|| "placeholder@example.com".to_string()); - let staff_users = fetch_github::>( - &client, - &github_token, - "https://api.github.com/orgs/zed-industries/teams/staff/members", - ) - .await; - - let mut zed_users = Vec::new(); - zed_users.push((current_user, true)); - zed_users.extend(staff_users.into_iter().map(|user| (user, true))); + // Create admin users for all of the users in `.admins.toml` or `.admins.default.toml`. + for admin_login in admin_logins { + let user = fetch_github::( + &client, + &format!("https://api.github.com/users/{admin_login}"), + ) + .await; + db.create_user( + &user.email.unwrap_or(format!("{admin_login}@example.com")), + true, + NewUserParams { + github_login: user.login, + github_user_id: user.id, + }, + ) + .await + .expect("failed to create admin user"); + } - let user_count = db + // Fetch 100 other random users from GitHub and insert them into the database. + let mut user_count = db .get_all_users(0, 200) .await .expect("failed to load users from db") .len(); - if user_count < 100 { - let mut last_user_id = None; - for _ in 0..10 { - let mut uri = "https://api.github.com/users?per_page=100".to_string(); - if let Some(last_user_id) = last_user_id { - write!(&mut uri, "&since={}", last_user_id).unwrap(); - } - let users = fetch_github::>(&client, &github_token, &uri).await; - if let Some(last_user) = users.last() { - last_user_id = Some(last_user.id); - zed_users.extend(users.into_iter().map(|user| (user, false))); - } else { - break; - } + let mut last_user_id = None; + while user_count < 100 { + let mut uri = "https://api.github.com/users?per_page=100".to_string(); + if let Some(last_user_id) = last_user_id { + write!(&mut uri, "&since={}", last_user_id).unwrap(); } - } + let users = fetch_github::>(&client, &uri).await; - for (github_user, admin) in zed_users { - if db - .get_user_by_github_login(&github_user.login) + for github_user in users { + last_user_id = Some(github_user.id); + user_count += 1; + db.get_or_create_user_by_github_account( + &github_user.login, + Some(github_user.id), + github_user.email.as_deref(), + ) .await - .expect("failed to fetch user") - .is_none() - { - if admin { - db.create_user( - &format!("{}@zed.dev", github_user.login), - admin, - db::NewUserParams { - github_login: github_user.login, - github_user_id: github_user.id, - }, - ) - .await - .expect("failed to insert user"); - } else { - db.get_or_create_user_by_github_account( - &github_user.login, - Some(github_user.id), - github_user.email.as_deref(), - ) - .await - .expect("failed to insert user"); - } + .expect("failed to insert user"); } } } -async fn fetch_github( - client: &reqwest::Client, - access_token: &str, - url: &str, -) -> T { +fn load_admins(path: &str) -> anyhow::Result> { + let file_content = fs::read_to_string(path)?; + Ok(serde_json::from_str(&file_content)?) +} + +async fn fetch_github(client: &reqwest::Client, url: &str) -> T { let response = client .get(url) - .bearer_auth(&access_token) .header("user-agent", "zed") .send() .await diff --git a/crates/collab/src/db/queries/users.rs b/crates/collab/src/db/queries/users.rs index 954ec5f0d80c5b58149cd935abca648fa40a82ef..8f975b5cbe5d8b4239e34e8770b57b979d6ac378 100644 --- a/crates/collab/src/db/queries/users.rs +++ b/crates/collab/src/db/queries/users.rs @@ -20,7 +20,11 @@ impl Database { }) .on_conflict( OnConflict::column(user::Column::GithubLogin) - .update_column(user::Column::GithubLogin) + .update_columns([ + user::Column::Admin, + user::Column::EmailAddress, + user::Column::GithubUserId, + ]) .to_owned(), ) .exec_with_returning(&*tx) diff --git a/script/seed-db b/script/seed-db index 6bb0f969330fec3936d2008c6013e9b37b8437cd..277ea89ba3b8e8dc056f6fab052531cec86cf102 100755 --- a/script/seed-db +++ b/script/seed-db @@ -2,8 +2,4 @@ set -e cd crates/collab - -# Export contents of .env.toml -eval "$(cargo run --quiet --bin dotenv)" - cargo run --quiet --package=collab --features seed-support --bin seed -- $@ diff --git a/script/zed-local b/script/zed-local index 090fbd58760cd04779340be19d06a199d2a0a01d..4ae4013a4c3fe7b4eaee1b24b7ea8c5bc70a2259 100755 --- a/script/zed-local +++ b/script/zed-local @@ -4,6 +4,11 @@ const HELP = ` USAGE zed-local [options] [zed args] +SUMMARY + Runs 1-4 instances of Zed using a locally-running collaboration server. + Each instance of Zed will be signed in as a different user specified in + either \`.admins.json\` or \`.admins.default.json\`. + OPTIONS --help Print this help message --release Build Zed in release mode @@ -12,6 +17,16 @@ OPTIONS `.trim(); const { spawn, execFileSync } = require("child_process"); +const assert = require("assert"); + +const defaultUsers = require("../crates/collab/.admins.default.json"); +let users = defaultUsers; +try { + const customUsers = require("../crates/collab/.admins.json"); + assert(customUsers.length > 0); + assert(customUsers.every((user) => typeof user === "string")); + users.splice(0, 0, ...customUsers); +} catch (_) {} const RESOLUTION_REGEX = /(\d+) x (\d+)/; const DIGIT_FLAG_REGEX = /^--?(\d+)$/; @@ -71,10 +86,6 @@ if (instanceCount > 1) { } } -let users = ["nathansobo", "as-cii", "maxbrunsfeld", "iamnbutler"]; - -const RUST_LOG = process.env.RUST_LOG || "info"; - // If a user is specified, make sure it's first in the list const user = process.env.ZED_IMPERSONATE; if (user) { @@ -88,18 +99,12 @@ const positions = [ `${instanceWidth},${instanceHeight}`, ]; -const buildArgs = (() => { - const buildArgs = ["build"]; - if (isReleaseMode) { - buildArgs.push("--release"); - } - - return buildArgs; -})(); -const zedBinary = (() => { - const target = isReleaseMode ? "release" : "debug"; - return `target/${target}/Zed`; -})(); +let buildArgs = ["build"]; +let zedBinary = "target/debug/Zed"; +if (isReleaseMode) { + buildArgs.push("--release"); + zedBinary = "target/release/Zed"; +} execFileSync("cargo", buildArgs, { stdio: "inherit" }); setTimeout(() => { @@ -115,7 +120,7 @@ setTimeout(() => { ZED_ADMIN_API_TOKEN: "secret", ZED_WINDOW_SIZE: `${instanceWidth},${instanceHeight}`, PATH: process.env.PATH, - RUST_LOG, + RUST_LOG: process.env.RUST_LOG || "info", }, }); } From ad2b4f288e493d0f3564cd220c7055776c88298e Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Wed, 17 Jan 2024 13:24:55 -0800 Subject: [PATCH 2/2] Update procfile and local development docs, zed.dev is no longer needed --- Procfile | 2 - docs/src/SUMMARY.md | 3 + docs/src/developing_zed__building_zed.md | 136 +++++------------- .../developing_zed__local_collaboration.md | 52 +++++-- 4 files changed, 79 insertions(+), 114 deletions(-) diff --git a/Procfile b/Procfile index 3f42c3a9677477bd7dcd04ca61a749206559be40..7bd9114dad4ec3e89c4699d0924bbf1ef1243867 100644 --- a/Procfile +++ b/Procfile @@ -1,4 +1,2 @@ -web: cd ../zed.dev && PORT=3000 npm run dev collab: cd crates/collab && RUST_LOG=${RUST_LOG:-warn,collab=info} cargo run serve livekit: livekit-server --dev -postgrest: postgrest crates/collab/admin_api.conf diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md index ad1cd6332c4a5200a66ff5767db3673abc88f921..e300e9906956c9eb44fe730de93f4de4a8eea90a 100644 --- a/docs/src/SUMMARY.md +++ b/docs/src/SUMMARY.md @@ -4,15 +4,18 @@ [Feedback](./feedback.md) # Configuring Zed + - [Settings](./configuring_zed.md) - [Vim Mode](./configuring_zed__configuring_vim.md) # Using Zed + - [Workflows]() - [Collaboration]() - [Using AI]() # Contributing to Zed + - [How to Contribute]() - [Building from Source](./developing_zed__building_zed.md) - [Local Collaboration](./developing_zed__local_collaboration.md) diff --git a/docs/src/developing_zed__building_zed.md b/docs/src/developing_zed__building_zed.md index a5270e2b2abc8f89e88e23d016e2e3a21a065efd..7535ceb4d0193e01b46e1cf1f9e2c818c086f138 100644 --- a/docs/src/developing_zed__building_zed.md +++ b/docs/src/developing_zed__building_zed.md @@ -1,133 +1,73 @@ # Building Zed -🚧 TODO: +## Dependencies -- [ ] Remove ZI-specific things -- [ ] Rework any steps that currently require a ZI-specific account +- Install [Rust](https://www.rust-lang.org/tools/install) +- Install [Xcode](https://apps.apple.com/us/app/xcode/id497799835?mt=12) from the macOS App Store -How to build Zed from source for the first time. +- Install [Xcode command line tools](https://developer.apple.com/xcode/resources/) -### Prerequisites + ```bash + xcode-select --install + ``` -🚧 TODO 🚧 Update for open source +- Ensure that the Xcode command line tools are using your newly installed copy of Xcode: -- Be added to the GitHub organization -- Be added to the Vercel team -- Create a [Personal Access Token](https://github.com/settings/personal-access-tokens/new) on Github - - 🚧 TODO 🚧 What permissions are required? - - 🚧 TODO 🚧 What changes when repo isn't private? - - Go to https://github.com/settings/tokens and Generate new token - - GitHub currently provides two kinds of tokens: - - Classic Tokens, where only `repo` (Full control of private repositories) OAuth scope has to be selected - Unfortunately, unselecting `repo` scope and selecting every its inner scope instead does not allow the token users to read from private repositories - - (not applicable) Fine-grained Tokens, at the moment of writing, did not allow any kind of access of non-owned private repos - - Keep the token in the browser tab/editor for the next two steps + ``` + sudo xcode-select --switch /Applications/Xcode.app/Contents/Developer. + ``` -### Dependencies +* Install the Rust wasm toolchain: -- Install [Rust](https://www.rust-lang.org/tools/install) + ```bash + rustup target add wasm32-wasi + ``` -- Install the [GitHub CLI](https://cli.github.com/), [Livekit](https://formulae.brew.sh/formula/livekit) & [Foreman](https://formulae.brew.sh/formula/foreman) +## Backend Dependencies -```bash -brew install gh -brew install livekit -brew install foreman -``` +If you are developing collaborative features of Zed, you'll need to install the dependencies of zed's `collab` server: -- Install [Xcode](https://apps.apple.com/us/app/xcode/id497799835?mt=12) from the macOS App Store +- Install [Postgres](https://postgresapp.com) +- Install [Livekit](https://formulae.brew.sh/formula/livekit) and [Foreman](https://formulae.brew.sh/formula/foreman) -- Install [Xcode command line tools](https://developer.apple.com/xcode/resources/) + ```bash + brew install livekit foreman + ``` -```bash -xcode-select --install -``` - -- If `xcode-select --print-path prints /Library/Developer/CommandLineTools…` run `sudo xcode-select --switch /Applications/Xcode.app/Contents/Developer.` +## Building Zed from Source -* Install [Postgres](https://postgresapp.com) +Once you have the dependencies installed, you can build Zed using [Cargo](https://doc.rust-lang.org/cargo/). -* Install the wasm toolchain +For a debug build: -```bash -rustup target add wasm32-wasi ``` - -### Building Zed from Source - -1. Clone the `zed` repo - -```bash -gh repo clone zed-industries/zed +cargo run ``` -1. (Optional but recommended) Add your GITHUB_TOKEN to your `.zshrc` or `.bashrc` like this: `export GITHUB_TOKEN=yourGithubAPIToken` -1. (🚧 TODO 🚧 - Will this be relevant for open source?) Ensure the Zed.dev website is checked out in a sibling directory and install its dependencies: +For a release build: -```bash -cd .. -git clone https://github.com/zed-industries/zed.dev -cd zed.dev && npm install -pnpm install -g vercel +``` +cargo run --release ``` -1. (🚧 TODO 🚧 - Will this be relevant for open source?) Link your zed.dev project to Vercel - -- `vercel link` -- Select the `zed-industries` team. If you don't have this get someone on the team to add you to it. -- Select the `zed.dev` project - -1. (🚧 TODO 🚧 - Will this be relevant for open source?) Run `vercel pull` to pull down the environment variables and project info from Vercel -1. Open Postgres.app -1. From `./path/to/zed/` run `GITHUB_TOKEN={yourGithubAPIToken} script/bootstrap` - -- You don't need to include the GITHUB_TOKEN if you exported it above. -- Consider removing the token (if it's fine for you to recreate such tokens during occasional migrations) or store this token somewhere safe (like your Zed 1Password vault). +And to run the tests: -1. To run the Zed app: - - If you are working on zed: - - `cargo run` - - If you are just using the latest version, but not working on zed: - - `cargo run --release` - - If you need to run the collaboration server locally: - - `script/zed-local` +``` +cargo test --workspace +``` ## Troubleshooting -**`error: failed to run custom build command for gpui v0.1.0 (/Users/path/to/zed)`** - -- Try `xcode-select --switch /Applications/Xcode.app/Contents/Developer` - -**`xcrun: error: unable to find utility "metal", not a developer tool or in PATH`** +### Error compiling metal shaders -### `script/bootstrap` - -```bash -Error: Cannot install in Homebrew on ARM processor in Intel default prefix (/usr/local)! -Please create a new installation in /opt/homebrew using one of the -"Alternative Installs" from: -https://docs.brew.sh/Installation ``` +error: failed to run custom build command for gpui v0.1.0 (/Users/path/to/zed)`** -- In that case try `/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"` - -- If Homebrew is not in your PATH: - - Replace `{username}` with your home folder name (usually your login name) - - `echo 'eval "$(/opt/homebrew/bin/brew shellenv)"' >> /Users/{username}/.zprofile` - - `eval "$(/opt/homebrew/bin/brew shellenv)"` - +xcrun: error: unable to find utility "metal", not a developer tool or in PATH ``` -seeding database... -thread 'main' panicked at 'failed to deserialize github user from 'https://api.github.com/orgs/zed-industries/teams/staff/members': reqwest::Error { kind: Decode, source: Error("invalid type: map, expected a sequence", line: 1, column: 0) }', crates/collab/src/bin/seed.rs:111:10 -``` - -Wrong permissions for `GITHUB_TOKEN` token used, the token needs to be able to read from private repos. -For Classic GitHub Tokens, that required OAuth scope `repo` (seacrh the scope name above for more details) - -Same command -`sudo xcode-select --switch /Applications/Xcode.app/Contents/Developer` +Try `xcode-select --switch /Applications/Xcode.app/Contents/Developer` -### If you experience errors that mention some dependency is using unstable features +### Cargo errors claiming that a dependency is using unstable features Try `cargo clean` and `cargo build`, diff --git a/docs/src/developing_zed__local_collaboration.md b/docs/src/developing_zed__local_collaboration.md index 7bbbda36457174816ccb0da6d98dac8543b5f065..0fc08ef767df89ab9c059f30dbbc4cb95c227129 100644 --- a/docs/src/developing_zed__local_collaboration.md +++ b/docs/src/developing_zed__local_collaboration.md @@ -1,22 +1,46 @@ # Local Collaboration -## Setting up the local collaboration server +First, make sure you've installed Zed's [backend dependencies](/developing_zed__building_zed.html#backend-dependencies). -### Setting up for the first time? +## Database setup -1. Make sure you have livekit installed (`brew install livekit`) -1. Install [Postgres](https://postgresapp.com) and run it. -1. Then, from the root of the repo, run `script/bootstrap`. +Before you can run the `collab` server locally, you'll need to set up a `zed` Postgres database. -### Have a db that is out of date? / Need to migrate? +``` +script/bootstrap +``` -1. Make sure you have livekit installed (`brew install livekit`) -1. Try `cd crates/collab && cargo run -- migrate` from the root of the repo. -1. Run `script/seed-db` +This script will set up the `zed` Postgres database, and populate it with some users. It requires internet access, because it fetches some users from the GitHub API. -## Testing collab locally +The script will create several *admin* users, who you'll sign in as by default when developing locally. The GitHub logins for these default admin users are specified in this file: -1. Run `foreman start` from the root of the repo. -1. In another terminal run `script/zed-local`. -1. Two copies of Zed will open. Add yourself as a contact in the one that is not you. -1. Start a collaboration session as normal with any open project. +``` +cat crates/collab/.admins.default.json +``` + +To use a different set of admin users, you can create a file called `.admins.json` in the same directory: + +``` +cat > crates/collab/.admins.json <