Max Brunsfeld and Marshall Bowers created
This will be used in the
[extensions](https://github.com/zed-industries/extensions) repository
for packaging the extensions that users submit.
Release Notes:
- N/A
---------
Co-authored-by: Marshall Bowers <elliott.codes@gmail.com>Cargo.lock | 18 ++
Cargo.toml | 2
crates/collab/Cargo.toml | 2
crates/extension/src/extension_builder.rs | 31 ++--
crates/extension/src/extension_store.rs | 11 +
crates/extension_cli/Cargo.toml | 23 +++
crates/extension_cli/LICENSE-GPL | 1
crates/extension_cli/src/main.rs | 159 +++++++++++++++++++++++++
8 files changed, 229 insertions(+), 18 deletions(-)
@@ -3514,6 +3514,24 @@ dependencies = [
"wit-component 0.20.3",
]
+[[package]]
+name = "extension_cli"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "clap 4.4.4",
+ "env_logger",
+ "extension",
+ "fs",
+ "language",
+ "log",
+ "theme",
+ "tokio",
+ "toml 0.8.10",
+ "tree-sitter",
+ "wasmtime",
+]
+
[[package]]
name = "extensions_ui"
version = "0.1.0"
@@ -24,6 +24,7 @@ members = [
"crates/editor",
"crates/extension",
"crates/extension_api",
+ "crates/extension_cli",
"crates/extensions_ui",
"crates/feature_flags",
"crates/feedback",
@@ -273,6 +274,7 @@ time = { version = "0.3", features = [
"formatting",
] }
toml = "0.8"
+tokio = { version = "1", features = ["full"] }
tower-http = "0.4.4"
tree-sitter = { version = "0.20", features = ["wasm"] }
tree-sitter-astro = { git = "https://github.com/virchau13/tree-sitter-astro.git", rev = "e924787e12e8a03194f36a113290ac11d6dc10f3" }
@@ -54,7 +54,7 @@ rustc-demangle.workspace = true
telemetry_events.workspace = true
text.workspace = true
time.workspace = true
-tokio = { version = "1", features = ["full"] }
+tokio.workspace = true
toml.workspace = true
tower = "0.4"
tower-http = { workspace = true, features = ["trace"] }
@@ -6,9 +6,8 @@ use async_tar::Archive;
use futures::io::BufReader;
use futures::AsyncReadExt;
use serde::Deserialize;
-use std::mem;
use std::{
- env, fs,
+ env, fs, mem,
path::{Path, PathBuf},
process::{Command, Stdio},
sync::Arc,
@@ -72,22 +71,27 @@ impl ExtensionBuilder {
pub async fn compile_extension(
&self,
extension_dir: &Path,
+ extension_manifest: &ExtensionManifest,
options: CompileExtensionOptions,
) -> Result<()> {
+ if extension_dir.is_relative() {
+ bail!(
+ "extension dir {} is not an absolute path",
+ extension_dir.display()
+ );
+ }
+
fs::create_dir_all(&self.cache_dir)?;
- let extension_toml_path = extension_dir.join("extension.toml");
- let extension_toml_content = fs::read_to_string(&extension_toml_path)?;
- let extension_toml: ExtensionManifest = toml::from_str(&extension_toml_content)?;
let cargo_toml_path = extension_dir.join("Cargo.toml");
- if extension_toml.lib.kind == Some(ExtensionLibraryKind::Rust)
+ if extension_manifest.lib.kind == Some(ExtensionLibraryKind::Rust)
|| fs::metadata(&cargo_toml_path)?.is_file()
{
self.compile_rust_extension(extension_dir, options).await?;
}
- for (grammar_name, grammar_metadata) in extension_toml.grammars {
- self.compile_grammar(extension_dir, grammar_name, grammar_metadata)
+ for (grammar_name, grammar_metadata) in &extension_manifest.grammars {
+ self.compile_grammar(extension_dir, grammar_name.as_ref(), grammar_metadata)
.await?;
}
@@ -157,13 +161,13 @@ impl ExtensionBuilder {
async fn compile_grammar(
&self,
extension_dir: &Path,
- grammar_name: Arc<str>,
- grammar_metadata: GrammarManifestEntry,
+ grammar_name: &str,
+ grammar_metadata: &GrammarManifestEntry,
) -> Result<()> {
let clang_path = self.install_wasi_sdk_if_needed().await?;
let mut grammar_repo_dir = extension_dir.to_path_buf();
- grammar_repo_dir.extend(["grammars", grammar_name.as_ref()]);
+ grammar_repo_dir.extend(["grammars", grammar_name]);
let mut grammar_wasm_path = grammar_repo_dir.clone();
grammar_wasm_path.set_extension("wasm");
@@ -277,9 +281,10 @@ impl ExtensionBuilder {
);
}
bail!(
- "failed to checkout revision {} in directory '{}'",
+ "failed to checkout revision {} in directory '{}': {}",
rev,
- directory.display()
+ directory.display(),
+ String::from_utf8_lossy(&checkout_output.stderr)
);
}
@@ -1,4 +1,4 @@
-mod build_extension;
+pub mod extension_builder;
mod extension_lsp_adapter;
mod extension_manifest;
mod wasm_host;
@@ -10,8 +10,8 @@ use crate::{extension_lsp_adapter::ExtensionLspAdapter, wasm_host::wit};
use anyhow::{anyhow, bail, Context as _, Result};
use async_compression::futures::bufread::GzipDecoder;
use async_tar::Archive;
-use build_extension::{CompileExtensionOptions, ExtensionBuilder};
use collections::{hash_map, BTreeMap, HashMap, HashSet};
+use extension_builder::{CompileExtensionOptions, ExtensionBuilder};
use extension_manifest::ExtensionLibraryKind;
use fs::{Fs, RemoveOptions};
use futures::{
@@ -545,6 +545,7 @@ impl ExtensionStore {
builder
.compile_extension(
&extension_source_path,
+ &extension_manifest,
CompileExtensionOptions { release: false },
)
.await
@@ -580,6 +581,7 @@ impl ExtensionStore {
pub fn rebuild_dev_extension(&mut self, extension_id: Arc<str>, cx: &mut ModelContext<Self>) {
let path = self.installed_dir.join(extension_id.as_ref());
let builder = self.builder.clone();
+ let fs = self.fs.clone();
match self.outstanding_operations.entry(extension_id.clone()) {
hash_map::Entry::Occupied(_) => return,
@@ -588,8 +590,9 @@ impl ExtensionStore {
cx.notify();
let compile = cx.background_executor().spawn(async move {
+ let manifest = Self::load_extension_manifest(fs, &path).await?;
builder
- .compile_extension(&path, CompileExtensionOptions { release: true })
+ .compile_extension(&path, &manifest, CompileExtensionOptions { release: true })
.await
});
@@ -1000,7 +1003,7 @@ impl ExtensionStore {
Ok(())
}
- async fn load_extension_manifest(
+ pub async fn load_extension_manifest(
fs: Arc<dyn Fs>,
extension_dir: &Path,
) -> Result<ExtensionManifest> {
@@ -0,0 +1,23 @@
+[package]
+name = "extension_cli"
+version = "0.1.0"
+edition = "2021"
+publish = false
+license = "GPL-3.0-or-later"
+
+[lints]
+workspace = true
+
+[dependencies]
+anyhow.workspace = true
+clap = { workspace = true, features = ["derive"] }
+env_logger.workspace = true
+fs.workspace = true
+extension.workspace = true
+language.workspace = true
+log.workspace = true
+theme.workspace = true
+tokio.workspace = true
+toml.workspace = true
+tree-sitter.workspace = true
+wasmtime.workspace = true
@@ -0,0 +1 @@
+../../LICENSE-GPL
@@ -0,0 +1,159 @@
+use std::{
+ collections::HashMap,
+ fs,
+ path::{Path, PathBuf},
+ sync::Arc,
+};
+
+use ::fs::{Fs, RealFs};
+use anyhow::{anyhow, Context, Result};
+use clap::Parser;
+use extension::{
+ extension_builder::{CompileExtensionOptions, ExtensionBuilder},
+ ExtensionStore,
+};
+use language::LanguageConfig;
+use theme::ThemeRegistry;
+use tree_sitter::{Language, Query, WasmStore};
+
+#[derive(Parser, Debug)]
+#[command(name = "zed-extension")]
+struct Args {
+ /// The path to the extension directory
+ extension_path: PathBuf,
+ /// Whether to compile with optimizations
+ #[arg(long)]
+ release: bool,
+ /// The path to a directory where build dependencies are downloaded
+ #[arg(long)]
+ scratch_dir: PathBuf,
+}
+
+#[tokio::main]
+async fn main() -> Result<()> {
+ env_logger::init();
+
+ let args = Args::parse();
+ let fs = Arc::new(RealFs);
+ let engine = wasmtime::Engine::default();
+ let mut wasm_store = WasmStore::new(engine)?;
+
+ let extension_path = args
+ .extension_path
+ .canonicalize()
+ .context("can't canonicalize extension_path")?;
+ let scratch_dir = args
+ .scratch_dir
+ .canonicalize()
+ .context("can't canonicalize scratch_dir")?;
+
+ let manifest = ExtensionStore::load_extension_manifest(fs.clone(), &extension_path).await?;
+ let builder = ExtensionBuilder::new(scratch_dir);
+ builder
+ .compile_extension(
+ &extension_path,
+ &manifest,
+ CompileExtensionOptions {
+ release: args.release,
+ },
+ )
+ .await?;
+
+ let grammars = test_grammars(&extension_path, &mut wasm_store)?;
+ test_languages(&extension_path, &grammars)?;
+ test_themes(&extension_path, fs.clone()).await?;
+
+ Ok(())
+}
+
+fn test_grammars(
+ extension_path: &Path,
+ wasm_store: &mut WasmStore,
+) -> Result<HashMap<String, Language>> {
+ let mut grammars = HashMap::default();
+ let grammars_dir = extension_path.join("grammars");
+ if !grammars_dir.exists() {
+ return Ok(grammars);
+ }
+
+ let entries = fs::read_dir(&grammars_dir)?;
+ for entry in entries {
+ let entry = entry?;
+ let grammar_path = entry.path();
+ let grammar_name = grammar_path.file_stem().unwrap().to_str().unwrap();
+ if grammar_path.extension() == Some("wasm".as_ref()) {
+ let wasm = fs::read(&grammar_path)?;
+ let language = wasm_store.load_language(grammar_name, &wasm)?;
+ log::info!("loaded grammar {grammar_name}");
+ grammars.insert(grammar_name.into(), language);
+ }
+ }
+
+ Ok(grammars)
+}
+
+fn test_languages(extension_path: &Path, grammars: &HashMap<String, Language>) -> Result<()> {
+ let languages_dir = extension_path.join("languages");
+ if !languages_dir.exists() {
+ return Ok(());
+ }
+
+ let entries = fs::read_dir(&languages_dir)?;
+ for entry in entries {
+ let entry = entry?;
+ let language_dir = entry.path();
+ let config_path = language_dir.join("config.toml");
+ let config_content = fs::read_to_string(&config_path)?;
+ let config: LanguageConfig = toml::from_str(&config_content)?;
+ let grammar = if let Some(name) = &config.grammar {
+ Some(
+ grammars
+ .get(name.as_ref())
+ .ok_or_else(|| anyhow!("language"))?,
+ )
+ } else {
+ None
+ };
+
+ let query_entries = fs::read_dir(&language_dir)?;
+ for entry in query_entries {
+ let entry = entry?;
+ let query_path = entry.path();
+ if query_path.extension() == Some("scm".as_ref()) {
+ let grammar = grammar.ok_or_else(|| {
+ anyhow!(
+ "language {} provides query {} but no grammar",
+ config.name,
+ query_path.display()
+ )
+ })?;
+
+ let query_source = fs::read_to_string(&query_path)?;
+ let _query = Query::new(grammar, &query_source)?;
+ }
+ }
+
+ log::info!("loaded language {}", config.name);
+ }
+
+ Ok(())
+}
+
+async fn test_themes(extension_path: &Path, fs: Arc<dyn Fs>) -> Result<()> {
+ let themes_dir = extension_path.join("themes");
+ if !themes_dir.exists() {
+ return Ok(());
+ }
+
+ let entries = fs::read_dir(&themes_dir)?;
+ for entry in entries {
+ let entry = entry?;
+ let theme_path = entry.path();
+ if theme_path.extension() == Some("json".as_ref()) {
+ let theme_family = ThemeRegistry::read_user_theme(&entry.path(), fs.clone()).await?;
+ log::info!("loaded theme family {}", theme_family.name);
+ }
+ }
+
+ Ok(())
+}