1use crate::{
2 parse_wasm_extension_version, ExtensionLibraryKind, ExtensionManifest, GrammarManifestEntry,
3};
4use anyhow::{anyhow, bail, Context as _, Result};
5use async_compression::futures::bufread::GzipDecoder;
6use async_tar::Archive;
7use futures::io::BufReader;
8use futures::AsyncReadExt;
9use http_client::{self, AsyncBody, HttpClient};
10use serde::Deserialize;
11use std::{
12 env, fs, mem,
13 path::{Path, PathBuf},
14 process::Stdio,
15 sync::Arc,
16};
17use wasm_encoder::{ComponentSectionId, Encode as _, RawSection, Section as _};
18use wasmparser::Parser;
19use wit_component::ComponentEncoder;
20
21/// Currently, we compile with Rust's `wasm32-wasip1` target, which works with WASI `preview1`.
22/// But the WASM component model is based on WASI `preview2`. So we need an 'adapter' WASM
23/// module, which implements the `preview1` interface in terms of `preview2`.
24///
25/// Once Rust 1.78 is released, there will be a `wasm32-wasip2` target available, so we will
26/// not need the adapter anymore.
27const RUST_TARGET: &str = "wasm32-wasip1";
28const WASI_ADAPTER_URL: &str =
29 "https://github.com/bytecodealliance/wasmtime/releases/download/v18.0.2/wasi_snapshot_preview1.reactor.wasm";
30
31/// Compiling Tree-sitter parsers from C to WASM requires Clang 17, and a WASM build of libc
32/// and clang's runtime library. The `wasi-sdk` provides these binaries.
33///
34/// Once Clang 17 and its wasm target are available via system package managers, we won't need
35/// to download this.
36const WASI_SDK_URL: &str = "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-21/";
37const WASI_SDK_ASSET_NAME: Option<&str> = if cfg!(target_os = "macos") {
38 Some("wasi-sdk-21.0-macos.tar.gz")
39} else if cfg!(any(target_os = "linux", target_os = "freebsd")) {
40 Some("wasi-sdk-21.0-linux.tar.gz")
41} else if cfg!(target_os = "windows") {
42 Some("wasi-sdk-21.0.m-mingw.tar.gz")
43} else {
44 None
45};
46
47pub struct ExtensionBuilder {
48 cache_dir: PathBuf,
49 pub http: Arc<dyn HttpClient>,
50}
51
52pub struct CompileExtensionOptions {
53 pub release: bool,
54}
55
56#[derive(Deserialize)]
57struct CargoToml {
58 package: CargoTomlPackage,
59}
60
61#[derive(Deserialize)]
62struct CargoTomlPackage {
63 name: String,
64}
65
66impl ExtensionBuilder {
67 pub fn new(http_client: Arc<dyn HttpClient>, cache_dir: PathBuf) -> Self {
68 Self {
69 cache_dir,
70 http: http_client,
71 }
72 }
73
74 pub async fn compile_extension(
75 &self,
76 extension_dir: &Path,
77 extension_manifest: &mut ExtensionManifest,
78 options: CompileExtensionOptions,
79 ) -> Result<()> {
80 populate_defaults(extension_manifest, extension_dir)?;
81
82 if extension_dir.is_relative() {
83 bail!(
84 "extension dir {} is not an absolute path",
85 extension_dir.display()
86 );
87 }
88
89 fs::create_dir_all(&self.cache_dir).context("failed to create cache dir")?;
90
91 if extension_manifest.lib.kind == Some(ExtensionLibraryKind::Rust) {
92 log::info!("compiling Rust extension {}", extension_dir.display());
93 self.compile_rust_extension(extension_dir, extension_manifest, options)
94 .await
95 .context("failed to compile Rust extension")?;
96 log::info!("compiled Rust extension {}", extension_dir.display());
97 }
98
99 for (grammar_name, grammar_metadata) in &extension_manifest.grammars {
100 log::info!(
101 "compiling grammar {grammar_name} for extension {}",
102 extension_dir.display()
103 );
104 self.compile_grammar(extension_dir, grammar_name.as_ref(), grammar_metadata)
105 .await
106 .with_context(|| format!("failed to compile grammar '{grammar_name}'"))?;
107 log::info!(
108 "compiled grammar {grammar_name} for extension {}",
109 extension_dir.display()
110 );
111 }
112
113 log::info!("finished compiling extension {}", extension_dir.display());
114 Ok(())
115 }
116
117 async fn compile_rust_extension(
118 &self,
119 extension_dir: &Path,
120 manifest: &mut ExtensionManifest,
121 options: CompileExtensionOptions,
122 ) -> Result<(), anyhow::Error> {
123 self.install_rust_wasm_target_if_needed()?;
124 let adapter_bytes = self.install_wasi_preview1_adapter_if_needed().await?;
125
126 let cargo_toml_content = fs::read_to_string(extension_dir.join("Cargo.toml"))?;
127 let cargo_toml: CargoToml = toml::from_str(&cargo_toml_content)?;
128
129 log::info!(
130 "compiling Rust crate for extension {}",
131 extension_dir.display()
132 );
133 let output = util::command::new_std_command("cargo")
134 .args(["build", "--target", RUST_TARGET])
135 .args(options.release.then_some("--release"))
136 .arg("--target-dir")
137 .arg(extension_dir.join("target"))
138 // WASI builds do not work with sccache and just stuck, so disable it.
139 .env("RUSTC_WRAPPER", "")
140 .current_dir(extension_dir)
141 .output()
142 .context("failed to run `cargo`")?;
143 if !output.status.success() {
144 bail!(
145 "failed to build extension {}",
146 String::from_utf8_lossy(&output.stderr)
147 );
148 }
149
150 log::info!(
151 "compiled Rust crate for extension {}",
152 extension_dir.display()
153 );
154
155 let mut wasm_path = PathBuf::from(extension_dir);
156 wasm_path.extend([
157 "target",
158 RUST_TARGET,
159 if options.release { "release" } else { "debug" },
160 &cargo_toml
161 .package
162 .name
163 // The wasm32-wasip1 target normalizes `-` in package names to `_` in the resulting `.wasm` file.
164 .replace('-', "_"),
165 ]);
166 wasm_path.set_extension("wasm");
167
168 let wasm_bytes = fs::read(&wasm_path)
169 .with_context(|| format!("failed to read output module `{}`", wasm_path.display()))?;
170
171 let mut encoder = ComponentEncoder::default()
172 .module(&wasm_bytes)?
173 .adapter("wasi_snapshot_preview1", &adapter_bytes)
174 .context("failed to load adapter module")?
175 .validate(true);
176
177 log::info!(
178 "encoding wasm component for extension {}",
179 extension_dir.display()
180 );
181
182 let component_bytes = encoder
183 .encode()
184 .context("failed to encode wasm component")?;
185
186 let component_bytes = self
187 .strip_custom_sections(&component_bytes)
188 .context("failed to strip debug sections from wasm component")?;
189
190 let wasm_extension_api_version =
191 parse_wasm_extension_version(&manifest.id, &component_bytes)
192 .context("compiled wasm did not contain a valid zed extension api version")?;
193 manifest.lib.version = Some(wasm_extension_api_version);
194
195 let extension_file = extension_dir.join("extension.wasm");
196 fs::write(extension_file.clone(), &component_bytes)
197 .context("failed to write extension.wasm")?;
198
199 log::info!(
200 "extension {} written to {}",
201 extension_dir.display(),
202 extension_file.display()
203 );
204
205 Ok(())
206 }
207
208 async fn compile_grammar(
209 &self,
210 extension_dir: &Path,
211 grammar_name: &str,
212 grammar_metadata: &GrammarManifestEntry,
213 ) -> Result<()> {
214 let clang_path = self.install_wasi_sdk_if_needed().await?;
215
216 let mut grammar_repo_dir = extension_dir.to_path_buf();
217 grammar_repo_dir.extend(["grammars", grammar_name]);
218
219 let mut grammar_wasm_path = grammar_repo_dir.clone();
220 grammar_wasm_path.set_extension("wasm");
221
222 log::info!("checking out {grammar_name} parser");
223 self.checkout_repo(
224 &grammar_repo_dir,
225 &grammar_metadata.repository,
226 &grammar_metadata.rev,
227 )?;
228
229 let base_grammar_path = grammar_metadata
230 .path
231 .as_ref()
232 .map(|path| grammar_repo_dir.join(path))
233 .unwrap_or(grammar_repo_dir);
234
235 let src_path = base_grammar_path.join("src");
236 let parser_path = src_path.join("parser.c");
237 let scanner_path = src_path.join("scanner.c");
238
239 log::info!("compiling {grammar_name} parser");
240 let clang_output = util::command::new_std_command(&clang_path)
241 .args(["-fPIC", "-shared", "-Os"])
242 .arg(format!("-Wl,--export=tree_sitter_{grammar_name}"))
243 .arg("-o")
244 .arg(&grammar_wasm_path)
245 .arg("-I")
246 .arg(&src_path)
247 .arg(&parser_path)
248 .args(scanner_path.exists().then_some(scanner_path))
249 .output()
250 .context("failed to run clang")?;
251
252 if !clang_output.status.success() {
253 bail!(
254 "failed to compile {} parser with clang: {}",
255 grammar_name,
256 String::from_utf8_lossy(&clang_output.stderr),
257 );
258 }
259
260 Ok(())
261 }
262
263 fn checkout_repo(&self, directory: &Path, url: &str, rev: &str) -> Result<()> {
264 let git_dir = directory.join(".git");
265
266 if directory.exists() {
267 let remotes_output = util::command::new_std_command("git")
268 .arg("--git-dir")
269 .arg(&git_dir)
270 .args(["remote", "-v"])
271 .output()?;
272 let has_remote = remotes_output.status.success()
273 && String::from_utf8_lossy(&remotes_output.stdout)
274 .lines()
275 .any(|line| {
276 let mut parts = line.split(|c: char| c.is_whitespace());
277 parts.next() == Some("origin") && parts.any(|part| part == url)
278 });
279 if !has_remote {
280 bail!(
281 "grammar directory '{}' already exists, but is not a git clone of '{}'",
282 directory.display(),
283 url
284 );
285 }
286 } else {
287 fs::create_dir_all(directory).with_context(|| {
288 format!("failed to create grammar directory {}", directory.display(),)
289 })?;
290 let init_output = util::command::new_std_command("git")
291 .arg("init")
292 .current_dir(directory)
293 .output()?;
294 if !init_output.status.success() {
295 bail!(
296 "failed to run `git init` in directory '{}'",
297 directory.display()
298 );
299 }
300
301 let remote_add_output = util::command::new_std_command("git")
302 .arg("--git-dir")
303 .arg(&git_dir)
304 .args(["remote", "add", "origin", url])
305 .output()
306 .context("failed to execute `git remote add`")?;
307 if !remote_add_output.status.success() {
308 bail!(
309 "failed to add remote {url} for git repository {}",
310 git_dir.display()
311 );
312 }
313 }
314
315 let fetch_output = util::command::new_std_command("git")
316 .arg("--git-dir")
317 .arg(&git_dir)
318 .args(["fetch", "--depth", "1", "origin", rev])
319 .output()
320 .context("failed to execute `git fetch`")?;
321
322 let checkout_output = util::command::new_std_command("git")
323 .arg("--git-dir")
324 .arg(&git_dir)
325 .args(["checkout", rev])
326 .current_dir(directory)
327 .output()
328 .context("failed to execute `git checkout`")?;
329 if !checkout_output.status.success() {
330 if !fetch_output.status.success() {
331 bail!(
332 "failed to fetch revision {} in directory '{}'",
333 rev,
334 directory.display()
335 );
336 }
337 bail!(
338 "failed to checkout revision {} in directory '{}': {}",
339 rev,
340 directory.display(),
341 String::from_utf8_lossy(&checkout_output.stderr)
342 );
343 }
344
345 Ok(())
346 }
347
348 fn install_rust_wasm_target_if_needed(&self) -> Result<()> {
349 let rustc_output = util::command::new_std_command("rustc")
350 .arg("--print")
351 .arg("sysroot")
352 .output()
353 .context("failed to run rustc")?;
354 if !rustc_output.status.success() {
355 bail!(
356 "failed to retrieve rust sysroot: {}",
357 String::from_utf8_lossy(&rustc_output.stderr)
358 );
359 }
360
361 let sysroot = PathBuf::from(String::from_utf8(rustc_output.stdout)?.trim());
362 if sysroot.join("lib/rustlib").join(RUST_TARGET).exists() {
363 return Ok(());
364 }
365
366 let output = util::command::new_std_command("rustup")
367 .args(["target", "add", RUST_TARGET])
368 .stderr(Stdio::piped())
369 .stdout(Stdio::inherit())
370 .output()
371 .context("failed to run `rustup target add`")?;
372 if !output.status.success() {
373 bail!(
374 "failed to install the `{RUST_TARGET}` target: {}",
375 String::from_utf8_lossy(&rustc_output.stderr)
376 );
377 }
378
379 Ok(())
380 }
381
382 async fn install_wasi_preview1_adapter_if_needed(&self) -> Result<Vec<u8>> {
383 let cache_path = self.cache_dir.join("wasi_snapshot_preview1.reactor.wasm");
384 if let Ok(content) = fs::read(&cache_path) {
385 if Parser::is_core_wasm(&content) {
386 return Ok(content);
387 }
388 }
389
390 fs::remove_file(&cache_path).ok();
391
392 log::info!(
393 "downloading wasi adapter module to {}",
394 cache_path.display()
395 );
396 let mut response = self
397 .http
398 .get(WASI_ADAPTER_URL, AsyncBody::default(), true)
399 .await?;
400
401 let mut content = Vec::new();
402 let mut body = BufReader::new(response.body_mut());
403 body.read_to_end(&mut content).await?;
404
405 fs::write(&cache_path, &content)
406 .with_context(|| format!("failed to save file {}", cache_path.display()))?;
407
408 if !Parser::is_core_wasm(&content) {
409 bail!("downloaded wasi adapter is invalid");
410 }
411 Ok(content)
412 }
413
414 async fn install_wasi_sdk_if_needed(&self) -> Result<PathBuf> {
415 let url = if let Some(asset_name) = WASI_SDK_ASSET_NAME {
416 format!("{WASI_SDK_URL}/{asset_name}")
417 } else {
418 bail!("wasi-sdk is not available for platform {}", env::consts::OS);
419 };
420
421 let wasi_sdk_dir = self.cache_dir.join("wasi-sdk");
422 let mut clang_path = wasi_sdk_dir.clone();
423 clang_path.extend(["bin", &format!("clang{}", env::consts::EXE_SUFFIX)]);
424
425 if fs::metadata(&clang_path).map_or(false, |metadata| metadata.is_file()) {
426 return Ok(clang_path);
427 }
428
429 let mut tar_out_dir = wasi_sdk_dir.clone();
430 tar_out_dir.set_extension("archive");
431
432 fs::remove_dir_all(&wasi_sdk_dir).ok();
433 fs::remove_dir_all(&tar_out_dir).ok();
434
435 log::info!("downloading wasi-sdk to {}", wasi_sdk_dir.display());
436 let mut response = self.http.get(&url, AsyncBody::default(), true).await?;
437 let body = BufReader::new(response.body_mut());
438 let body = GzipDecoder::new(body);
439 let tar = Archive::new(body);
440
441 tar.unpack(&tar_out_dir)
442 .await
443 .context("failed to unpack wasi-sdk archive")?;
444
445 let inner_dir = fs::read_dir(&tar_out_dir)?
446 .next()
447 .ok_or_else(|| anyhow!("no content"))?
448 .context("failed to read contents of extracted wasi archive directory")?
449 .path();
450 fs::rename(&inner_dir, &wasi_sdk_dir).context("failed to move extracted wasi dir")?;
451 fs::remove_dir_all(&tar_out_dir).ok();
452
453 Ok(clang_path)
454 }
455
456 // This was adapted from:
457 // https://github.com/bytecodealliance/wasm-tools/blob/1791a8f139722e9f8679a2bd3d8e423e55132b22/src/bin/wasm-tools/strip.rs
458 fn strip_custom_sections(&self, input: &Vec<u8>) -> Result<Vec<u8>> {
459 use wasmparser::Payload::*;
460
461 let strip_custom_section = |name: &str| name.starts_with(".debug");
462
463 let mut output = Vec::new();
464 let mut stack = Vec::new();
465
466 for payload in Parser::new(0).parse_all(input) {
467 let payload = payload?;
468 let component_header = wasm_encoder::Component::HEADER;
469 let module_header = wasm_encoder::Module::HEADER;
470
471 // Track nesting depth, so that we don't mess with inner producer sections:
472 match payload {
473 Version { encoding, .. } => {
474 output.extend_from_slice(match encoding {
475 wasmparser::Encoding::Component => &component_header,
476 wasmparser::Encoding::Module => &module_header,
477 });
478 }
479 ModuleSection { .. } | ComponentSection { .. } => {
480 stack.push(mem::take(&mut output));
481 continue;
482 }
483 End { .. } => {
484 let mut parent = match stack.pop() {
485 Some(c) => c,
486 None => break,
487 };
488 if output.starts_with(&component_header) {
489 parent.push(ComponentSectionId::Component as u8);
490 output.encode(&mut parent);
491 } else {
492 parent.push(ComponentSectionId::CoreModule as u8);
493 output.encode(&mut parent);
494 }
495 output = parent;
496 }
497 _ => {}
498 }
499
500 if let CustomSection(c) = &payload {
501 if strip_custom_section(c.name()) {
502 continue;
503 }
504 }
505
506 if let Some((id, range)) = payload.as_section() {
507 RawSection {
508 id,
509 data: &input[range],
510 }
511 .append_to(&mut output);
512 }
513 }
514
515 Ok(output)
516 }
517}
518
519fn populate_defaults(manifest: &mut ExtensionManifest, extension_path: &Path) -> Result<()> {
520 // For legacy extensions on the v0 schema (aka, using `extension.json`), clear out any existing
521 // contents of the computed fields, since we don't care what the existing values are.
522 if manifest.schema_version.is_v0() {
523 manifest.languages.clear();
524 manifest.grammars.clear();
525 manifest.themes.clear();
526 }
527
528 let cargo_toml_path = extension_path.join("Cargo.toml");
529 if cargo_toml_path.exists() {
530 manifest.lib.kind = Some(ExtensionLibraryKind::Rust);
531 }
532
533 let languages_dir = extension_path.join("languages");
534 if languages_dir.exists() {
535 for entry in fs::read_dir(&languages_dir).context("failed to list languages dir")? {
536 let entry = entry?;
537 let language_dir = entry.path();
538 let config_path = language_dir.join("config.toml");
539 if config_path.exists() {
540 let relative_language_dir =
541 language_dir.strip_prefix(extension_path)?.to_path_buf();
542 if !manifest.languages.contains(&relative_language_dir) {
543 manifest.languages.push(relative_language_dir);
544 }
545 }
546 }
547 }
548
549 let themes_dir = extension_path.join("themes");
550 if themes_dir.exists() {
551 for entry in fs::read_dir(&themes_dir).context("failed to list themes dir")? {
552 let entry = entry?;
553 let theme_path = entry.path();
554 if theme_path.extension() == Some("json".as_ref()) {
555 let relative_theme_path = theme_path.strip_prefix(extension_path)?.to_path_buf();
556 if !manifest.themes.contains(&relative_theme_path) {
557 manifest.themes.push(relative_theme_path);
558 }
559 }
560 }
561 }
562
563 let icon_themes_dir = extension_path.join("icon_themes");
564 if icon_themes_dir.exists() {
565 for entry in fs::read_dir(&icon_themes_dir).context("failed to list icon themes dir")? {
566 let entry = entry?;
567 let icon_theme_path = entry.path();
568 if icon_theme_path.extension() == Some("json".as_ref()) {
569 let relative_icon_theme_path =
570 icon_theme_path.strip_prefix(extension_path)?.to_path_buf();
571 if !manifest.icon_themes.contains(&relative_icon_theme_path) {
572 manifest.icon_themes.push(relative_icon_theme_path);
573 }
574 }
575 }
576 }
577
578 let snippets_json_path = extension_path.join("snippets.json");
579 if snippets_json_path.exists() {
580 manifest.snippets = Some(snippets_json_path);
581 }
582
583 // For legacy extensions on the v0 schema (aka, using `extension.json`), we want to populate the grammars in
584 // the manifest using the contents of the `grammars` directory.
585 if manifest.schema_version.is_v0() {
586 let grammars_dir = extension_path.join("grammars");
587 if grammars_dir.exists() {
588 for entry in fs::read_dir(&grammars_dir).context("failed to list grammars dir")? {
589 let entry = entry?;
590 let grammar_path = entry.path();
591 if grammar_path.extension() == Some("toml".as_ref()) {
592 #[derive(Deserialize)]
593 struct GrammarConfigToml {
594 pub repository: String,
595 pub commit: String,
596 #[serde(default)]
597 pub path: Option<String>,
598 }
599
600 let grammar_config = fs::read_to_string(&grammar_path)?;
601 let grammar_config: GrammarConfigToml = toml::from_str(&grammar_config)?;
602
603 let grammar_name = grammar_path
604 .file_stem()
605 .and_then(|stem| stem.to_str())
606 .ok_or_else(|| anyhow!("no grammar name"))?;
607 if !manifest.grammars.contains_key(grammar_name) {
608 manifest.grammars.insert(
609 grammar_name.into(),
610 GrammarManifestEntry {
611 repository: grammar_config.repository,
612 rev: grammar_config.commit,
613 path: grammar_config.path,
614 },
615 );
616 }
617 }
618 }
619 }
620 }
621
622 Ok(())
623}