1use crate::wasm_host::parse_wasm_extension_version;
2use crate::ExtensionManifest;
3use crate::{extension_manifest::ExtensionLibraryKind, GrammarManifestEntry};
4use anyhow::{anyhow, bail, Context as _, Result};
5use async_compression::futures::bufread::GzipDecoder;
6use async_tar::Archive;
7use futures::io::BufReader;
8use futures::AsyncReadExt;
9use http::{self, AsyncBody, HttpClient};
10use serde::Deserialize;
11use std::{
12 env, fs, mem,
13 path::{Path, PathBuf},
14 process::{Command, Stdio},
15 sync::Arc,
16};
17use wasm_encoder::{ComponentSectionId, Encode as _, RawSection, Section as _};
18use wasmparser::Parser;
19use wit_component::ComponentEncoder;
20
21/// Currently, we compile with Rust's `wasm32-wasi` target, which works with WASI `preview1`.
22/// But the WASM component model is based on WASI `preview2`. So we need an 'adapter' WASM
23/// module, which implements the `preview1` interface in terms of `preview2`.
24///
25/// Once Rust 1.78 is released, there will be a `wasm32-wasip2` target available, so we will
26/// not need the adapter anymore.
27const RUST_TARGET: &str = "wasm32-wasi";
28const WASI_ADAPTER_URL: &str =
29 "https://github.com/bytecodealliance/wasmtime/releases/download/v18.0.2/wasi_snapshot_preview1.reactor.wasm";
30
31/// Compiling Tree-sitter parsers from C to WASM requires Clang 17, and a WASM build of libc
32/// and clang's runtime library. The `wasi-sdk` provides these binaries.
33///
34/// Once Clang 17 and its wasm target are available via system package managers, we won't need
35/// to download this.
36const WASI_SDK_URL: &str = "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-21/";
37const WASI_SDK_ASSET_NAME: Option<&str> = if cfg!(target_os = "macos") {
38 Some("wasi-sdk-21.0-macos.tar.gz")
39} else if cfg!(target_os = "linux") {
40 Some("wasi-sdk-21.0-linux.tar.gz")
41} else if cfg!(target_os = "windows") {
42 Some("wasi-sdk-21.0.m-mingw.tar.gz")
43} else {
44 None
45};
46
47pub struct ExtensionBuilder {
48 cache_dir: PathBuf,
49 pub http: Arc<dyn HttpClient>,
50}
51
52pub struct CompileExtensionOptions {
53 pub release: bool,
54}
55
56#[derive(Deserialize)]
57struct CargoToml {
58 package: CargoTomlPackage,
59}
60
61#[derive(Deserialize)]
62struct CargoTomlPackage {
63 name: String,
64}
65
66impl ExtensionBuilder {
67 pub fn new(cache_dir: PathBuf) -> Self {
68 Self {
69 cache_dir,
70 http: http::client(None),
71 }
72 }
73
74 pub async fn compile_extension(
75 &self,
76 extension_dir: &Path,
77 extension_manifest: &mut ExtensionManifest,
78 options: CompileExtensionOptions,
79 ) -> Result<()> {
80 populate_defaults(extension_manifest, &extension_dir)?;
81
82 if extension_dir.is_relative() {
83 bail!(
84 "extension dir {} is not an absolute path",
85 extension_dir.display()
86 );
87 }
88
89 fs::create_dir_all(&self.cache_dir).context("failed to create cache dir")?;
90
91 if extension_manifest.lib.kind == Some(ExtensionLibraryKind::Rust) {
92 log::info!("compiling Rust extension {}", extension_dir.display());
93 self.compile_rust_extension(extension_dir, extension_manifest, options)
94 .await
95 .context("failed to compile Rust extension")?;
96 }
97
98 for (grammar_name, grammar_metadata) in &extension_manifest.grammars {
99 self.compile_grammar(extension_dir, grammar_name.as_ref(), grammar_metadata)
100 .await
101 .with_context(|| format!("failed to compile grammar '{grammar_name}'"))?;
102 }
103
104 log::info!("finished compiling extension {}", extension_dir.display());
105 Ok(())
106 }
107
108 async fn compile_rust_extension(
109 &self,
110 extension_dir: &Path,
111 manifest: &mut ExtensionManifest,
112 options: CompileExtensionOptions,
113 ) -> Result<(), anyhow::Error> {
114 self.install_rust_wasm_target_if_needed()?;
115 let adapter_bytes = self.install_wasi_preview1_adapter_if_needed().await?;
116
117 let cargo_toml_content = fs::read_to_string(&extension_dir.join("Cargo.toml"))?;
118 let cargo_toml: CargoToml = toml::from_str(&cargo_toml_content)?;
119
120 log::info!("compiling rust extension {}", extension_dir.display());
121 let output = Command::new("cargo")
122 .args(["build", "--target", RUST_TARGET])
123 .args(options.release.then_some("--release"))
124 .arg("--target-dir")
125 .arg(extension_dir.join("target"))
126 .current_dir(&extension_dir)
127 .output()
128 .context("failed to run `cargo`")?;
129 if !output.status.success() {
130 bail!(
131 "failed to build extension {}",
132 String::from_utf8_lossy(&output.stderr)
133 );
134 }
135
136 let mut wasm_path = PathBuf::from(extension_dir);
137 wasm_path.extend([
138 "target",
139 RUST_TARGET,
140 if options.release { "release" } else { "debug" },
141 &cargo_toml
142 .package
143 .name
144 // The wasm32-wasi target normalizes `-` in package names to `_` in the resulting `.wasm` file.
145 .replace('-', "_"),
146 ]);
147 wasm_path.set_extension("wasm");
148
149 let wasm_bytes = fs::read(&wasm_path)
150 .with_context(|| format!("failed to read output module `{}`", wasm_path.display()))?;
151
152 let encoder = ComponentEncoder::default()
153 .module(&wasm_bytes)?
154 .adapter("wasi_snapshot_preview1", &adapter_bytes)
155 .context("failed to load adapter module")?
156 .validate(true);
157
158 let component_bytes = encoder
159 .encode()
160 .context("failed to encode wasm component")?;
161
162 let component_bytes = self
163 .strip_custom_sections(&component_bytes)
164 .context("failed to strip debug sections from wasm component")?;
165
166 let wasm_extension_api_version =
167 parse_wasm_extension_version(&manifest.id, &component_bytes)
168 .context("compiled wasm did not contain a valid zed extension api version")?;
169 manifest.lib.version = Some(wasm_extension_api_version);
170
171 fs::write(extension_dir.join("extension.wasm"), &component_bytes)
172 .context("failed to write extension.wasm")?;
173
174 Ok(())
175 }
176
177 async fn compile_grammar(
178 &self,
179 extension_dir: &Path,
180 grammar_name: &str,
181 grammar_metadata: &GrammarManifestEntry,
182 ) -> Result<()> {
183 let clang_path = self.install_wasi_sdk_if_needed().await?;
184
185 let mut grammar_repo_dir = extension_dir.to_path_buf();
186 grammar_repo_dir.extend(["grammars", grammar_name]);
187
188 let mut grammar_wasm_path = grammar_repo_dir.clone();
189 grammar_wasm_path.set_extension("wasm");
190
191 log::info!("checking out {grammar_name} parser");
192 self.checkout_repo(
193 &grammar_repo_dir,
194 &grammar_metadata.repository,
195 &grammar_metadata.rev,
196 )?;
197
198 let base_grammar_path = grammar_metadata
199 .path
200 .as_ref()
201 .map(|path| grammar_repo_dir.join(path))
202 .unwrap_or(grammar_repo_dir);
203
204 let src_path = base_grammar_path.join("src");
205 let parser_path = src_path.join("parser.c");
206 let scanner_path = src_path.join("scanner.c");
207
208 log::info!("compiling {grammar_name} parser");
209 let clang_output = Command::new(&clang_path)
210 .args(["-fPIC", "-shared", "-Os"])
211 .arg(format!("-Wl,--export=tree_sitter_{grammar_name}"))
212 .arg("-o")
213 .arg(&grammar_wasm_path)
214 .arg("-I")
215 .arg(&src_path)
216 .arg(&parser_path)
217 .args(scanner_path.exists().then_some(scanner_path))
218 .output()
219 .context("failed to run clang")?;
220 if !clang_output.status.success() {
221 bail!(
222 "failed to compile {} parser with clang: {}",
223 grammar_name,
224 String::from_utf8_lossy(&clang_output.stderr),
225 );
226 }
227
228 Ok(())
229 }
230
231 fn checkout_repo(&self, directory: &Path, url: &str, rev: &str) -> Result<()> {
232 let git_dir = directory.join(".git");
233
234 if directory.exists() {
235 let remotes_output = Command::new("git")
236 .arg("--git-dir")
237 .arg(&git_dir)
238 .args(["remote", "-v"])
239 .output()?;
240 let has_remote = remotes_output.status.success()
241 && String::from_utf8_lossy(&remotes_output.stdout)
242 .lines()
243 .any(|line| {
244 let mut parts = line.split(|c: char| c.is_whitespace());
245 parts.next() == Some("origin") && parts.any(|part| part == url)
246 });
247 if !has_remote {
248 bail!(
249 "grammar directory '{}' already exists, but is not a git clone of '{}'",
250 directory.display(),
251 url
252 );
253 }
254 } else {
255 fs::create_dir_all(&directory).with_context(|| {
256 format!("failed to create grammar directory {}", directory.display(),)
257 })?;
258 let init_output = Command::new("git")
259 .arg("init")
260 .current_dir(&directory)
261 .output()?;
262 if !init_output.status.success() {
263 bail!(
264 "failed to run `git init` in directory '{}'",
265 directory.display()
266 );
267 }
268
269 let remote_add_output = Command::new("git")
270 .arg("--git-dir")
271 .arg(&git_dir)
272 .args(["remote", "add", "origin", url])
273 .output()
274 .context("failed to execute `git remote add`")?;
275 if !remote_add_output.status.success() {
276 bail!(
277 "failed to add remote {url} for git repository {}",
278 git_dir.display()
279 );
280 }
281 }
282
283 let fetch_output = Command::new("git")
284 .arg("--git-dir")
285 .arg(&git_dir)
286 .args(["fetch", "--depth", "1", "origin", &rev])
287 .output()
288 .context("failed to execute `git fetch`")?;
289
290 let checkout_output = Command::new("git")
291 .arg("--git-dir")
292 .arg(&git_dir)
293 .args(["checkout", &rev])
294 .current_dir(&directory)
295 .output()
296 .context("failed to execute `git checkout`")?;
297 if !checkout_output.status.success() {
298 if !fetch_output.status.success() {
299 bail!(
300 "failed to fetch revision {} in directory '{}'",
301 rev,
302 directory.display()
303 );
304 }
305 bail!(
306 "failed to checkout revision {} in directory '{}': {}",
307 rev,
308 directory.display(),
309 String::from_utf8_lossy(&checkout_output.stderr)
310 );
311 }
312
313 Ok(())
314 }
315
316 fn install_rust_wasm_target_if_needed(&self) -> Result<()> {
317 let rustc_output = Command::new("rustc")
318 .arg("--print")
319 .arg("sysroot")
320 .output()
321 .context("failed to run rustc")?;
322 if !rustc_output.status.success() {
323 bail!(
324 "failed to retrieve rust sysroot: {}",
325 String::from_utf8_lossy(&rustc_output.stderr)
326 );
327 }
328
329 let sysroot = PathBuf::from(String::from_utf8(rustc_output.stdout)?.trim());
330 if sysroot.join("lib/rustlib").join(RUST_TARGET).exists() {
331 return Ok(());
332 }
333
334 let output = Command::new("rustup")
335 .args(["target", "add", RUST_TARGET])
336 .stderr(Stdio::inherit())
337 .stdout(Stdio::inherit())
338 .output()
339 .context("failed to run `rustup target add`")?;
340 if !output.status.success() {
341 bail!("failed to install the `{RUST_TARGET}` target");
342 }
343
344 Ok(())
345 }
346
347 async fn install_wasi_preview1_adapter_if_needed(&self) -> Result<Vec<u8>> {
348 let cache_path = self.cache_dir.join("wasi_snapshot_preview1.reactor.wasm");
349 if let Ok(content) = fs::read(&cache_path) {
350 if Parser::is_core_wasm(&content) {
351 return Ok(content);
352 }
353 }
354
355 fs::remove_file(&cache_path).ok();
356
357 log::info!(
358 "downloading wasi adapter module to {}",
359 cache_path.display()
360 );
361 let mut response = self
362 .http
363 .get(WASI_ADAPTER_URL, AsyncBody::default(), true)
364 .await?;
365
366 let mut content = Vec::new();
367 let mut body = BufReader::new(response.body_mut());
368 body.read_to_end(&mut content).await?;
369
370 fs::write(&cache_path, &content)
371 .with_context(|| format!("failed to save file {}", cache_path.display()))?;
372
373 if !Parser::is_core_wasm(&content) {
374 bail!("downloaded wasi adapter is invalid");
375 }
376 Ok(content)
377 }
378
379 async fn install_wasi_sdk_if_needed(&self) -> Result<PathBuf> {
380 let url = if let Some(asset_name) = WASI_SDK_ASSET_NAME {
381 format!("{WASI_SDK_URL}/{asset_name}")
382 } else {
383 bail!("wasi-sdk is not available for platform {}", env::consts::OS);
384 };
385
386 let wasi_sdk_dir = self.cache_dir.join("wasi-sdk");
387 let mut clang_path = wasi_sdk_dir.clone();
388 clang_path.extend(["bin", &format!("clang{}", env::consts::EXE_SUFFIX)]);
389
390 if fs::metadata(&clang_path).map_or(false, |metadata| metadata.is_file()) {
391 return Ok(clang_path);
392 }
393
394 let mut tar_out_dir = wasi_sdk_dir.clone();
395 tar_out_dir.set_extension("archive");
396
397 fs::remove_dir_all(&wasi_sdk_dir).ok();
398 fs::remove_dir_all(&tar_out_dir).ok();
399
400 log::info!("downloading wasi-sdk to {}", wasi_sdk_dir.display());
401 let mut response = self.http.get(&url, AsyncBody::default(), true).await?;
402 let body = BufReader::new(response.body_mut());
403 let body = GzipDecoder::new(body);
404 let tar = Archive::new(body);
405 tar.unpack(&tar_out_dir)
406 .await
407 .context("failed to unpack wasi-sdk archive")?;
408
409 let inner_dir = fs::read_dir(&tar_out_dir)?
410 .next()
411 .ok_or_else(|| anyhow!("no content"))?
412 .context("failed to read contents of extracted wasi archive directory")?
413 .path();
414 fs::rename(&inner_dir, &wasi_sdk_dir).context("failed to move extracted wasi dir")?;
415 fs::remove_dir_all(&tar_out_dir).ok();
416
417 Ok(clang_path)
418 }
419
420 // This was adapted from:
421 // https://github.com/bytecodealliance/wasm-tools/1791a8f139722e9f8679a2bd3d8e423e55132b22/src/bin/wasm-tools/strip.rs
422 fn strip_custom_sections(&self, input: &Vec<u8>) -> Result<Vec<u8>> {
423 use wasmparser::Payload::*;
424
425 let strip_custom_section = |name: &str| name.starts_with(".debug");
426
427 let mut output = Vec::new();
428 let mut stack = Vec::new();
429
430 for payload in Parser::new(0).parse_all(input) {
431 let payload = payload?;
432
433 // Track nesting depth, so that we don't mess with inner producer sections:
434 match payload {
435 Version { encoding, .. } => {
436 output.extend_from_slice(match encoding {
437 wasmparser::Encoding::Component => &wasm_encoder::Component::HEADER,
438 wasmparser::Encoding::Module => &wasm_encoder::Module::HEADER,
439 });
440 }
441 ModuleSection { .. } | ComponentSection { .. } => {
442 stack.push(mem::take(&mut output));
443 continue;
444 }
445 End { .. } => {
446 let mut parent = match stack.pop() {
447 Some(c) => c,
448 None => break,
449 };
450 if output.starts_with(&wasm_encoder::Component::HEADER) {
451 parent.push(ComponentSectionId::Component as u8);
452 output.encode(&mut parent);
453 } else {
454 parent.push(ComponentSectionId::CoreModule as u8);
455 output.encode(&mut parent);
456 }
457 output = parent;
458 }
459 _ => {}
460 }
461
462 match &payload {
463 CustomSection(c) => {
464 if strip_custom_section(c.name()) {
465 continue;
466 }
467 }
468
469 _ => {}
470 }
471
472 if let Some((id, range)) = payload.as_section() {
473 RawSection {
474 id,
475 data: &input[range],
476 }
477 .append_to(&mut output);
478 }
479 }
480
481 Ok(output)
482 }
483}
484
485fn populate_defaults(manifest: &mut ExtensionManifest, extension_path: &Path) -> Result<()> {
486 // For legacy extensions on the v0 schema (aka, using `extension.json`), clear out any existing
487 // contents of the computed fields, since we don't care what the existing values are.
488 if manifest.schema_version.is_v0() {
489 manifest.languages.clear();
490 manifest.grammars.clear();
491 manifest.themes.clear();
492 }
493
494 let cargo_toml_path = extension_path.join("Cargo.toml");
495 if cargo_toml_path.exists() {
496 manifest.lib.kind = Some(ExtensionLibraryKind::Rust);
497 }
498
499 let languages_dir = extension_path.join("languages");
500 if languages_dir.exists() {
501 for entry in fs::read_dir(&languages_dir).context("failed to list languages dir")? {
502 let entry = entry?;
503 let language_dir = entry.path();
504 let config_path = language_dir.join("config.toml");
505 if config_path.exists() {
506 let relative_language_dir =
507 language_dir.strip_prefix(extension_path)?.to_path_buf();
508 if !manifest.languages.contains(&relative_language_dir) {
509 manifest.languages.push(relative_language_dir);
510 }
511 }
512 }
513 }
514
515 let themes_dir = extension_path.join("themes");
516 if themes_dir.exists() {
517 for entry in fs::read_dir(&themes_dir).context("failed to list themes dir")? {
518 let entry = entry?;
519 let theme_path = entry.path();
520 if theme_path.extension() == Some("json".as_ref()) {
521 let relative_theme_path = theme_path.strip_prefix(extension_path)?.to_path_buf();
522 if !manifest.themes.contains(&relative_theme_path) {
523 manifest.themes.push(relative_theme_path);
524 }
525 }
526 }
527 }
528
529 // For legacy extensions on the v0 schema (aka, using `extension.json`), we want to populate the grammars in
530 // the manifest using the contents of the `grammars` directory.
531 if manifest.schema_version.is_v0() {
532 let grammars_dir = extension_path.join("grammars");
533 if grammars_dir.exists() {
534 for entry in fs::read_dir(&grammars_dir).context("failed to list grammars dir")? {
535 let entry = entry?;
536 let grammar_path = entry.path();
537 if grammar_path.extension() == Some("toml".as_ref()) {
538 #[derive(Deserialize)]
539 struct GrammarConfigToml {
540 pub repository: String,
541 pub commit: String,
542 #[serde(default)]
543 pub path: Option<String>,
544 }
545
546 let grammar_config = fs::read_to_string(&grammar_path)?;
547 let grammar_config: GrammarConfigToml = toml::from_str(&grammar_config)?;
548
549 let grammar_name = grammar_path
550 .file_stem()
551 .and_then(|stem| stem.to_str())
552 .ok_or_else(|| anyhow!("no grammar name"))?;
553 if !manifest.grammars.contains_key(grammar_name) {
554 manifest.grammars.insert(
555 grammar_name.into(),
556 GrammarManifestEntry {
557 repository: grammar_config.repository,
558 rev: grammar_config.commit,
559 path: grammar_config.path,
560 },
561 );
562 }
563 }
564 }
565 }
566 }
567
568 Ok(())
569}