From 8475280eb11fa79b3388073967ec8c3beb001a52 Mon Sep 17 00:00:00 2001 From: Finn Evers Date: Mon, 9 Mar 2026 11:47:12 +0100 Subject: [PATCH] extension_cli: Add tests for semantic token rules and language tasks (#50750) This adds checks to the extension CLI to ensure that tasks and semantic token rules are actually valid for the compiled extensions. Release Notes: - N/A --- Cargo.lock | 2 + crates/extension/src/extension_builder.rs | 3 +- crates/extension_cli/Cargo.toml | 2 + crates/extension_cli/src/main.rs | 61 ++++++++++++++++----- crates/extension_host/src/extension_host.rs | 29 ++++------ crates/extension_host/src/headless_host.rs | 4 +- crates/language/Cargo.toml | 1 + crates/language/src/language.rs | 9 +++ crates/settings_content/src/project.rs | 26 ++++++++- crates/task/src/task_template.rs | 1 + 10 files changed, 102 insertions(+), 36 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2436baad07e78670837490cf8e9bc897ba0b6716..6cfbab0d585fe93d7b984f674475dfbc411ca14b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6082,7 +6082,9 @@ dependencies = [ "serde", "serde_json", "serde_json_lenient", + "settings_content", "snippet_provider", + "task", "theme", "tokio", "toml 0.8.23", diff --git a/crates/extension/src/extension_builder.rs b/crates/extension/src/extension_builder.rs index eae51846f164d4aa6baf2fac897d25a8961b4d6c..1c204398c34728cab6b05687050243b4a988902c 100644 --- a/crates/extension/src/extension_builder.rs +++ b/crates/extension/src/extension_builder.rs @@ -7,6 +7,7 @@ use anyhow::{Context as _, Result, bail}; use futures::{StreamExt, io}; use heck::ToSnakeCase; use http_client::{self, AsyncBody, HttpClient}; +use language::LanguageConfig; use serde::Deserialize; use std::{ env, fs, mem, @@ -583,7 +584,7 @@ async fn populate_defaults( while let Some(language_dir) = language_dir_entries.next().await { let language_dir = language_dir?; - let config_path = language_dir.join("config.toml"); + let config_path = language_dir.join(LanguageConfig::FILE_NAME); if fs.is_file(config_path.as_path()).await { let relative_language_dir = language_dir.strip_prefix(extension_path)?.to_path_buf(); diff --git a/crates/extension_cli/Cargo.toml b/crates/extension_cli/Cargo.toml index 9795c13e75864184299fba026f499bbcbefee117..24ea9cfafadc61b2753f7b739fd4b7cbbd24dbfe 100644 --- a/crates/extension_cli/Cargo.toml +++ b/crates/extension_cli/Cargo.toml @@ -26,7 +26,9 @@ reqwest_client.workspace = true serde.workspace = true serde_json.workspace = true serde_json_lenient.workspace = true +settings_content.workspace = true snippet_provider.workspace = true +task.workspace = true theme.workspace = true tokio = { workspace = true, features = ["full"] } toml.workspace = true diff --git a/crates/extension_cli/src/main.rs b/crates/extension_cli/src/main.rs index baefb72fe4bd986edbfaa866e50663b159eff3c9..d0a533bfeb331c196d802df9894e726201794ce7 100644 --- a/crates/extension_cli/src/main.rs +++ b/crates/extension_cli/src/main.rs @@ -11,8 +11,10 @@ use extension::extension_builder::{CompileExtensionOptions, ExtensionBuilder}; use extension::{ExtensionManifest, ExtensionSnippets}; use language::LanguageConfig; use reqwest_client::ReqwestClient; +use settings_content::SemanticTokenRules; use snippet_provider::file_to_snippets; use snippet_provider::format::VsSnippetsFile; +use task::TaskTemplates; use tokio::process::Command; use tree_sitter::{Language, Query, WasmStore}; @@ -323,9 +325,8 @@ fn test_languages( ) -> Result<()> { for relative_language_dir in &manifest.languages { let language_dir = extension_path.join(relative_language_dir); - let config_path = language_dir.join("config.toml"); - let config_content = fs::read_to_string(&config_path)?; - let config: LanguageConfig = toml::from_str(&config_content)?; + let config_path = language_dir.join(LanguageConfig::FILE_NAME); + let config = LanguageConfig::load(&config_path)?; let grammar = if let Some(name) = &config.grammar { Some( grammars @@ -339,18 +340,48 @@ fn test_languages( let query_entries = fs::read_dir(&language_dir)?; for entry in query_entries { let entry = entry?; - let query_path = entry.path(); - if query_path.extension() == Some("scm".as_ref()) { - let grammar = grammar.with_context(|| { - format! { - "language {} provides query {} but no grammar", - config.name, - query_path.display() - } - })?; - - let query_source = fs::read_to_string(&query_path)?; - let _query = Query::new(grammar, &query_source)?; + let file_path = entry.path(); + + let Some(file_name) = file_path.file_name().and_then(|name| name.to_str()) else { + continue; + }; + + match file_name { + LanguageConfig::FILE_NAME => { + // Loaded above + } + SemanticTokenRules::FILE_NAME => { + let _token_rules = SemanticTokenRules::load(&file_path)?; + } + TaskTemplates::FILE_NAME => { + let task_file_content = std::fs::read(&file_path).with_context(|| { + anyhow!( + "Failed to read tasks file at {path}", + path = file_path.display() + ) + })?; + let _task_templates = + serde_json_lenient::from_slice::(&task_file_content) + .with_context(|| { + anyhow!( + "Failed to parse tasks file at {path}", + path = file_path.display() + ) + })?; + } + _ if file_name.ends_with(".scm") => { + let grammar = grammar.with_context(|| { + format! { + "language {} provides query {} but no grammar", + config.name, + file_path.display() + } + })?; + + let query_source = fs::read_to_string(&file_path)?; + let _query = Query::new(grammar, &query_source)?; + } + _ => {} } } diff --git a/crates/extension_host/src/extension_host.rs b/crates/extension_host/src/extension_host.rs index c691296d61183c9bb0fcd41ff6c74eed6cb61149..5418f630537c1acd98edc8c6af753d9358b23e8f 100644 --- a/crates/extension_host/src/extension_host.rs +++ b/crates/extension_host/src/extension_host.rs @@ -55,6 +55,7 @@ use std::{ sync::Arc, time::{Duration, Instant}, }; +use task::TaskTemplates; use url::Url; use util::{ResultExt, paths::RemotePathBuf}; use wasm_host::{ @@ -1285,19 +1286,11 @@ impl ExtensionStore { ]); // Load semantic token rules if present in the language directory. - let rules_path = language_path.join("semantic_token_rules.json"); - if let Ok(rules_json) = std::fs::read_to_string(&rules_path) { - match serde_json_lenient::from_str::(&rules_json) { - Ok(rules) => { - semantic_token_rules_to_add.push((language_name.clone(), rules)); - } - Err(err) => { - log::error!( - "Failed to parse semantic token rules from {}: {err:#}", - rules_path.display() - ); - } - } + let rules_path = language_path.join(SemanticTokenRules::FILE_NAME); + if std::fs::exists(&rules_path).is_ok_and(|exists| exists) + && let Some(rules) = SemanticTokenRules::load(&rules_path).log_err() + { + semantic_token_rules_to_add.push((language_name.clone(), rules)); } self.proxy.register_language( @@ -1306,11 +1299,11 @@ impl ExtensionStore { language.matcher.clone(), language.hidden, Arc::new(move || { - let config = std::fs::read_to_string(language_path.join("config.toml"))?; - let config: LanguageConfig = ::toml::from_str(&config)?; + let config = + LanguageConfig::load(language_path.join(LanguageConfig::FILE_NAME))?; let queries = load_plugin_queries(&language_path); let context_provider = - std::fs::read_to_string(language_path.join("tasks.json")) + std::fs::read_to_string(language_path.join(TaskTemplates::FILE_NAME)) .ok() .and_then(|contents| { let definitions = @@ -1580,7 +1573,7 @@ impl ExtensionStore { if !fs_metadata.is_dir { continue; } - let language_config_path = language_path.join("config.toml"); + let language_config_path = language_path.join(LanguageConfig::FILE_NAME); let config = fs.load(&language_config_path).await.with_context(|| { format!("loading language config from {language_config_path:?}") })?; @@ -1703,7 +1696,7 @@ impl ExtensionStore { cx.background_spawn(async move { const EXTENSION_TOML: &str = "extension.toml"; const EXTENSION_WASM: &str = "extension.wasm"; - const CONFIG_TOML: &str = "config.toml"; + const CONFIG_TOML: &str = LanguageConfig::FILE_NAME; if is_dev { let manifest_toml = toml::to_string(&loaded_extension.manifest)?; diff --git a/crates/extension_host/src/headless_host.rs b/crates/extension_host/src/headless_host.rs index 290dbb6fd40fc3c15dcb210c767b9102b7117544..0aff06fdddcf5c075bd669528b5c52137f745863 100644 --- a/crates/extension_host/src/headless_host.rs +++ b/crates/extension_host/src/headless_host.rs @@ -138,7 +138,9 @@ impl HeadlessExtensionStore { for language_path in &manifest.languages { let language_path = extension_dir.join(language_path); - let config = fs.load(&language_path.join("config.toml")).await?; + let config = fs + .load(&language_path.join(LanguageConfig::FILE_NAME)) + .await?; let mut config = ::toml::from_str::(&config)?; this.update(cx, |this, _cx| { diff --git a/crates/language/Cargo.toml b/crates/language/Cargo.toml index 58db79afe59f0e6d27e23eceb9861ea493d853fd..37c19172f7c48743e1436ba41e30d0c7ebf99d1d 100644 --- a/crates/language/Cargo.toml +++ b/crates/language/Cargo.toml @@ -62,6 +62,7 @@ sum_tree.workspace = true task.workspace = true text.workspace = true theme.workspace = true +toml.workspace = true tracing.workspace = true tree-sitter-md = { workspace = true, optional = true } tree-sitter-python = { workspace = true, optional = true } diff --git a/crates/language/src/language.rs b/crates/language/src/language.rs index 29b569ba1aa68fe83f3456a2eaf9911b4c83677d..4e994a7e60f58b6e4ccd50c2cb0584f91bd351f2 100644 --- a/crates/language/src/language.rs +++ b/crates/language/src/language.rs @@ -961,6 +961,15 @@ pub struct LanguageConfig { pub import_path_strip_regex: Option, } +impl LanguageConfig { + pub const FILE_NAME: &str = "config.toml"; + + pub fn load(config_path: impl AsRef) -> Result { + let config = std::fs::read_to_string(config_path.as_ref())?; + toml::from_str(&config).map_err(Into::into) + } +} + #[derive(Clone, Debug, Deserialize, Default, JsonSchema)] pub struct DecreaseIndentConfig { #[serde(default, deserialize_with = "deserialize_regex")] diff --git a/crates/settings_content/src/project.rs b/crates/settings_content/src/project.rs index 70544646b1878c163bf5c17d2364eeebd98f6908..85a39f389efc621e902154431278c2050c81a210 100644 --- a/crates/settings_content/src/project.rs +++ b/crates/settings_content/src/project.rs @@ -1,5 +1,9 @@ -use std::{path::PathBuf, sync::Arc}; +use std::{ + path::{Path, PathBuf}, + sync::Arc, +}; +use anyhow::Context; use collections::{BTreeMap, HashMap}; use gpui::Rgba; use schemars::JsonSchema; @@ -233,6 +237,26 @@ pub struct SemanticTokenRules { pub rules: Vec, } +impl SemanticTokenRules { + pub const FILE_NAME: &'static str = "semantic_token_rules.json"; + + pub fn load(file_path: &Path) -> anyhow::Result { + let rules_content = std::fs::read(file_path).with_context(|| { + anyhow::anyhow!( + "Could not read semantic token rules from {}", + file_path.display() + ) + })?; + + serde_json_lenient::from_slice::(&rules_content).with_context(|| { + anyhow::anyhow!( + "Failed to parse semantic token rules from {}", + file_path.display() + ) + }) + } +} + impl crate::merge_from::MergeFrom for SemanticTokenRules { fn merge_from(&mut self, other: &Self) { self.rules.splice(0..0, other.rules.iter().cloned()); diff --git a/crates/task/src/task_template.rs b/crates/task/src/task_template.rs index 539b2779cc85b5830af90aeb4ffd28596c2c29c3..a85c3565e2869e10f093a47f71024384e496fbd2 100644 --- a/crates/task/src/task_template.rs +++ b/crates/task/src/task_template.rs @@ -114,6 +114,7 @@ pub enum HideStrategy { pub struct TaskTemplates(pub Vec); impl TaskTemplates { + pub const FILE_NAME: &str = "tasks.json"; /// Generates JSON schema of Tasks JSON template format. pub fn generate_json_schema() -> serde_json::Value { let schema = schemars::generate::SchemaSettings::draft2019_09()