Max Brunsfeld and Marshall created
* Store extensions versions' wasm API version in the database
* Share a common struct for extension API responses between collab and
client
* Add wasm API version and schema version to extension API responses
Release Notes:
- N/A
Co-authored-by: Marshall <marshall@zed.dev>Cargo.lock | 2
crates/client/src/telemetry.rs | 10
crates/collab/migrations.sqlite/20221109000000_test_schema.sql | 1
crates/collab/migrations/20240335123500_add_extension_wasm_api_version.sql | 1
crates/collab/src/api/events.rs | 99
crates/collab/src/api/extensions.rs | 10
crates/collab/src/db.rs | 16
crates/collab/src/db/queries/extensions.rs | 92
crates/collab/src/db/tables/extension_version.rs | 1
crates/collab/src/db/tests/extension_tests.rs | 126
crates/extension/Cargo.toml | 1
crates/extension/src/extension_builder.rs | 101
crates/extension/src/extension_manifest.rs | 75
crates/extension/src/extension_store.rs | 108
crates/extension/src/extension_store_test.rs | 3
crates/extension/src/wasm_host.rs | 49
crates/extension_cli/src/main.rs | 92
crates/rpc/Cargo.toml | 1
crates/rpc/src/extension.rs | 13
crates/telemetry_events/src/telemetry_events.rs | 10
crates/util/src/semantic_version.rs | 24
crates/zed/src/main.rs | 2
22 files changed, 531 insertions(+), 306 deletions(-)
@@ -3472,6 +3472,7 @@ dependencies = [
"async-tar",
"async-trait",
"cap-std",
+ "client",
"collections",
"ctor",
"env_logger",
@@ -7799,6 +7800,7 @@ dependencies = [
"anyhow",
"async-tungstenite",
"base64 0.13.1",
+ "chrono",
"collections",
"env_logger",
"futures 0.3.28",
@@ -15,7 +15,8 @@ use std::{env, mem, path::PathBuf, sync::Arc, time::Duration};
use sysinfo::{CpuRefreshKind, MemoryRefreshKind, Pid, ProcessRefreshKind, RefreshKind, System};
use telemetry_events::{
ActionEvent, AppEvent, AssistantEvent, AssistantKind, CallEvent, CopilotEvent, CpuEvent,
- EditEvent, EditorEvent, Event, EventRequestBody, EventWrapper, MemoryEvent, SettingEvent,
+ EditEvent, EditorEvent, Event, EventRequestBody, EventWrapper, ExtensionEvent, MemoryEvent,
+ SettingEvent,
};
use tempfile::NamedTempFile;
use util::http::{self, HttpClient, HttpClientWithUrl, Method};
@@ -326,6 +327,13 @@ impl Telemetry {
self.report_event(event)
}
+ pub fn report_extension_event(self: &Arc<Self>, extension_id: Arc<str>, version: Arc<str>) {
+ self.report_event(Event::Extension(ExtensionEvent {
+ extension_id,
+ version,
+ }))
+ }
+
pub fn log_edit_event(self: &Arc<Self>, environment: &'static str) {
let mut state = self.state.lock();
let period_data = state.event_coalescer.log_event(environment);
@@ -374,6 +374,7 @@ CREATE TABLE extension_versions (
repository TEXT NOT NULL,
description TEXT NOT NULL,
schema_version INTEGER NOT NULL DEFAULT 0,
+ wasm_api_version TEXT,
download_count INTEGER NOT NULL DEFAULT 0,
PRIMARY KEY (extension_id, version)
);
@@ -0,0 +1 @@
+ALTER TABLE extension_versions ADD COLUMN wasm_api_version TEXT;
@@ -1,5 +1,5 @@
-use std::sync::{Arc, OnceLock};
-
+use super::ips_file::IpsFile;
+use crate::{api::slack, AppState, Error, Result};
use anyhow::{anyhow, Context};
use aws_sdk_s3::primitives::ByteStream;
use axum::{
@@ -9,18 +9,16 @@ use axum::{
routing::post,
Extension, Router, TypedHeader,
};
+use rpc::ExtensionMetadata;
use serde::{Serialize, Serializer};
use sha2::{Digest, Sha256};
+use std::sync::{Arc, OnceLock};
use telemetry_events::{
ActionEvent, AppEvent, AssistantEvent, CallEvent, CopilotEvent, CpuEvent, EditEvent,
- EditorEvent, Event, EventRequestBody, EventWrapper, MemoryEvent, SettingEvent,
+ EditorEvent, Event, EventRequestBody, EventWrapper, ExtensionEvent, MemoryEvent, SettingEvent,
};
use util::SemanticVersion;
-use crate::{api::slack, AppState, Error, Result};
-
-use super::ips_file::IpsFile;
-
pub fn router() -> Router {
Router::new()
.route("/telemetry/events", post(post_events))
@@ -331,6 +329,21 @@ pub async fn post_events(
&request_body,
first_event_at,
)),
+ Event::Extension(event) => {
+ let metadata = app
+ .db
+ .get_extension_version(&event.extension_id, &event.version)
+ .await?;
+ to_upload
+ .extension_events
+ .push(ExtensionEventRow::from_event(
+ event.clone(),
+ &wrapper,
+ &request_body,
+ metadata,
+ first_event_at,
+ ))
+ }
}
}
@@ -352,6 +365,7 @@ struct ToUpload {
memory_events: Vec<MemoryEventRow>,
app_events: Vec<AppEventRow>,
setting_events: Vec<SettingEventRow>,
+ extension_events: Vec<ExtensionEventRow>,
edit_events: Vec<EditEventRow>,
action_events: Vec<ActionEventRow>,
}
@@ -410,6 +424,15 @@ impl ToUpload {
.await
.with_context(|| format!("failed to upload to table '{SETTING_EVENTS_TABLE}'"))?;
+ const EXTENSION_EVENTS_TABLE: &str = "extension_events";
+ Self::upload_to_table(
+ EXTENSION_EVENTS_TABLE,
+ &self.extension_events,
+ clickhouse_client,
+ )
+ .await
+ .with_context(|| format!("failed to upload to table '{EXTENSION_EVENTS_TABLE}'"))?;
+
const EDIT_EVENTS_TABLE: &str = "edit_events";
Self::upload_to_table(EDIT_EVENTS_TABLE, &self.edit_events, clickhouse_client)
.await
@@ -861,6 +884,68 @@ impl SettingEventRow {
}
}
+#[derive(Serialize, Debug, clickhouse::Row)]
+pub struct ExtensionEventRow {
+ // AppInfoBase
+ app_version: String,
+ major: Option<i32>,
+ minor: Option<i32>,
+ patch: Option<i32>,
+ release_channel: String,
+
+ // ClientEventBase
+ installation_id: Option<String>,
+ session_id: Option<String>,
+ is_staff: Option<bool>,
+ time: i64,
+
+ // ExtensionEventRow
+ extension_id: Arc<str>,
+ extension_version: Arc<str>,
+ dev: bool,
+ schema_version: Option<i32>,
+ wasm_api_version: Option<String>,
+}
+
+impl ExtensionEventRow {
+ fn from_event(
+ event: ExtensionEvent,
+ wrapper: &EventWrapper,
+ body: &EventRequestBody,
+ extension_metadata: Option<ExtensionMetadata>,
+ first_event_at: chrono::DateTime<chrono::Utc>,
+ ) -> Self {
+ let semver = body.semver();
+ let time =
+ first_event_at + chrono::Duration::milliseconds(wrapper.milliseconds_since_first_event);
+
+ Self {
+ app_version: body.app_version.clone(),
+ major: semver.map(|s| s.major as i32),
+ minor: semver.map(|s| s.minor as i32),
+ patch: semver.map(|s| s.patch as i32),
+ release_channel: body.release_channel.clone().unwrap_or_default(),
+ installation_id: body.installation_id.clone(),
+ session_id: body.session_id.clone(),
+ is_staff: body.is_staff,
+ time: time.timestamp_millis(),
+ extension_id: event.extension_id,
+ extension_version: event.version,
+ dev: extension_metadata.is_none(),
+ schema_version: extension_metadata
+ .as_ref()
+ .and_then(|metadata| metadata.manifest.schema_version),
+ wasm_api_version: extension_metadata.as_ref().and_then(|metadata| {
+ metadata
+ .manifest
+ .wasm_api_version
+ .as_ref()
+ .map(|version| version.to_string())
+ }),
+ }
+ }
+}
+
#[derive(Serialize, Debug, clickhouse::Row)]
pub struct EditEventRow {
// AppInfoBase
@@ -1,7 +1,4 @@
-use crate::{
- db::{ExtensionMetadata, NewExtensionVersion},
- AppState, Error, Result,
-};
+use crate::{db::NewExtensionVersion, AppState, Error, Result};
use anyhow::{anyhow, Context as _};
use aws_sdk_s3::presigning::PresigningConfig;
use axum::{
@@ -12,7 +9,7 @@ use axum::{
Extension, Json, Router,
};
use collections::HashMap;
-use rpc::ExtensionApiManifest;
+use rpc::{ExtensionApiManifest, ExtensionMetadata};
use serde::{Deserialize, Serialize};
use std::{sync::Arc, time::Duration};
use time::PrimitiveDateTime;
@@ -78,7 +75,7 @@ async fn download_latest_extension(
Extension(app),
Path(DownloadExtensionParams {
extension_id: params.extension_id,
- version: extension.version,
+ version: extension.manifest.version,
}),
)
.await
@@ -285,6 +282,7 @@ async fn fetch_extension_manifest(
authors: manifest.authors,
repository: manifest.repository,
schema_version: manifest.schema_version.unwrap_or(0),
+ wasm_api_version: manifest.wasm_api_version,
published_at,
})
}
@@ -12,7 +12,7 @@ use futures::StreamExt;
use rand::{prelude::StdRng, Rng, SeedableRng};
use rpc::{
proto::{self},
- ConnectionId,
+ ConnectionId, ExtensionMetadata,
};
use sea_orm::{
entity::prelude::*,
@@ -726,22 +726,10 @@ pub struct NewExtensionVersion {
pub authors: Vec<String>,
pub repository: String,
pub schema_version: i32,
+ pub wasm_api_version: Option<String>,
pub published_at: PrimitiveDateTime,
}
-#[derive(Debug, Serialize, PartialEq)]
-pub struct ExtensionMetadata {
- pub id: String,
- pub name: String,
- pub version: String,
- pub authors: Vec<String>,
- pub description: String,
- pub repository: String,
- #[serde(serialize_with = "serialize_iso8601")]
- pub published_at: PrimitiveDateTime,
- pub download_count: u64,
-}
-
pub fn serialize_iso8601<S: Serializer>(
datetime: &PrimitiveDateTime,
serializer: S,
@@ -1,3 +1,5 @@
+use chrono::Utc;
+
use super::*;
impl Database {
@@ -31,22 +33,8 @@ impl Database {
Ok(extensions
.into_iter()
- .filter_map(|(extension, latest_version)| {
- let version = latest_version?;
- Some(ExtensionMetadata {
- id: extension.external_id,
- name: extension.name,
- version: version.version,
- authors: version
- .authors
- .split(',')
- .map(|author| author.trim().to_string())
- .collect::<Vec<_>>(),
- description: version.description,
- repository: version.repository,
- published_at: version.published_at,
- download_count: extension.total_download_count as u64,
- })
+ .filter_map(|(extension, version)| {
+ Some(metadata_from_extension_and_version(extension, version?))
})
.collect())
})
@@ -67,22 +55,29 @@ impl Database {
.one(&*tx)
.await?;
- Ok(extension.and_then(|(extension, latest_version)| {
- let version = latest_version?;
- Some(ExtensionMetadata {
- id: extension.external_id,
- name: extension.name,
- version: version.version,
- authors: version
- .authors
- .split(',')
- .map(|author| author.trim().to_string())
- .collect::<Vec<_>>(),
- description: version.description,
- repository: version.repository,
- published_at: version.published_at,
- download_count: extension.total_download_count as u64,
- })
+ Ok(extension.and_then(|(extension, version)| {
+ Some(metadata_from_extension_and_version(extension, version?))
+ }))
+ })
+ .await
+ }
+
+ pub async fn get_extension_version(
+ &self,
+ extension_id: &str,
+ version: &str,
+ ) -> Result<Option<ExtensionMetadata>> {
+ self.transaction(|tx| async move {
+ let extension = extension::Entity::find()
+ .filter(extension::Column::ExternalId.eq(extension_id))
+ .filter(extension_version::Column::Version.eq(version))
+ .inner_join(extension_version::Entity)
+ .select_also(extension_version::Entity)
+ .one(&*tx)
+ .await?;
+
+ Ok(extension.and_then(|(extension, version)| {
+ Some(metadata_from_extension_and_version(extension, version?))
}))
})
.await
@@ -172,6 +167,7 @@ impl Database {
repository: ActiveValue::Set(version.repository.clone()),
description: ActiveValue::Set(version.description.clone()),
schema_version: ActiveValue::Set(version.schema_version),
+ wasm_api_version: ActiveValue::Set(version.wasm_api_version.clone()),
download_count: ActiveValue::NotSet,
}
}))
@@ -241,3 +237,35 @@ impl Database {
.await
}
}
+
+fn metadata_from_extension_and_version(
+ extension: extension::Model,
+ version: extension_version::Model,
+) -> ExtensionMetadata {
+ ExtensionMetadata {
+ id: extension.external_id,
+ manifest: rpc::ExtensionApiManifest {
+ name: extension.name,
+ version: version.version,
+ authors: version
+ .authors
+ .split(',')
+ .map(|author| author.trim().to_string())
+ .collect::<Vec<_>>(),
+ description: Some(version.description),
+ repository: version.repository,
+ schema_version: Some(version.schema_version),
+ wasm_api_version: version.wasm_api_version,
+ },
+
+ published_at: convert_time_to_chrono(version.published_at),
+ download_count: extension.total_download_count as u64,
+ }
+}
+
+pub fn convert_time_to_chrono(time: time::PrimitiveDateTime) -> chrono::DateTime<Utc> {
+ chrono::DateTime::from_naive_utc_and_offset(
+ chrono::NaiveDateTime::from_timestamp_opt(time.assume_utc().unix_timestamp(), 0).unwrap(),
+ Utc,
+ )
+}
@@ -14,6 +14,7 @@ pub struct Model {
pub repository: String,
pub description: String,
pub schema_version: i32,
+ pub wasm_api_version: Option<String>,
pub download_count: i64,
}
@@ -1,10 +1,9 @@
use super::Database;
use crate::{
- db::{ExtensionMetadata, NewExtensionVersion},
+ db::{queries::extensions::convert_time_to_chrono, ExtensionMetadata, NewExtensionVersion},
test_both_dbs,
};
use std::sync::Arc;
-use time::{OffsetDateTime, PrimitiveDateTime};
test_both_dbs!(
test_extensions,
@@ -19,8 +18,10 @@ async fn test_extensions(db: &Arc<Database>) {
let extensions = db.get_extensions(None, 1, 5).await.unwrap();
assert!(extensions.is_empty());
- let t0 = OffsetDateTime::from_unix_timestamp_nanos(0).unwrap();
- let t0 = PrimitiveDateTime::new(t0.date(), t0.time());
+ let t0 = time::OffsetDateTime::from_unix_timestamp_nanos(0).unwrap();
+ let t0 = time::PrimitiveDateTime::new(t0.date(), t0.time());
+
+ let t0_chrono = convert_time_to_chrono(t0);
db.insert_extension_versions(
&[
@@ -34,6 +35,7 @@ async fn test_extensions(db: &Arc<Database>) {
authors: vec!["max".into()],
repository: "ext1/repo".into(),
schema_version: 1,
+ wasm_api_version: None,
published_at: t0,
},
NewExtensionVersion {
@@ -43,6 +45,7 @@ async fn test_extensions(db: &Arc<Database>) {
authors: vec!["max".into(), "marshall".into()],
repository: "ext1/repo".into(),
schema_version: 1,
+ wasm_api_version: None,
published_at: t0,
},
],
@@ -56,6 +59,7 @@ async fn test_extensions(db: &Arc<Database>) {
authors: vec!["marshall".into()],
repository: "ext2/repo".into(),
schema_version: 0,
+ wasm_api_version: None,
published_at: t0,
}],
),
@@ -84,22 +88,30 @@ async fn test_extensions(db: &Arc<Database>) {
&[
ExtensionMetadata {
id: "ext1".into(),
- name: "Extension One".into(),
- version: "0.0.2".into(),
- authors: vec!["max".into(), "marshall".into()],
- description: "a good extension".into(),
- repository: "ext1/repo".into(),
- published_at: t0,
+ manifest: rpc::ExtensionApiManifest {
+ name: "Extension One".into(),
+ version: "0.0.2".into(),
+ authors: vec!["max".into(), "marshall".into()],
+ description: Some("a good extension".into()),
+ repository: "ext1/repo".into(),
+ schema_version: Some(1),
+ wasm_api_version: None,
+ },
+ published_at: t0_chrono,
download_count: 0,
},
ExtensionMetadata {
id: "ext2".into(),
- name: "Extension Two".into(),
- version: "0.2.0".into(),
- authors: vec!["marshall".into()],
- description: "a great extension".into(),
- repository: "ext2/repo".into(),
- published_at: t0,
+ manifest: rpc::ExtensionApiManifest {
+ name: "Extension Two".into(),
+ version: "0.2.0".into(),
+ authors: vec!["marshall".into()],
+ description: Some("a great extension".into()),
+ repository: "ext2/repo".into(),
+ schema_version: Some(0),
+ wasm_api_version: None,
+ },
+ published_at: t0_chrono,
download_count: 0
},
]
@@ -111,12 +123,16 @@ async fn test_extensions(db: &Arc<Database>) {
extensions,
&[ExtensionMetadata {
id: "ext2".into(),
- name: "Extension Two".into(),
- version: "0.2.0".into(),
- authors: vec!["marshall".into()],
- description: "a great extension".into(),
- repository: "ext2/repo".into(),
- published_at: t0,
+ manifest: rpc::ExtensionApiManifest {
+ name: "Extension Two".into(),
+ version: "0.2.0".into(),
+ authors: vec!["marshall".into()],
+ description: Some("a great extension".into()),
+ repository: "ext2/repo".into(),
+ schema_version: Some(0),
+ wasm_api_version: None,
+ },
+ published_at: t0_chrono,
download_count: 0
},]
);
@@ -147,22 +163,30 @@ async fn test_extensions(db: &Arc<Database>) {
&[
ExtensionMetadata {
id: "ext2".into(),
- name: "Extension Two".into(),
- version: "0.2.0".into(),
- authors: vec!["marshall".into()],
- description: "a great extension".into(),
- repository: "ext2/repo".into(),
- published_at: t0,
+ manifest: rpc::ExtensionApiManifest {
+ name: "Extension Two".into(),
+ version: "0.2.0".into(),
+ authors: vec!["marshall".into()],
+ description: Some("a great extension".into()),
+ repository: "ext2/repo".into(),
+ schema_version: Some(0),
+ wasm_api_version: None,
+ },
+ published_at: t0_chrono,
download_count: 7
},
ExtensionMetadata {
id: "ext1".into(),
- name: "Extension One".into(),
- version: "0.0.2".into(),
- authors: vec!["max".into(), "marshall".into()],
- description: "a good extension".into(),
- repository: "ext1/repo".into(),
- published_at: t0,
+ manifest: rpc::ExtensionApiManifest {
+ name: "Extension One".into(),
+ version: "0.0.2".into(),
+ authors: vec!["max".into(), "marshall".into()],
+ description: Some("a good extension".into()),
+ repository: "ext1/repo".into(),
+ schema_version: Some(1),
+ wasm_api_version: None,
+ },
+ published_at: t0_chrono,
download_count: 5,
},
]
@@ -181,6 +205,7 @@ async fn test_extensions(db: &Arc<Database>) {
authors: vec!["max".into(), "marshall".into()],
repository: "ext1/repo".into(),
schema_version: 1,
+ wasm_api_version: None,
published_at: t0,
}],
),
@@ -193,6 +218,7 @@ async fn test_extensions(db: &Arc<Database>) {
authors: vec!["marshall".into()],
repository: "ext2/repo".into(),
schema_version: 0,
+ wasm_api_version: None,
published_at: t0,
}],
),
@@ -223,22 +249,30 @@ async fn test_extensions(db: &Arc<Database>) {
&[
ExtensionMetadata {
id: "ext2".into(),
- name: "Extension Two".into(),
- version: "0.2.0".into(),
- authors: vec!["marshall".into()],
- description: "a great extension".into(),
- repository: "ext2/repo".into(),
- published_at: t0,
+ manifest: rpc::ExtensionApiManifest {
+ name: "Extension Two".into(),
+ version: "0.2.0".into(),
+ authors: vec!["marshall".into()],
+ description: Some("a great extension".into()),
+ repository: "ext2/repo".into(),
+ schema_version: Some(0),
+ wasm_api_version: None,
+ },
+ published_at: t0_chrono,
download_count: 7
},
ExtensionMetadata {
id: "ext1".into(),
- name: "Extension One".into(),
- version: "0.0.3".into(),
- authors: vec!["max".into(), "marshall".into()],
- description: "a real good extension".into(),
- repository: "ext1/repo".into(),
- published_at: t0,
+ manifest: rpc::ExtensionApiManifest {
+ name: "Extension One".into(),
+ version: "0.0.3".into(),
+ authors: vec!["max".into(), "marshall".into()],
+ description: Some("a real good extension".into()),
+ repository: "ext1/repo".into(),
+ schema_version: Some(1),
+ wasm_api_version: None,
+ },
+ published_at: t0_chrono,
download_count: 5,
},
]
@@ -22,6 +22,7 @@ async-compression.workspace = true
async-tar.workspace = true
async-trait.workspace = true
cap-std.workspace = true
+client.workspace = true
collections.workspace = true
fs.workspace = true
futures.workspace = true
@@ -1,3 +1,4 @@
+use crate::wasm_host::parse_wasm_extension_version;
use crate::ExtensionManifest;
use crate::{extension_manifest::ExtensionLibraryKind, GrammarManifestEntry};
use anyhow::{anyhow, bail, Context as _, Result};
@@ -73,9 +74,11 @@ impl ExtensionBuilder {
pub async fn compile_extension(
&self,
extension_dir: &Path,
- extension_manifest: &ExtensionManifest,
+ extension_manifest: &mut ExtensionManifest,
options: CompileExtensionOptions,
) -> Result<()> {
+ populate_defaults(extension_manifest, &extension_dir)?;
+
if extension_dir.is_relative() {
bail!(
"extension dir {} is not an absolute path",
@@ -85,12 +88,9 @@ impl ExtensionBuilder {
fs::create_dir_all(&self.cache_dir).context("failed to create cache dir")?;
- let cargo_toml_path = extension_dir.join("Cargo.toml");
- if extension_manifest.lib.kind == Some(ExtensionLibraryKind::Rust)
- || fs::metadata(&cargo_toml_path).map_or(false, |stat| stat.is_file())
- {
+ if extension_manifest.lib.kind == Some(ExtensionLibraryKind::Rust) {
log::info!("compiling Rust extension {}", extension_dir.display());
- self.compile_rust_extension(extension_dir, options)
+ self.compile_rust_extension(extension_dir, extension_manifest, options)
.await
.context("failed to compile Rust extension")?;
}
@@ -108,6 +108,7 @@ impl ExtensionBuilder {
async fn compile_rust_extension(
&self,
extension_dir: &Path,
+ manifest: &mut ExtensionManifest,
options: CompileExtensionOptions,
) -> Result<(), anyhow::Error> {
self.install_rust_wasm_target_if_needed()?;
@@ -162,6 +163,11 @@ impl ExtensionBuilder {
.strip_custom_sections(&component_bytes)
.context("failed to strip debug sections from wasm component")?;
+ let wasm_extension_api_version =
+ parse_wasm_extension_version(&manifest.id, &component_bytes)
+ .context("compiled wasm did not contain a valid zed extension api version")?;
+ manifest.lib.version = Some(wasm_extension_api_version);
+
fs::write(extension_dir.join("extension.wasm"), &component_bytes)
.context("failed to write extension.wasm")?;
@@ -469,3 +475,86 @@ impl ExtensionBuilder {
Ok(output)
}
}
+
+fn populate_defaults(manifest: &mut ExtensionManifest, extension_path: &Path) -> Result<()> {
+ // For legacy extensions on the v0 schema (aka, using `extension.json`), clear out any existing
+ // contents of the computed fields, since we don't care what the existing values are.
+ if manifest.schema_version == 0 {
+ manifest.languages.clear();
+ manifest.grammars.clear();
+ manifest.themes.clear();
+ }
+
+ let cargo_toml_path = extension_path.join("Cargo.toml");
+ if cargo_toml_path.exists() {
+ manifest.lib.kind = Some(ExtensionLibraryKind::Rust);
+ }
+
+ let languages_dir = extension_path.join("languages");
+ if languages_dir.exists() {
+ for entry in fs::read_dir(&languages_dir).context("failed to list languages dir")? {
+ let entry = entry?;
+ let language_dir = entry.path();
+ let config_path = language_dir.join("config.toml");
+ if config_path.exists() {
+ let relative_language_dir =
+ language_dir.strip_prefix(extension_path)?.to_path_buf();
+ if !manifest.languages.contains(&relative_language_dir) {
+ manifest.languages.push(relative_language_dir);
+ }
+ }
+ }
+ }
+
+ let themes_dir = extension_path.join("themes");
+ if themes_dir.exists() {
+ for entry in fs::read_dir(&themes_dir).context("failed to list themes dir")? {
+ let entry = entry?;
+ let theme_path = entry.path();
+ if theme_path.extension() == Some("json".as_ref()) {
+ let relative_theme_path = theme_path.strip_prefix(extension_path)?.to_path_buf();
+ if !manifest.themes.contains(&relative_theme_path) {
+ manifest.themes.push(relative_theme_path);
+ }
+ }
+ }
+ }
+
+ // For legacy extensions on the v0 schema (aka, using `extension.json`), we want to populate the grammars in
+ // the manifest using the contents of the `grammars` directory.
+ if manifest.schema_version == 0 {
+ let grammars_dir = extension_path.join("grammars");
+ if grammars_dir.exists() {
+ for entry in fs::read_dir(&grammars_dir).context("failed to list grammars dir")? {
+ let entry = entry?;
+ let grammar_path = entry.path();
+ if grammar_path.extension() == Some("toml".as_ref()) {
+ #[derive(Deserialize)]
+ struct GrammarConfigToml {
+ pub repository: String,
+ pub commit: String,
+ }
+
+ let grammar_config = fs::read_to_string(&grammar_path)?;
+ let grammar_config: GrammarConfigToml = toml::from_str(&grammar_config)?;
+
+ let grammar_name = grammar_path
+ .file_stem()
+ .and_then(|stem| stem.to_str())
+ .ok_or_else(|| anyhow!("no grammar name"))?;
+ if !manifest.grammars.contains_key(grammar_name) {
+ manifest.grammars.insert(
+ grammar_name.into(),
+ GrammarManifestEntry {
+ repository: grammar_config.repository,
+ rev: grammar_config.commit,
+ },
+ );
+ }
+ }
+ }
+ }
+ }
+
+ Ok(())
+}
@@ -1,7 +1,14 @@
+use anyhow::{anyhow, Context, Result};
use collections::BTreeMap;
+use fs::Fs;
use language::LanguageServerName;
use serde::{Deserialize, Serialize};
-use std::{path::PathBuf, sync::Arc};
+use std::{
+ ffi::OsStr,
+ path::{Path, PathBuf},
+ sync::Arc,
+};
+use util::SemanticVersion;
/// This is the old version of the extension manifest, from when it was `extension.json`.
#[derive(Clone, PartialEq, Eq, Debug, Deserialize, Serialize)]
@@ -53,6 +60,7 @@ pub struct ExtensionManifest {
#[derive(Clone, Default, PartialEq, Eq, Debug, Deserialize, Serialize)]
pub struct LibManifestEntry {
pub kind: Option<ExtensionLibraryKind>,
+ pub version: Option<SemanticVersion>,
}
#[derive(Clone, PartialEq, Eq, Debug, Deserialize, Serialize)]
@@ -71,3 +79,68 @@ pub struct GrammarManifestEntry {
pub struct LanguageServerManifestEntry {
pub language: Arc<str>,
}
+
+impl ExtensionManifest {
+ pub async fn load(fs: Arc<dyn Fs>, extension_dir: &Path) -> Result<Self> {
+ let extension_name = extension_dir
+ .file_name()
+ .and_then(OsStr::to_str)
+ .ok_or_else(|| anyhow!("invalid extension name"))?;
+
+ let mut extension_manifest_path = extension_dir.join("extension.json");
+ if fs.is_file(&extension_manifest_path).await {
+ let manifest_content = fs
+ .load(&extension_manifest_path)
+ .await
+ .with_context(|| format!("failed to load {extension_name} extension.json"))?;
+ let manifest_json = serde_json::from_str::<OldExtensionManifest>(&manifest_content)
+ .with_context(|| {
+ format!("invalid extension.json for extension {extension_name}")
+ })?;
+
+ Ok(manifest_from_old_manifest(manifest_json, extension_name))
+ } else {
+ extension_manifest_path.set_extension("toml");
+ let manifest_content = fs
+ .load(&extension_manifest_path)
+ .await
+ .with_context(|| format!("failed to load {extension_name} extension.toml"))?;
+ toml::from_str(&manifest_content)
+ .with_context(|| format!("invalid extension.json for extension {extension_name}"))
+ }
+ }
+}
+
+fn manifest_from_old_manifest(
+ manifest_json: OldExtensionManifest,
+ extension_id: &str,
+) -> ExtensionManifest {
+ ExtensionManifest {
+ id: extension_id.into(),
+ name: manifest_json.name,
+ version: manifest_json.version,
+ description: manifest_json.description,
+ repository: manifest_json.repository,
+ authors: manifest_json.authors,
+ schema_version: 0,
+ lib: Default::default(),
+ themes: {
+ let mut themes = manifest_json.themes.into_values().collect::<Vec<_>>();
+ themes.sort();
+ themes.dedup();
+ themes
+ },
+ languages: {
+ let mut languages = manifest_json.languages.into_values().collect::<Vec<_>>();
+ languages.sort();
+ languages.dedup();
+ languages
+ },
+ grammars: manifest_json
+ .grammars
+ .into_keys()
+ .map(|grammar_name| (grammar_name, Default::default()))
+ .collect(),
+ language_servers: Default::default(),
+ }
+}
@@ -10,6 +10,7 @@ use crate::{extension_lsp_adapter::ExtensionLspAdapter, wasm_host::wit};
use anyhow::{anyhow, bail, Context as _, Result};
use async_compression::futures::bufread::GzipDecoder;
use async_tar::Archive;
+use client::{telemetry::Telemetry, Client};
use collections::{hash_map, BTreeMap, HashMap, HashSet};
use extension_builder::{CompileExtensionOptions, ExtensionBuilder};
use fs::{Fs, RemoveOptions};
@@ -30,7 +31,6 @@ use node_runtime::NodeRuntime;
use serde::{Deserialize, Serialize};
use std::{
cmp::Ordering,
- ffi::OsStr,
path::{self, Path, PathBuf},
sync::Arc,
time::{Duration, Instant},
@@ -75,6 +75,7 @@ pub struct ExtensionStore {
extension_index: ExtensionIndex,
fs: Arc<dyn Fs>,
http_client: Arc<HttpClientWithUrl>,
+ telemetry: Option<Arc<Telemetry>>,
reload_tx: UnboundedSender<Option<Arc<str>>>,
reload_complete_senders: Vec<oneshot::Sender<()>>,
installed_dir: PathBuf,
@@ -149,7 +150,7 @@ actions!(zed, [ReloadExtensions]);
pub fn init(
fs: Arc<fs::RealFs>,
- http_client: Arc<HttpClientWithUrl>,
+ client: Arc<Client>,
node_runtime: Arc<dyn NodeRuntime>,
language_registry: Arc<LanguageRegistry>,
theme_registry: Arc<ThemeRegistry>,
@@ -160,7 +161,8 @@ pub fn init(
EXTENSIONS_DIR.clone(),
None,
fs,
- http_client,
+ client.http_client().clone(),
+ Some(client.telemetry().clone()),
node_runtime,
language_registry,
theme_registry,
@@ -187,6 +189,7 @@ impl ExtensionStore {
build_dir: Option<PathBuf>,
fs: Arc<dyn Fs>,
http_client: Arc<HttpClientWithUrl>,
+ telemetry: Option<Arc<Telemetry>>,
node_runtime: Arc<dyn NodeRuntime>,
language_registry: Arc<LanguageRegistry>,
theme_registry: Arc<ThemeRegistry>,
@@ -216,6 +219,7 @@ impl ExtensionStore {
wasm_extensions: Vec::new(),
fs,
http_client,
+ telemetry,
language_registry,
theme_registry,
reload_tx,
@@ -587,8 +591,8 @@ impl ExtensionStore {
let builder = self.builder.clone();
cx.spawn(move |this, mut cx| async move {
- let extension_manifest =
- Self::load_extension_manifest(fs.clone(), &extension_source_path).await?;
+ let mut extension_manifest =
+ ExtensionManifest::load(fs.clone(), &extension_source_path).await?;
let extension_id = extension_manifest.id.clone();
if !this.update(&mut cx, |this, cx| {
@@ -622,7 +626,7 @@ impl ExtensionStore {
builder
.compile_extension(
&extension_source_path,
- &extension_manifest,
+ &mut extension_manifest,
CompileExtensionOptions { release: false },
)
.await
@@ -667,9 +671,13 @@ impl ExtensionStore {
cx.notify();
let compile = cx.background_executor().spawn(async move {
- let manifest = Self::load_extension_manifest(fs, &path).await?;
+ let mut manifest = ExtensionManifest::load(fs, &path).await?;
builder
- .compile_extension(&path, &manifest, CompileExtensionOptions { release: true })
+ .compile_extension(
+ &path,
+ &mut manifest,
+ CompileExtensionOptions { release: true },
+ )
.await
});
@@ -759,6 +767,17 @@ impl ExtensionStore {
extensions_to_unload.len() - reload_count
);
+ if let Some(telemetry) = &self.telemetry {
+ for extension_id in &extensions_to_load {
+ if let Some(extension) = self.extension_index.extensions.get(extension_id) {
+ telemetry.report_extension_event(
+ extension_id.clone(),
+ extension.manifest.version.clone(),
+ );
+ }
+ }
+ }
+
let themes_to_remove = old_index
.themes
.iter()
@@ -908,7 +927,9 @@ impl ExtensionStore {
cx.background_executor().clone(),
)
.await
- .context("failed to load wasm extension")
+ .with_context(|| {
+ format!("failed to load wasm extension {}", extension.manifest.id)
+ })
})
.await;
@@ -989,8 +1010,7 @@ impl ExtensionStore {
extension_dir: PathBuf,
index: &mut ExtensionIndex,
) -> Result<()> {
- let mut extension_manifest =
- Self::load_extension_manifest(fs.clone(), &extension_dir).await?;
+ let mut extension_manifest = ExtensionManifest::load(fs.clone(), &extension_dir).await?;
let extension_id = extension_manifest.id.clone();
// TODO: distinguish dev extensions more explicitly, by the absence
@@ -1082,72 +1102,6 @@ impl ExtensionStore {
Ok(())
}
-
- pub async fn load_extension_manifest(
- fs: Arc<dyn Fs>,
- extension_dir: &Path,
- ) -> Result<ExtensionManifest> {
- let extension_name = extension_dir
- .file_name()
- .and_then(OsStr::to_str)
- .ok_or_else(|| anyhow!("invalid extension name"))?;
-
- let mut extension_manifest_path = extension_dir.join("extension.json");
- if fs.is_file(&extension_manifest_path).await {
- let manifest_content = fs
- .load(&extension_manifest_path)
- .await
- .with_context(|| format!("failed to load {extension_name} extension.json"))?;
- let manifest_json = serde_json::from_str::<OldExtensionManifest>(&manifest_content)
- .with_context(|| {
- format!("invalid extension.json for extension {extension_name}")
- })?;
-
- Ok(manifest_from_old_manifest(manifest_json, extension_name))
- } else {
- extension_manifest_path.set_extension("toml");
- let manifest_content = fs
- .load(&extension_manifest_path)
- .await
- .with_context(|| format!("failed to load {extension_name} extension.toml"))?;
- toml::from_str(&manifest_content)
- .with_context(|| format!("invalid extension.json for extension {extension_name}"))
- }
- }
-}
-
-fn manifest_from_old_manifest(
- manifest_json: OldExtensionManifest,
- extension_id: &str,
-) -> ExtensionManifest {
- ExtensionManifest {
- id: extension_id.into(),
- name: manifest_json.name,
- version: manifest_json.version,
- description: manifest_json.description,
- repository: manifest_json.repository,
- authors: manifest_json.authors,
- schema_version: 0,
- lib: Default::default(),
- themes: {
- let mut themes = manifest_json.themes.into_values().collect::<Vec<_>>();
- themes.sort();
- themes.dedup();
- themes
- },
- languages: {
- let mut languages = manifest_json.languages.into_values().collect::<Vec<_>>();
- languages.sort();
- languages.dedup();
- languages
- },
- grammars: manifest_json
- .grammars
- .into_keys()
- .map(|grammar_name| (grammar_name, Default::default()))
- .collect(),
- language_servers: Default::default(),
- }
}
fn load_plugin_queries(root_path: &Path) -> LanguageQueries {
@@ -262,6 +262,7 @@ async fn test_extension_store(cx: &mut TestAppContext) {
None,
fs.clone(),
http_client.clone(),
+ None,
node_runtime.clone(),
language_registry.clone(),
theme_registry.clone(),
@@ -381,6 +382,7 @@ async fn test_extension_store(cx: &mut TestAppContext) {
None,
fs.clone(),
http_client.clone(),
+ None,
node_runtime.clone(),
language_registry.clone(),
theme_registry.clone(),
@@ -538,6 +540,7 @@ async fn test_extension_store_with_gleam_extension(cx: &mut TestAppContext) {
Some(cache_dir),
fs.clone(),
http_client.clone(),
+ None,
node_runtime,
language_registry.clone(),
theme_registry.clone(),
@@ -40,7 +40,7 @@ pub struct WasmExtension {
tx: UnboundedSender<ExtensionCall>,
pub(crate) manifest: Arc<ExtensionManifest>,
#[allow(unused)]
- zed_api_version: SemanticVersion,
+ pub zed_api_version: SemanticVersion,
}
pub(crate) struct WasmState {
@@ -93,29 +93,11 @@ impl WasmHost {
) -> impl 'static + Future<Output = Result<WasmExtension>> {
let this = self.clone();
async move {
+ let zed_api_version = parse_wasm_extension_version(&manifest.id, &wasm_bytes)?;
+
let component = Component::from_binary(&this.engine, &wasm_bytes)
.context("failed to compile wasm component")?;
- let mut zed_api_version = None;
- for part in wasmparser::Parser::new(0).parse_all(&wasm_bytes) {
- if let wasmparser::Payload::CustomSection(s) = part? {
- if s.name() == "zed:api-version" {
- zed_api_version = parse_extension_version(s.data());
- if zed_api_version.is_none() {
- bail!(
- "extension {} has invalid zed:api-version section: {:?}",
- manifest.id,
- s.data()
- );
- }
- }
- }
- }
-
- let Some(zed_api_version) = zed_api_version else {
- bail!("extension {} has no zed:api-version section", manifest.id);
- };
-
let mut store = wasmtime::Store::new(
&this.engine,
WasmState {
@@ -196,7 +178,30 @@ impl WasmHost {
}
}
-fn parse_extension_version(data: &[u8]) -> Option<SemanticVersion> {
+pub fn parse_wasm_extension_version(
+ extension_id: &str,
+ wasm_bytes: &[u8],
+) -> Result<SemanticVersion> {
+ for part in wasmparser::Parser::new(0).parse_all(wasm_bytes) {
+ if let wasmparser::Payload::CustomSection(s) = part? {
+ if s.name() == "zed:api-version" {
+ let version = parse_wasm_extension_version_custom_section(s.data());
+ if let Some(version) = version {
+ return Ok(version);
+ } else {
+ bail!(
+ "extension {} has invalid zed:api-version section: {:?}",
+ extension_id,
+ s.data()
+ );
+ }
+ }
+ }
+ }
+ bail!("extension {} has no zed:api-version section", extension_id)
+}
+
+fn parse_wasm_extension_version_custom_section(data: &[u8]) -> Option<SemanticVersion> {
if data.len() == 6 {
Some(SemanticVersion {
major: u16::from_be_bytes([data[0], data[1]]) as _,
@@ -11,10 +11,9 @@ use anyhow::{anyhow, bail, Context, Result};
use clap::Parser;
use extension::{
extension_builder::{CompileExtensionOptions, ExtensionBuilder},
- ExtensionLibraryKind, ExtensionManifest, ExtensionStore, GrammarManifestEntry,
+ ExtensionManifest,
};
use language::LanguageConfig;
-use serde::Deserialize;
use theme::ThemeRegistry;
use tree_sitter::{Language, Query, WasmStore};
@@ -56,15 +55,14 @@ async fn main() -> Result<()> {
};
log::info!("loading extension manifest");
- let mut manifest = ExtensionStore::load_extension_manifest(fs.clone(), &extension_path).await?;
- populate_default_paths(&mut manifest, &extension_path)?;
+ let mut manifest = ExtensionManifest::load(fs.clone(), &extension_path).await?;
log::info!("compiling extension");
let builder = ExtensionBuilder::new(scratch_dir);
builder
.compile_extension(
&extension_path,
- &manifest,
+ &mut manifest,
CompileExtensionOptions { release: true },
)
.await
@@ -101,6 +99,7 @@ async fn main() -> Result<()> {
repository: manifest
.repository
.ok_or_else(|| anyhow!("missing repository in extension manifest"))?,
+ wasm_api_version: manifest.lib.version.map(|version| version.to_string()),
})?;
fs::remove_dir_all(&archive_dir)?;
fs::write(output_dir.join("manifest.json"), manifest_json.as_bytes())?;
@@ -108,89 +107,6 @@ async fn main() -> Result<()> {
Ok(())
}
-fn populate_default_paths(manifest: &mut ExtensionManifest, extension_path: &Path) -> Result<()> {
- // For legacy extensions on the v0 schema (aka, using `extension.json`), clear out any existing
- // contents of the computed fields, since we don't care what the existing values are.
- if manifest.schema_version == 0 {
- manifest.languages.clear();
- manifest.grammars.clear();
- manifest.themes.clear();
- }
-
- let cargo_toml_path = extension_path.join("Cargo.toml");
- if cargo_toml_path.exists() {
- manifest.lib.kind = Some(ExtensionLibraryKind::Rust);
- }
-
- let languages_dir = extension_path.join("languages");
- if languages_dir.exists() {
- for entry in fs::read_dir(&languages_dir).context("failed to list languages dir")? {
- let entry = entry?;
- let language_dir = entry.path();
- let config_path = language_dir.join("config.toml");
- if config_path.exists() {
- let relative_language_dir =
- language_dir.strip_prefix(extension_path)?.to_path_buf();
- if !manifest.languages.contains(&relative_language_dir) {
- manifest.languages.push(relative_language_dir);
- }
- }
- }
- }
-
- let themes_dir = extension_path.join("themes");
- if themes_dir.exists() {
- for entry in fs::read_dir(&themes_dir).context("failed to list themes dir")? {
- let entry = entry?;
- let theme_path = entry.path();
- if theme_path.extension() == Some("json".as_ref()) {
- let relative_theme_path = theme_path.strip_prefix(extension_path)?.to_path_buf();
- if !manifest.themes.contains(&relative_theme_path) {
- manifest.themes.push(relative_theme_path);
- }
- }
- }
- }
-
- // For legacy extensions on the v0 schema (aka, using `extension.json`), we want to populate the grammars in
- // the manifest using the contents of the `grammars` directory.
- if manifest.schema_version == 0 {
- let grammars_dir = extension_path.join("grammars");
- if grammars_dir.exists() {
- for entry in fs::read_dir(&grammars_dir).context("failed to list grammars dir")? {
- let entry = entry?;
- let grammar_path = entry.path();
- if grammar_path.extension() == Some("toml".as_ref()) {
- #[derive(Deserialize)]
- struct GrammarConfigToml {
- pub repository: String,
- pub commit: String,
- }
-
- let grammar_config = fs::read_to_string(&grammar_path)?;
- let grammar_config: GrammarConfigToml = toml::from_str(&grammar_config)?;
-
- let grammar_name = grammar_path
- .file_stem()
- .and_then(|stem| stem.to_str())
- .ok_or_else(|| anyhow!("no grammar name"))?;
- if !manifest.grammars.contains_key(grammar_name) {
- manifest.grammars.insert(
- grammar_name.into(),
- GrammarManifestEntry {
- repository: grammar_config.repository,
- rev: grammar_config.commit,
- },
- );
- }
- }
- }
- }
- }
-
- Ok(())
-}
-
async fn copy_extension_resources(
manifest: &ExtensionManifest,
extension_path: &Path,
@@ -20,6 +20,7 @@ test-support = ["collections/test-support", "gpui/test-support"]
anyhow.workspace = true
async-tungstenite = "0.16"
base64.workspace = true
+chrono.workspace = true
collections.workspace = true
futures.workspace = true
gpui = { workspace = true, optional = true }
@@ -1,6 +1,7 @@
+use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
-#[derive(Serialize, Deserialize)]
+#[derive(Serialize, Deserialize, Debug, PartialEq)]
pub struct ExtensionApiManifest {
pub name: String,
pub version: String,
@@ -8,4 +9,14 @@ pub struct ExtensionApiManifest {
pub authors: Vec<String>,
pub repository: String,
pub schema_version: Option<i32>,
+ pub wasm_api_version: Option<String>,
+}
+
+#[derive(Debug, Serialize, PartialEq)]
+pub struct ExtensionMetadata {
+ pub id: String,
+ #[serde(flatten)]
+ pub manifest: ExtensionApiManifest,
+ pub published_at: DateTime<Utc>,
+ pub download_count: u64,
}
@@ -1,6 +1,5 @@
-use std::fmt::Display;
-
use serde::{Deserialize, Serialize};
+use std::{fmt::Display, sync::Arc};
use util::SemanticVersion;
#[derive(Serialize, Deserialize, Debug)]
@@ -61,6 +60,7 @@ pub enum Event {
Memory(MemoryEvent),
App(AppEvent),
Setting(SettingEvent),
+ Extension(ExtensionEvent),
Edit(EditEvent),
Action(ActionEvent),
}
@@ -125,6 +125,12 @@ pub struct SettingEvent {
pub value: String,
}
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+pub struct ExtensionEvent {
+ pub extension_id: Arc<str>,
+ pub version: Arc<str>,
+}
+
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct AppEvent {
pub operation: String,
@@ -4,10 +4,10 @@ use std::{
};
use anyhow::{anyhow, Result};
-use serde::Serialize;
+use serde::{de::Error, Deserialize, Serialize};
/// A datastructure representing a semantic version number
-#[derive(Clone, Copy, Debug, Default, Eq, Ord, PartialEq, PartialOrd, Serialize)]
+#[derive(Clone, Copy, Debug, Default, Eq, Ord, PartialEq, PartialOrd)]
pub struct SemanticVersion {
pub major: usize,
pub minor: usize,
@@ -61,3 +61,23 @@ impl Display for SemanticVersion {
write!(f, "{}.{}.{}", self.major, self.minor, self.patch)
}
}
+
+impl Serialize for SemanticVersion {
+ fn serialize<S>(&self, serializer: S) -> std::prelude::v1::Result<S::Ok, S::Error>
+ where
+ S: serde::Serializer,
+ {
+ serializer.serialize_str(&self.to_string())
+ }
+}
+
+impl<'de> Deserialize<'de> for SemanticVersion {
+ fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+ where
+ D: serde::Deserializer<'de>,
+ {
+ let string = String::deserialize(deserializer)?;
+ Self::from_str(&string)
+ .map_err(|_| Error::custom(format!("Invalid version string \"{string}\"")))
+ }
+}
@@ -179,7 +179,7 @@ fn main() {
extension::init(
fs.clone(),
- http.clone(),
+ client.clone(),
node_runtime.clone(),
languages.clone(),
ThemeRegistry::global(cx),