From fc64843dd55a7e6d937aa1a7b3a08337cf28e616 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Mon, 12 Aug 2024 08:36:49 -0600 Subject: [PATCH] Enhance HTTP API for extensions (#16067) # HTTP Client Improvements for Extension API This PR enhances the HTTP client functionality in the Zed extension API, providing more control over requests and allowing for streaming responses. ## Key Changes 1. Extended `HttpRequest` struct: - Added `method` field to specify HTTP method - Added `headers` field for custom headers - Added optional `body` field for request payload 2. Introduced `HttpMethod` enum for supported HTTP methods 3. Updated `HttpResponse` struct: - Added `headers` field to access response headers - Changed `body` type from `String` to `Vec` for binary data support 4. Added streaming support: - New `fetch_stream` function to get a response stream - Introduced `HttpResponseStream` resource for chunked reading 5. Updated internal implementations to support these new features 6. Modified the Gleam extension to use the new API structure ## Motivation These changes provide extension developers with more flexibility and control over HTTP requests. The streaming support is particularly useful for handling large responses efficiently or ideally streaming into the UI. ## Testing - [x] Updated existing tests - [ ] Added new tests for streaming functionality ## Next Steps - Consider adding more comprehensive examples in the documentation - Evaluate performance impact of streaming for large responses Please review and let me know if any adjustments are needed. Release Notes: - N/A --------- Co-authored-by: Marshall Bowers --- .../src/wasm_host/wit/since_v0_0_7.rs | 130 +++++++++++++++--- crates/extension_api/src/extension_api.rs | 4 +- .../wit/since_v0.0.7/http-client.wit | 31 ++++- extensions/gleam/src/gleam.rs | 15 +- extensions/gleam/src/hexdocs.rs | 21 ++- 5 files changed, 171 insertions(+), 30 deletions(-) diff --git a/crates/extension/src/wasm_host/wit/since_v0_0_7.rs b/crates/extension/src/wasm_host/wit/since_v0_0_7.rs index ca52f1d47395bd8857504d25b54bb41635b031cf..e5cf38c076cb157257f0c72ee9dfaa07acee530a 100644 --- a/crates/extension/src/wasm_host/wit/since_v0_0_7.rs +++ b/crates/extension/src/wasm_host/wit/since_v0_0_7.rs @@ -5,9 +5,10 @@ use anyhow::{anyhow, bail, Context, Result}; use async_compression::futures::bufread::GzipDecoder; use async_tar::Archive; use async_trait::async_trait; -use futures::AsyncReadExt; use futures::{io::BufReader, FutureExt as _}; +use futures::{lock::Mutex, AsyncReadExt}; use indexed_docs::IndexedDocsDatabase; +use isahc::config::{Configurable, RedirectPolicy}; use language::{ language_settings::AllLanguageSettings, LanguageServerBinaryStatus, LspAdapterDelegate, }; @@ -30,7 +31,8 @@ wasmtime::component::bindgen!({ path: "../extension_api/wit/since_v0.0.7", with: { "worktree": ExtensionWorktree, - "key-value-store": ExtensionKeyValueStore + "key-value-store": ExtensionKeyValueStore, + "zed:extension/http-client/http-response-stream": ExtensionHttpResponseStream }, }); @@ -41,8 +43,8 @@ mod settings { } pub type ExtensionWorktree = Arc; - pub type ExtensionKeyValueStore = Arc; +pub type ExtensionHttpResponseStream = Arc>>; pub fn linker() -> &'static Linker { static LINKER: OnceLock> = OnceLock::new(); @@ -130,35 +132,123 @@ impl common::Host for WasmState {} impl http_client::Host for WasmState { async fn fetch( &mut self, - req: http_client::HttpRequest, + request: http_client::HttpRequest, ) -> wasmtime::Result> { maybe!(async { - let url = &req.url; - - let mut response = self - .host - .http_client - .get(url, AsyncBody::default(), true) - .await?; + let url = &request.url; + let request = convert_request(&request, true)?; + let mut response = self.host.http_client.send(request).await?; if response.status().is_client_error() || response.status().is_server_error() { bail!("failed to fetch '{url}': status code {}", response.status()) } + convert_response(&mut response).await + }) + .await + .to_wasmtime_result() + } - let mut body = Vec::new(); - response - .body_mut() - .read_to_end(&mut body) - .await - .with_context(|| format!("failed to read response body from '{url}'"))?; + async fn fetch_stream( + &mut self, + request: http_client::HttpRequest, + ) -> wasmtime::Result, String>> { + let request = convert_request(&request, true)?; + let response = self.host.http_client.send(request); + maybe!(async { + let response = response.await?; + let stream = Arc::new(Mutex::new(response)); + let resource = self.table.push(stream)?; + Ok(resource) + }) + .await + .to_wasmtime_result() + } +} - Ok(http_client::HttpResponse { - body: String::from_utf8(body)?, - }) +#[async_trait] +impl http_client::HostHttpResponseStream for WasmState { + async fn next_chunk( + &mut self, + resource: Resource, + ) -> wasmtime::Result>, String>> { + let stream = self.table.get(&resource)?.clone(); + maybe!(async move { + let mut response = stream.lock().await; + let mut buffer = vec![0; 8192]; // 8KB buffer + let bytes_read = response.body_mut().read(&mut buffer).await?; + if bytes_read == 0 { + Ok(None) + } else { + buffer.truncate(bytes_read); + Ok(Some(buffer)) + } }) .await .to_wasmtime_result() } + + fn drop(&mut self, _resource: Resource) -> Result<()> { + Ok(()) + } +} + +impl From for ::http_client::Method { + fn from(value: http_client::HttpMethod) -> Self { + match value { + http_client::HttpMethod::Get => Self::GET, + http_client::HttpMethod::Post => Self::POST, + http_client::HttpMethod::Put => Self::PUT, + http_client::HttpMethod::Delete => Self::DELETE, + http_client::HttpMethod::Head => Self::HEAD, + http_client::HttpMethod::Options => Self::OPTIONS, + http_client::HttpMethod::Patch => Self::PATCH, + } + } +} + +fn convert_request( + extension_request: &http_client::HttpRequest, + follow_redirects: bool, +) -> Result<::http_client::Request, anyhow::Error> { + let mut request = ::http_client::Request::builder() + .method(::http_client::Method::from(extension_request.method)) + .uri(&extension_request.url) + .redirect_policy(if follow_redirects { + RedirectPolicy::Follow + } else { + RedirectPolicy::None + }); + for (key, value) in &extension_request.headers { + request = request.header(key, value); + } + let body = extension_request + .body + .clone() + .map(AsyncBody::from) + .unwrap_or_default(); + request.body(body).map_err(anyhow::Error::from) +} + +async fn convert_response( + response: &mut ::http_client::Response, +) -> Result { + let mut extension_response = http_client::HttpResponse { + body: Vec::new(), + headers: Vec::new(), + }; + + for (key, value) in response.headers() { + extension_response + .headers + .push((key.to_string(), value.to_str().unwrap_or("").to_string())); + } + + response + .body_mut() + .read_to_end(&mut extension_response.body) + .await?; + + Ok(extension_response) } #[async_trait] diff --git a/crates/extension_api/src/extension_api.rs b/crates/extension_api/src/extension_api.rs index e07df1171d04878231e2ac361ba2f123ea8c7ff6..ed5b0b22a07e75f1272c06edaf36ef3248461d78 100644 --- a/crates/extension_api/src/extension_api.rs +++ b/crates/extension_api/src/extension_api.rs @@ -19,7 +19,9 @@ pub use wit::{ github_release_by_tag_name, latest_github_release, GithubRelease, GithubReleaseAsset, GithubReleaseOptions, }, - zed::extension::http_client::{fetch, HttpRequest, HttpResponse}, + zed::extension::http_client::{ + fetch, fetch_stream, HttpMethod, HttpRequest, HttpResponse, HttpResponseStream, + }, zed::extension::nodejs::{ node_binary_path, npm_install_package, npm_package_installed_version, npm_package_latest_version, diff --git a/crates/extension_api/wit/since_v0.0.7/http-client.wit b/crates/extension_api/wit/since_v0.0.7/http-client.wit index e1f7b69d49bf59019abe1cb191121df4efaaa3a2..a2a847c72d52a7b83f6345e9c6ad43e94d652557 100644 --- a/crates/extension_api/wit/since_v0.0.7/http-client.wit +++ b/crates/extension_api/wit/since_v0.0.7/http-client.wit @@ -1,16 +1,45 @@ interface http-client { /// An HTTP request. record http-request { + /// The HTTP method for the request. + method: http-method, /// The URL to which the request should be made. url: string, + /// Headers for the request. + headers: list>, + /// The request body. + body: option>, + } + + /// HTTP methods. + enum http-method { + get, + post, + put, + delete, + head, + options, + patch, } /// An HTTP response. record http-response { + /// The response headers. + headers: list>, /// The response body. - body: string, + body: list, } /// Performs an HTTP request and returns the response. fetch: func(req: http-request) -> result; + + /// An HTTP response stream. + resource http-response-stream { + /// Retrieves the next chunk of data from the response stream. + /// Returns None if the stream has ended. + next-chunk: func() -> result>, string>; + } + + /// Performs an HTTP request and returns a response stream. + fetch-stream: func(req: http-request) -> result; } diff --git a/extensions/gleam/src/gleam.rs b/extensions/gleam/src/gleam.rs index 244bc8ec9ba32f829c5ea64bd4a1bdc0d62f7ab3..a95231e15d11a9a89eb658b9d8dc0562c26d3209 100644 --- a/extensions/gleam/src/gleam.rs +++ b/extensions/gleam/src/gleam.rs @@ -1,10 +1,10 @@ mod hexdocs; -use std::fs; +use std::{fs, io}; use zed::lsp::CompletionKind; use zed::{ - CodeLabel, CodeLabelSpan, HttpRequest, KeyValueStore, LanguageServerId, SlashCommand, - SlashCommandArgumentCompletion, SlashCommandOutput, SlashCommandOutputSection, + CodeLabel, CodeLabelSpan, HttpMethod, HttpRequest, KeyValueStore, LanguageServerId, + SlashCommand, SlashCommandArgumentCompletion, SlashCommandOutput, SlashCommandOutputSection, }; use zed_extension_api::{self as zed, Result}; @@ -194,6 +194,7 @@ impl zed::Extension for GleamExtension { let module_path = components.map(ToString::to_string).collect::>(); let response = zed::fetch(&HttpRequest { + method: HttpMethod::Get, url: format!( "https://hexdocs.pm/{package_name}{maybe_path}", maybe_path = if !module_path.is_empty() { @@ -202,9 +203,15 @@ impl zed::Extension for GleamExtension { String::new() } ), + headers: vec![( + "User-Agent".to_string(), + "Zed (Gleam Extension)".to_string(), + )], + body: None, })?; - let (markdown, _modules) = convert_hexdocs_to_markdown(response.body.as_bytes())?; + let (markdown, _modules) = + convert_hexdocs_to_markdown(&mut io::Cursor::new(response.body))?; let mut text = String::new(); text.push_str(&markdown); diff --git a/extensions/gleam/src/hexdocs.rs b/extensions/gleam/src/hexdocs.rs index 26198fc5d160d7cc655ea031dd26688f42aeebf6..07be1424daccf8cf9d3ce85d6523157bcef1ff6a 100644 --- a/extensions/gleam/src/hexdocs.rs +++ b/extensions/gleam/src/hexdocs.rs @@ -1,6 +1,6 @@ use std::cell::RefCell; use std::collections::BTreeSet; -use std::io::Read; +use std::io::{self, Read}; use std::rc::Rc; use html_to_markdown::markdown::{ @@ -10,23 +10,36 @@ use html_to_markdown::{ convert_html_to_markdown, HandleTag, HandlerOutcome, HtmlElement, MarkdownWriter, StartTagOutcome, TagHandler, }; -use zed_extension_api::{self as zed, HttpRequest, KeyValueStore, Result}; +use zed_extension_api::{self as zed, HttpMethod, HttpRequest, KeyValueStore, Result}; pub fn index(package: String, database: &KeyValueStore) -> Result<()> { + let headers = vec![( + "User-Agent".to_string(), + "Zed (Gleam Extension)".to_string(), + )]; + let response = zed::fetch(&HttpRequest { + method: HttpMethod::Get, url: format!("https://hexdocs.pm/{package}"), + headers: headers.clone(), + body: None, })?; - let (package_root_markdown, modules) = convert_hexdocs_to_markdown(response.body.as_bytes())?; + let (package_root_markdown, modules) = + convert_hexdocs_to_markdown(&mut io::Cursor::new(&response.body))?; database.insert(&package, &package_root_markdown)?; for module in modules { let response = zed::fetch(&HttpRequest { + method: HttpMethod::Get, url: format!("https://hexdocs.pm/{package}/{module}.html"), + headers: headers.clone(), + body: None, })?; - let (markdown, _modules) = convert_hexdocs_to_markdown(response.body.as_bytes())?; + let (markdown, _modules) = + convert_hexdocs_to_markdown(&mut io::Cursor::new(&response.body))?; database.insert(&format!("{module} ({package})"), &markdown)?; }