Detailed changes
@@ -5,9 +5,10 @@ use anyhow::{anyhow, bail, Context, Result};
use async_compression::futures::bufread::GzipDecoder;
use async_tar::Archive;
use async_trait::async_trait;
-use futures::AsyncReadExt;
use futures::{io::BufReader, FutureExt as _};
+use futures::{lock::Mutex, AsyncReadExt};
use indexed_docs::IndexedDocsDatabase;
+use isahc::config::{Configurable, RedirectPolicy};
use language::{
language_settings::AllLanguageSettings, LanguageServerBinaryStatus, LspAdapterDelegate,
};
@@ -30,7 +31,8 @@ wasmtime::component::bindgen!({
path: "../extension_api/wit/since_v0.0.7",
with: {
"worktree": ExtensionWorktree,
- "key-value-store": ExtensionKeyValueStore
+ "key-value-store": ExtensionKeyValueStore,
+ "zed:extension/http-client/http-response-stream": ExtensionHttpResponseStream
},
});
@@ -41,8 +43,8 @@ mod settings {
}
pub type ExtensionWorktree = Arc<dyn LspAdapterDelegate>;
-
pub type ExtensionKeyValueStore = Arc<IndexedDocsDatabase>;
+pub type ExtensionHttpResponseStream = Arc<Mutex<::http_client::Response<AsyncBody>>>;
pub fn linker() -> &'static Linker<WasmState> {
static LINKER: OnceLock<Linker<WasmState>> = OnceLock::new();
@@ -130,35 +132,123 @@ impl common::Host for WasmState {}
impl http_client::Host for WasmState {
async fn fetch(
&mut self,
- req: http_client::HttpRequest,
+ request: http_client::HttpRequest,
) -> wasmtime::Result<Result<http_client::HttpResponse, String>> {
maybe!(async {
- let url = &req.url;
-
- let mut response = self
- .host
- .http_client
- .get(url, AsyncBody::default(), true)
- .await?;
+ let url = &request.url;
+ let request = convert_request(&request, true)?;
+ let mut response = self.host.http_client.send(request).await?;
if response.status().is_client_error() || response.status().is_server_error() {
bail!("failed to fetch '{url}': status code {}", response.status())
}
+ convert_response(&mut response).await
+ })
+ .await
+ .to_wasmtime_result()
+ }
- let mut body = Vec::new();
- response
- .body_mut()
- .read_to_end(&mut body)
- .await
- .with_context(|| format!("failed to read response body from '{url}'"))?;
+ async fn fetch_stream(
+ &mut self,
+ request: http_client::HttpRequest,
+ ) -> wasmtime::Result<Result<Resource<ExtensionHttpResponseStream>, String>> {
+ let request = convert_request(&request, true)?;
+ let response = self.host.http_client.send(request);
+ maybe!(async {
+ let response = response.await?;
+ let stream = Arc::new(Mutex::new(response));
+ let resource = self.table.push(stream)?;
+ Ok(resource)
+ })
+ .await
+ .to_wasmtime_result()
+ }
+}
- Ok(http_client::HttpResponse {
- body: String::from_utf8(body)?,
- })
+#[async_trait]
+impl http_client::HostHttpResponseStream for WasmState {
+ async fn next_chunk(
+ &mut self,
+ resource: Resource<ExtensionHttpResponseStream>,
+ ) -> wasmtime::Result<Result<Option<Vec<u8>>, String>> {
+ let stream = self.table.get(&resource)?.clone();
+ maybe!(async move {
+ let mut response = stream.lock().await;
+ let mut buffer = vec![0; 8192]; // 8KB buffer
+ let bytes_read = response.body_mut().read(&mut buffer).await?;
+ if bytes_read == 0 {
+ Ok(None)
+ } else {
+ buffer.truncate(bytes_read);
+ Ok(Some(buffer))
+ }
})
.await
.to_wasmtime_result()
}
+
+ fn drop(&mut self, _resource: Resource<ExtensionHttpResponseStream>) -> Result<()> {
+ Ok(())
+ }
+}
+
+impl From<http_client::HttpMethod> for ::http_client::Method {
+ fn from(value: http_client::HttpMethod) -> Self {
+ match value {
+ http_client::HttpMethod::Get => Self::GET,
+ http_client::HttpMethod::Post => Self::POST,
+ http_client::HttpMethod::Put => Self::PUT,
+ http_client::HttpMethod::Delete => Self::DELETE,
+ http_client::HttpMethod::Head => Self::HEAD,
+ http_client::HttpMethod::Options => Self::OPTIONS,
+ http_client::HttpMethod::Patch => Self::PATCH,
+ }
+ }
+}
+
+fn convert_request(
+ extension_request: &http_client::HttpRequest,
+ follow_redirects: bool,
+) -> Result<::http_client::Request<AsyncBody>, anyhow::Error> {
+ let mut request = ::http_client::Request::builder()
+ .method(::http_client::Method::from(extension_request.method))
+ .uri(&extension_request.url)
+ .redirect_policy(if follow_redirects {
+ RedirectPolicy::Follow
+ } else {
+ RedirectPolicy::None
+ });
+ for (key, value) in &extension_request.headers {
+ request = request.header(key, value);
+ }
+ let body = extension_request
+ .body
+ .clone()
+ .map(AsyncBody::from)
+ .unwrap_or_default();
+ request.body(body).map_err(anyhow::Error::from)
+}
+
+async fn convert_response(
+ response: &mut ::http_client::Response<AsyncBody>,
+) -> Result<http_client::HttpResponse, anyhow::Error> {
+ let mut extension_response = http_client::HttpResponse {
+ body: Vec::new(),
+ headers: Vec::new(),
+ };
+
+ for (key, value) in response.headers() {
+ extension_response
+ .headers
+ .push((key.to_string(), value.to_str().unwrap_or("").to_string()));
+ }
+
+ response
+ .body_mut()
+ .read_to_end(&mut extension_response.body)
+ .await?;
+
+ Ok(extension_response)
}
#[async_trait]
@@ -19,7 +19,9 @@ pub use wit::{
github_release_by_tag_name, latest_github_release, GithubRelease, GithubReleaseAsset,
GithubReleaseOptions,
},
- zed::extension::http_client::{fetch, HttpRequest, HttpResponse},
+ zed::extension::http_client::{
+ fetch, fetch_stream, HttpMethod, HttpRequest, HttpResponse, HttpResponseStream,
+ },
zed::extension::nodejs::{
node_binary_path, npm_install_package, npm_package_installed_version,
npm_package_latest_version,
@@ -1,16 +1,45 @@
interface http-client {
/// An HTTP request.
record http-request {
+ /// The HTTP method for the request.
+ method: http-method,
/// The URL to which the request should be made.
url: string,
+ /// Headers for the request.
+ headers: list<tuple<string, string>>,
+ /// The request body.
+ body: option<list<u8>>,
+ }
+
+ /// HTTP methods.
+ enum http-method {
+ get,
+ post,
+ put,
+ delete,
+ head,
+ options,
+ patch,
}
/// An HTTP response.
record http-response {
+ /// The response headers.
+ headers: list<tuple<string, string>>,
/// The response body.
- body: string,
+ body: list<u8>,
}
/// Performs an HTTP request and returns the response.
fetch: func(req: http-request) -> result<http-response, string>;
+
+ /// An HTTP response stream.
+ resource http-response-stream {
+ /// Retrieves the next chunk of data from the response stream.
+ /// Returns None if the stream has ended.
+ next-chunk: func() -> result<option<list<u8>>, string>;
+ }
+
+ /// Performs an HTTP request and returns a response stream.
+ fetch-stream: func(req: http-request) -> result<http-response-stream, string>;
}
@@ -1,10 +1,10 @@
mod hexdocs;
-use std::fs;
+use std::{fs, io};
use zed::lsp::CompletionKind;
use zed::{
- CodeLabel, CodeLabelSpan, HttpRequest, KeyValueStore, LanguageServerId, SlashCommand,
- SlashCommandArgumentCompletion, SlashCommandOutput, SlashCommandOutputSection,
+ CodeLabel, CodeLabelSpan, HttpMethod, HttpRequest, KeyValueStore, LanguageServerId,
+ SlashCommand, SlashCommandArgumentCompletion, SlashCommandOutput, SlashCommandOutputSection,
};
use zed_extension_api::{self as zed, Result};
@@ -194,6 +194,7 @@ impl zed::Extension for GleamExtension {
let module_path = components.map(ToString::to_string).collect::<Vec<_>>();
let response = zed::fetch(&HttpRequest {
+ method: HttpMethod::Get,
url: format!(
"https://hexdocs.pm/{package_name}{maybe_path}",
maybe_path = if !module_path.is_empty() {
@@ -202,9 +203,15 @@ impl zed::Extension for GleamExtension {
String::new()
}
),
+ headers: vec![(
+ "User-Agent".to_string(),
+ "Zed (Gleam Extension)".to_string(),
+ )],
+ body: None,
})?;
- let (markdown, _modules) = convert_hexdocs_to_markdown(response.body.as_bytes())?;
+ let (markdown, _modules) =
+ convert_hexdocs_to_markdown(&mut io::Cursor::new(response.body))?;
let mut text = String::new();
text.push_str(&markdown);
@@ -1,6 +1,6 @@
use std::cell::RefCell;
use std::collections::BTreeSet;
-use std::io::Read;
+use std::io::{self, Read};
use std::rc::Rc;
use html_to_markdown::markdown::{
@@ -10,23 +10,36 @@ use html_to_markdown::{
convert_html_to_markdown, HandleTag, HandlerOutcome, HtmlElement, MarkdownWriter,
StartTagOutcome, TagHandler,
};
-use zed_extension_api::{self as zed, HttpRequest, KeyValueStore, Result};
+use zed_extension_api::{self as zed, HttpMethod, HttpRequest, KeyValueStore, Result};
pub fn index(package: String, database: &KeyValueStore) -> Result<()> {
+ let headers = vec![(
+ "User-Agent".to_string(),
+ "Zed (Gleam Extension)".to_string(),
+ )];
+
let response = zed::fetch(&HttpRequest {
+ method: HttpMethod::Get,
url: format!("https://hexdocs.pm/{package}"),
+ headers: headers.clone(),
+ body: None,
})?;
- let (package_root_markdown, modules) = convert_hexdocs_to_markdown(response.body.as_bytes())?;
+ let (package_root_markdown, modules) =
+ convert_hexdocs_to_markdown(&mut io::Cursor::new(&response.body))?;
database.insert(&package, &package_root_markdown)?;
for module in modules {
let response = zed::fetch(&HttpRequest {
+ method: HttpMethod::Get,
url: format!("https://hexdocs.pm/{package}/{module}.html"),
+ headers: headers.clone(),
+ body: None,
})?;
- let (markdown, _modules) = convert_hexdocs_to_markdown(response.body.as_bytes())?;
+ let (markdown, _modules) =
+ convert_hexdocs_to_markdown(&mut io::Cursor::new(&response.body))?;
database.insert(&format!("{module} ({package})"), &markdown)?;
}