From e4bb666eabed7d13536fe5d9df71e4d4b9e63374 Mon Sep 17 00:00:00 2001 From: Marshall Bowers Date: Tue, 4 Jun 2024 11:56:23 -0400 Subject: [PATCH] assistant: Add `/fetch` slash command (#12645) This PR adds a new `/fetch` slash command to the Assistant for fetching the content of an arbitrary URL as Markdown. Currently it's just using the same HTML to Markdown conversion that `/rustdoc` uses, but I'll be working to refine the output to be more widely useful. Release Notes: - N/A --- crates/assistant/src/assistant.rs | 5 +- crates/assistant/src/slash_command.rs | 1 + .../src/slash_command/fetch_command.rs | 133 ++++++++++++++++++ .../src/rustdoc_to_markdown.rs | 34 +++-- 4 files changed, 163 insertions(+), 10 deletions(-) create mode 100644 crates/assistant/src/slash_command/fetch_command.rs diff --git a/crates/assistant/src/assistant.rs b/crates/assistant/src/assistant.rs index 8d86988306ef0840a32b4f4510f22e0ea5c4d174..3f3ecbc86210264735618e65ffc1cd05ebe1e8a3 100644 --- a/crates/assistant/src/assistant.rs +++ b/crates/assistant/src/assistant.rs @@ -25,8 +25,8 @@ use semantic_index::{CloudEmbeddingProvider, SemanticIndex}; use serde::{Deserialize, Serialize}; use settings::{Settings, SettingsStore}; use slash_command::{ - active_command, file_command, project_command, prompt_command, rustdoc_command, search_command, - tabs_command, + active_command, fetch_command, file_command, project_command, prompt_command, rustdoc_command, + search_command, tabs_command, }; use std::{ fmt::{self, Display}, @@ -304,6 +304,7 @@ fn register_slash_commands(cx: &mut AppContext) { slash_command_registry.register_command(project_command::ProjectSlashCommand, true); slash_command_registry.register_command(search_command::SearchSlashCommand, true); slash_command_registry.register_command(rustdoc_command::RustdocSlashCommand, false); + slash_command_registry.register_command(fetch_command::FetchSlashCommand, false); let store = PromptStore::global(cx); cx.background_executor() diff --git a/crates/assistant/src/slash_command.rs b/crates/assistant/src/slash_command.rs index ce99ca141a071f89000d1d0bb04d70173e31b143..98ee1248a02c7a7d44f1bdfd544ad78c7479c29a 100644 --- a/crates/assistant/src/slash_command.rs +++ b/crates/assistant/src/slash_command.rs @@ -17,6 +17,7 @@ use std::{ use workspace::Workspace; pub mod active_command; +pub mod fetch_command; pub mod file_command; pub mod project_command; pub mod prompt_command; diff --git a/crates/assistant/src/slash_command/fetch_command.rs b/crates/assistant/src/slash_command/fetch_command.rs new file mode 100644 index 0000000000000000000000000000000000000000..b28bb77f5dba12581716ffa64e3615cb56fc7659 --- /dev/null +++ b/crates/assistant/src/slash_command/fetch_command.rs @@ -0,0 +1,133 @@ +use std::sync::atomic::AtomicBool; +use std::sync::Arc; + +use anyhow::{anyhow, bail, Context, Result}; +use assistant_slash_command::{SlashCommand, SlashCommandOutput, SlashCommandOutputSection}; +use futures::AsyncReadExt; +use gpui::{AppContext, Task, WeakView}; +use http::{AsyncBody, HttpClient, HttpClientWithUrl}; +use language::LspAdapterDelegate; +use rustdoc_to_markdown::convert_html_to_markdown; +use ui::{prelude::*, ButtonLike, ElevationIndex}; +use workspace::Workspace; + +pub(crate) struct FetchSlashCommand; + +impl FetchSlashCommand { + async fn build_message(http_client: Arc, url: &str) -> Result { + let mut url = url.to_owned(); + if !url.starts_with("https://") { + url = format!("https://{url}"); + } + + let mut response = http_client.get(&url, AsyncBody::default(), true).await?; + + let mut body = Vec::new(); + response + .body_mut() + .read_to_end(&mut body) + .await + .context("error reading response body")?; + + if response.status().is_client_error() { + let text = String::from_utf8_lossy(body.as_slice()); + bail!( + "status error {}, response: {text:?}", + response.status().as_u16() + ); + } + + convert_html_to_markdown(&body[..]) + } +} + +impl SlashCommand for FetchSlashCommand { + fn name(&self) -> String { + "fetch".into() + } + + fn description(&self) -> String { + "insert URL contents".into() + } + + fn menu_text(&self) -> String { + "Insert fetched URL contents".into() + } + + fn requires_argument(&self) -> bool { + true + } + + fn complete_argument( + &self, + _query: String, + _cancel: Arc, + _workspace: WeakView, + _cx: &mut AppContext, + ) -> Task>> { + Task::ready(Ok(Vec::new())) + } + + fn run( + self: Arc, + argument: Option<&str>, + workspace: WeakView, + _delegate: Arc, + cx: &mut WindowContext, + ) -> Task> { + let Some(argument) = argument else { + return Task::ready(Err(anyhow!("missing URL"))); + }; + let Some(workspace) = workspace.upgrade() else { + return Task::ready(Err(anyhow!("workspace was dropped"))); + }; + + let http_client = workspace.read(cx).client().http_client(); + let url = argument.to_string(); + + let text = cx.background_executor().spawn({ + let url = url.clone(); + async move { Self::build_message(http_client, &url).await } + }); + + let url = SharedString::from(url); + cx.foreground_executor().spawn(async move { + let text = text.await?; + let range = 0..text.len(); + Ok(SlashCommandOutput { + text, + sections: vec![SlashCommandOutputSection { + range, + render_placeholder: Arc::new(move |id, unfold, _cx| { + FetchPlaceholder { + id, + unfold, + url: url.clone(), + } + .into_any_element() + }), + }], + }) + }) + } +} + +#[derive(IntoElement)] +struct FetchPlaceholder { + pub id: ElementId, + pub unfold: Arc, + pub url: SharedString, +} + +impl RenderOnce for FetchPlaceholder { + fn render(self, _cx: &mut WindowContext) -> impl IntoElement { + let unfold = self.unfold; + + ButtonLike::new(self.id) + .style(ButtonStyle::Filled) + .layer(ElevationIndex::ElevatedSurface) + .child(Icon::new(IconName::AtSign)) + .child(Label::new(format!("fetch {url}", url = self.url))) + .on_click(move |_, cx| unfold(cx)) + } +} diff --git a/crates/rustdoc_to_markdown/src/rustdoc_to_markdown.rs b/crates/rustdoc_to_markdown/src/rustdoc_to_markdown.rs index 54cf11c5871a02c4f3785d63555bd9ff25d88f14..05d0b531289af8a2df4fd5e6bee7abc375fcd72c 100644 --- a/crates/rustdoc_to_markdown/src/rustdoc_to_markdown.rs +++ b/crates/rustdoc_to_markdown/src/rustdoc_to_markdown.rs @@ -16,8 +16,31 @@ use markup5ever_rcdom::RcDom; use crate::markdown_writer::MarkdownWriter; +/// Converts the provided HTML to Markdown. +pub fn convert_html_to_markdown(html: impl Read) -> Result { + let dom = parse_html(html).context("failed to parse HTML")?; + + let markdown_writer = MarkdownWriter::new(); + let markdown = markdown_writer + .run(&dom.document) + .context("failed to convert HTML to Markdown")?; + + Ok(markdown) +} + /// Converts the provided rustdoc HTML to Markdown. -pub fn convert_rustdoc_to_markdown(mut html: impl Read) -> Result { +pub fn convert_rustdoc_to_markdown(html: impl Read) -> Result { + let dom = parse_html(html).context("failed to parse rustdoc HTML")?; + + let markdown_writer = MarkdownWriter::new(); + let markdown = markdown_writer + .run(&dom.document) + .context("failed to convert rustdoc HTML to Markdown")?; + + Ok(markdown) +} + +fn parse_html(mut html: impl Read) -> Result { let parse_options = ParseOpts { tree_builder: TreeBuilderOpts { drop_doctype: true, @@ -28,14 +51,9 @@ pub fn convert_rustdoc_to_markdown(mut html: impl Read) -> Result { let dom = parse_document(RcDom::default(), parse_options) .from_utf8() .read_from(&mut html) - .context("failed to parse rustdoc HTML")?; + .context("failed to parse HTML document")?; - let markdown_writer = MarkdownWriter::new(); - let markdown = markdown_writer - .run(&dom.document) - .context("failed to convert rustdoc to HTML")?; - - Ok(markdown) + Ok(dom) } #[cfg(test)]