More assistant events (#18032)

Conrad Irwin created 1 year ago

Release Notes:

- N/A

Change summary

crates/assistant/src/context.rs                   |  3 +
crates/assistant/src/inline_assistant.rs          | 29 +++++++++++++++++
crates/assistant/src/terminal_inline_assistant.rs |  1 
crates/client/src/telemetry.rs                    |  8 ++-
crates/collab/src/api/events.rs                   |  2 +
crates/telemetry_events/src/telemetry_events.rs   | 28 +++++++++++++++
6 files changed, 66 insertions(+), 5 deletions(-)

Detailed changes

crates/assistant/src/context.rs

@@ -46,7 +46,7 @@ use std::{
     sync::Arc,
     time::{Duration, Instant},
 };
-use telemetry_events::AssistantKind;
+use telemetry_events::{AssistantKind, AssistantPhase};
 use text::BufferSnapshot;
 use util::{post_inc, ResultExt, TryFutureExt};
 use uuid::Uuid;
@@ -2134,6 +2134,7 @@ impl Context {
                         telemetry.report_assistant_event(
                             Some(this.id.0.clone()),
                             AssistantKind::Panel,
+                            AssistantPhase::Response,
                             model.telemetry_id(),
                             response_latency,
                             error_message,

crates/assistant/src/inline_assistant.rs

@@ -174,6 +174,18 @@ impl InlineAssistant {
         initial_prompt: Option<String>,
         cx: &mut WindowContext,
     ) {
+        if let Some(telemetry) = self.telemetry.as_ref() {
+            if let Some(model) = LanguageModelRegistry::read_global(cx).active_model() {
+                telemetry.report_assistant_event(
+                    None,
+                    telemetry_events::AssistantKind::Inline,
+                    telemetry_events::AssistantPhase::Invoked,
+                    model.telemetry_id(),
+                    None,
+                    None,
+                );
+            }
+        }
         let snapshot = editor.read(cx).buffer().read(cx).snapshot(cx);
 
         let mut selections = Vec::<Selection<Point>>::new();
@@ -708,6 +720,22 @@ impl InlineAssistant {
     }
 
     pub fn finish_assist(&mut self, assist_id: InlineAssistId, undo: bool, cx: &mut WindowContext) {
+        if let Some(telemetry) = self.telemetry.as_ref() {
+            if let Some(model) = LanguageModelRegistry::read_global(cx).active_model() {
+                telemetry.report_assistant_event(
+                    None,
+                    telemetry_events::AssistantKind::Inline,
+                    if undo {
+                        telemetry_events::AssistantPhase::Rejected
+                    } else {
+                        telemetry_events::AssistantPhase::Accepted
+                    },
+                    model.telemetry_id(),
+                    None,
+                    None,
+                );
+            }
+        }
         if let Some(assist) = self.assists.get(&assist_id) {
             let assist_group_id = assist.group_id;
             if self.assist_groups[&assist_group_id].linked {
@@ -2558,6 +2586,7 @@ impl Codegen {
                                 telemetry.report_assistant_event(
                                     None,
                                     telemetry_events::AssistantKind::Inline,
+                                    telemetry_events::AssistantPhase::Response,
                                     model_telemetry_id,
                                     response_latency,
                                     error_message,

crates/assistant/src/terminal_inline_assistant.rs

@@ -1066,6 +1066,7 @@ impl Codegen {
                         telemetry.report_assistant_event(
                             None,
                             telemetry_events::AssistantKind::Inline,
+                            telemetry_events::AssistantPhase::Response,
                             model_telemetry_id,
                             response_latency,
                             error_message,

crates/client/src/telemetry.rs

@@ -16,9 +16,9 @@ use std::io::Write;
 use std::{env, mem, path::PathBuf, sync::Arc, time::Duration};
 use sysinfo::{CpuRefreshKind, Pid, ProcessRefreshKind, RefreshKind, System};
 use telemetry_events::{
-    ActionEvent, AppEvent, AssistantEvent, AssistantKind, CallEvent, CpuEvent, EditEvent,
-    EditorEvent, Event, EventRequestBody, EventWrapper, ExtensionEvent, InlineCompletionEvent,
-    MemoryEvent, ReplEvent, SettingEvent,
+    ActionEvent, AppEvent, AssistantEvent, AssistantKind, AssistantPhase, CallEvent, CpuEvent,
+    EditEvent, EditorEvent, Event, EventRequestBody, EventWrapper, ExtensionEvent,
+    InlineCompletionEvent, MemoryEvent, ReplEvent, SettingEvent,
 };
 use tempfile::NamedTempFile;
 #[cfg(not(debug_assertions))]
@@ -391,6 +391,7 @@ impl Telemetry {
         self: &Arc<Self>,
         conversation_id: Option<String>,
         kind: AssistantKind,
+        phase: AssistantPhase,
         model: String,
         response_latency: Option<Duration>,
         error_message: Option<String>,
@@ -398,6 +399,7 @@ impl Telemetry {
         let event = Event::Assistant(AssistantEvent {
             conversation_id,
             kind,
+            phase,
             model: model.to_string(),
             response_latency,
             error_message,

crates/collab/src/api/events.rs

@@ -834,6 +834,7 @@ pub struct AssistantEventRow {
     // AssistantEventRow
     conversation_id: String,
     kind: String,
+    phase: String,
     model: String,
     response_latency_in_ms: Option<i64>,
     error_message: Option<String>,
@@ -866,6 +867,7 @@ impl AssistantEventRow {
             time: time.timestamp_millis(),
             conversation_id: event.conversation_id.unwrap_or_default(),
             kind: event.kind.to_string(),
+            phase: event.phase.to_string(),
             model: event.model,
             response_latency_in_ms: event
                 .response_latency

crates/telemetry_events/src/telemetry_events.rs

@@ -44,7 +44,6 @@ pub enum AssistantKind {
     Panel,
     Inline,
 }
-
 impl Display for AssistantKind {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         write!(
@@ -58,6 +57,31 @@ impl Display for AssistantKind {
     }
 }
 
+#[derive(Default, Clone, Debug, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum AssistantPhase {
+    #[default]
+    Response,
+    Invoked,
+    Accepted,
+    Rejected,
+}
+
+impl Display for AssistantPhase {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            "{}",
+            match self {
+                Self::Response => "response",
+                Self::Invoked => "invoked",
+                Self::Accepted => "accepted",
+                Self::Rejected => "rejected",
+            }
+        )
+    }
+}
+
 #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
 #[serde(tag = "type")]
 pub enum Event {
@@ -121,6 +145,8 @@ pub struct AssistantEvent {
     pub conversation_id: Option<String>,
     /// The kind of assistant (Panel, Inline)
     pub kind: AssistantKind,
+    #[serde(default)]
+    pub phase: AssistantPhase,
     /// Name of the AI model used (gpt-4o, claude-3-5-sonnet, etc)
     pub model: String,
     pub response_latency: Option<Duration>,