From 1870425b2b262e3f28c90931782aba435afd5b99 Mon Sep 17 00:00:00 2001 From: Ben Kunkle Date: Wed, 28 Jan 2026 18:36:44 -0600 Subject: [PATCH] ep_cli: `rated-after:` query (#47906) Closes #ISSUE Release Notes: - N/A *or* Added/Fixed/Improved ... --- Cargo.lock | 1 + crates/edit_prediction/src/capture_example.rs | 4 + crates/edit_prediction/src/example_spec.rs | 16 + crates/edit_prediction_cli/Cargo.toml | 1 + crates/edit_prediction_cli/src/main.rs | 134 ++++--- .../edit_prediction_cli/src/pull_examples.rs | 370 ++++++++++++++++-- .../edit_prediction_cli/src/split_commit.rs | 4 + crates/edit_prediction_cli/src/synthesize.rs | 2 + .../telemetry_events/src/telemetry_events.rs | 2 +- 9 files changed, 445 insertions(+), 89 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 27355fd88341c50063278fb2e19c86e8d14d1f8e..10f433d1f73c9eb4e3887582d7b446ded6108f87 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5399,6 +5399,7 @@ dependencies = [ "smol", "sqlez", "sqlez_macros", + "telemetry_events", "tempfile", "terminal_view", "toml 0.8.23", diff --git a/crates/edit_prediction/src/capture_example.rs b/crates/edit_prediction/src/capture_example.rs index 5d086627c081ce66b078f877c7f4e80fb0645f1c..177df98f135cc97a70de6ba73ca4c711bf6e40b8 100644 --- a/crates/edit_prediction/src/capture_example.rs +++ b/crates/edit_prediction/src/capture_example.rs @@ -175,6 +175,8 @@ pub fn capture_example( rejected_patch, captured_prompt_input: prompt_input, telemetry: None, + human_feedback: Vec::new(), + rating: None, }; spec.set_cursor_excerpt( &cursor_excerpt, @@ -606,6 +608,8 @@ mod tests { ), captured_prompt_input: example.captured_prompt_input.clone(), telemetry: None, + human_feedback: Vec::new(), + rating: None, } ); diff --git a/crates/edit_prediction/src/example_spec.rs b/crates/edit_prediction/src/example_spec.rs index 0d5832e0ac74f9a36372fbacded55864085dfd2a..530a6216216cdf05773c0b47bca07a0ef1e320af 100644 --- a/crates/edit_prediction/src/example_spec.rs +++ b/crates/edit_prediction/src/example_spec.rs @@ -1,6 +1,7 @@ use anyhow::{Context as _, Result}; use serde::{Deserialize, Serialize}; use std::{borrow::Cow, fmt::Write as _, mem, ops::Range, path::Path, sync::Arc}; +use telemetry_events::EditPredictionRating; pub const CURSOR_POSITION_MARKER: &str = "[CURSOR_POSITION]"; pub const INLINE_CURSOR_MARKER: &str = "<|user_cursor|>"; @@ -32,6 +33,15 @@ pub struct ExampleSpec { pub captured_prompt_input: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub telemetry: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub human_feedback: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub rating: Option, +} + +#[derive(Clone, Debug, PartialEq, Hash, Serialize, Deserialize)] +pub struct HumanFeedback { + pub message: String, } /// Metadata for examples sourced from production telemetry (rejected predictions). @@ -252,6 +262,8 @@ impl ExampleSpec { rejected_patch: None, captured_prompt_input: None, telemetry: None, + human_feedback: Vec::new(), + rating: None, }; if let Some(rest) = input.strip_prefix("+++\n") @@ -500,6 +512,8 @@ mod tests { rejected_patch: None, captured_prompt_input: None, telemetry: None, + human_feedback: Vec::new(), + rating: None, }; // Cursor before `42` @@ -635,6 +649,8 @@ mod tests { rejected_patch: None, captured_prompt_input: None, telemetry: None, + human_feedback: Vec::new(), + rating: None, }; // Cursor before `42` using inline marker diff --git a/crates/edit_prediction_cli/Cargo.toml b/crates/edit_prediction_cli/Cargo.toml index 1cb79add3019441c046dc4f9cb3ab0e6cd0b914d..0d81b6fd155f6ddc61a9fee8fcba3325a2acf39f 100644 --- a/crates/edit_prediction_cli/Cargo.toml +++ b/crates/edit_prediction_cli/Cargo.toml @@ -55,6 +55,7 @@ terminal_view.workspace = true util.workspace = true watch.workspace = true edit_prediction = { workspace = true, features = ["cli-support"] } +telemetry_events.workspace = true wasmtime.workspace = true zeta_prompt.workspace = true rand.workspace = true diff --git a/crates/edit_prediction_cli/src/main.rs b/crates/edit_prediction_cli/src/main.rs index 6790491c69ae2888ca78bbd05db76ccacb92f974..f42f8c05789b91a17e2422b316d4eb214e88e143 100644 --- a/crates/edit_prediction_cli/src/main.rs +++ b/crates/edit_prediction_cli/src/main.rs @@ -118,6 +118,19 @@ Inputs can be file paths or special specifiers: Fetch rejected edit predictions from Snowflake after the given RFC3339 timestamp. These are predictions that were shown to users but rejected (useful for DPO training). + rated-after:{timestamp} + Fetch user-rated edit predictions from Snowflake after the given RFC3339 timestamp. + These are predictions that users explicitly rated as positive or negative via the + rate completions modal. Only zeta2 predictions are included. + - Positive ratings: output becomes expected_patches + - Negative ratings: output becomes rejected_patch + + rated-positive-after:{timestamp} + Same as rated-after, but only fetches positively rated predictions. + + rated-negative-after:{timestamp} + Same as rated-after, but only fetches negatively rated predictions. + Required environment variables to connect to Snowflake: EP_SNOWFLAKE_API_KEY EP_SNOWFLAKE_BASE_URL @@ -136,6 +149,15 @@ Examples: # Read rejected predictions for DPO training ep read rejected-after:2025-01-01T00:00:00Z -o rejected.jsonl + # Read user-rated predictions + ep read rated-after:2025-01-01T00:00:00Z -o rated.jsonl + + # Read only positively rated predictions + ep read rated-positive-after:2025-01-01T00:00:00Z -o positive.jsonl + + # Read only negatively rated predictions + ep read rated-negative-after:2025-01-01T00:00:00Z -o negative.jsonl + # Mix multiple input sources ep predict examples.jsonl captured-after:2025-01-01T00:00:00Z "#; @@ -436,6 +458,8 @@ async fn load_examples( let mut captured_after_timestamps = Vec::new(); let mut rejected_after_timestamps = Vec::new(); let mut requested_after_timestamps = Vec::new(); + let mut rated_after_inputs: Vec<(String, Option)> = + Vec::new(); let mut file_inputs = Vec::new(); for input in &args.inputs { @@ -450,6 +474,10 @@ async fn load_examples( pull_examples::parse_requested_after_input(input_string.as_ref()) { requested_after_timestamps.push(timestamp.to_string()); + } else if let Some((timestamp, rating_filter)) = + pull_examples::parse_rated_after_input(input_string.as_ref()) + { + rated_after_inputs.push((timestamp.to_string(), rating_filter)); } else { file_inputs.push(input.clone()); } @@ -499,14 +527,27 @@ async fn load_examples( requested_after_timestamps.sort(); let mut requested_examples = pull_examples::fetch_requested_examples_after( - http_client, + http_client.clone(), &requested_after_timestamps, max_rows_per_timestamp, - background_executor, + background_executor.clone(), ) .await?; examples.append(&mut requested_examples); } + + if !rated_after_inputs.is_empty() { + rated_after_inputs.sort(); + + let mut rated_examples = pull_examples::fetch_rated_examples_after( + http_client, + &rated_after_inputs, + max_rows_per_timestamp, + background_executor, + ) + .await?; + examples.append(&mut rated_examples); + } } crate::example::sort_examples_by_repo_and_rev(&mut examples); @@ -785,63 +826,47 @@ fn main() { let failfast_on_single_example = examples.len() == 1; // For --markdown mode, create the output directory if it doesn't exist - let markdown_output_dir = if args.markdown { + if args.markdown { let dir = output.as_ref().expect("--markdown requires -o"); if !dir.exists() { std::fs::create_dir_all(dir) .expect("Failed to create markdown output directory"); } - Some(dir.clone()) - } else { - None - }; - - // For --in-place, write to a temp file and rename at the end to avoid data loss on interruption - let in_place_temp_path = if args.in_place { - output.as_ref().map(|path| { - let mut temp_path = path.clone(); - temp_path.set_extension("jsonl.tmp"); - temp_path - }) - } else { - None - }; + } - let output_sender: Option> = if !args.markdown - && (args.output.is_some() || !matches!(command, Command::Eval(_))) + // Set up JSONL output writer (not used in markdown mode) + let mut output_sender: Option> = None; + let mut in_place_temp_path: Option = None; + if !args.markdown + && let Some(output_path) = output.as_ref() { - let write_path = in_place_temp_path.as_ref().or(output.as_ref()); - write_path.map(|path| { - let file = if args.in_place { - // For --in-place, write to temp file (truncate if exists) - OpenOptions::new() - .create(true) - .write(true) - .truncate(true) - .open(path) - .expect("Failed to open temp output file") - } else { - // For regular output, append to support resuming - OpenOptions::new() - .create(true) - .append(true) - .open(path) - .expect("Failed to open output file") - }; - let mut writer = BufWriter::new(file); - let (sender, mut receiver) = mpsc::unbounded::(); - cx.background_spawn(async move { - while let Some(line) = receiver.next().await { - writeln!(writer, "{}", line).expect("Failed to write example"); - writer.flush().expect("Failed to flush output"); - } - }) - .detach(); - sender + let write_path = if args.in_place { + let temp = output_path.with_extension("jsonl.tmp"); + in_place_temp_path = Some(temp.clone()); + temp + } else { + output_path.clone() + }; + + let file = OpenOptions::new() + .create(true) + .write(true) + .truncate(args.in_place) + .append(!args.in_place) + .open(&write_path) + .expect("Failed to open output file"); + + let mut writer = BufWriter::new(file); + let (sender, mut receiver) = mpsc::unbounded::(); + cx.background_spawn(async move { + while let Some(line) = receiver.next().await { + writeln!(writer, "{}", line).expect("Failed to write example"); + writer.flush().expect("Failed to flush output"); + } }) - } else { - None - }; + .detach(); + output_sender = Some(sender); + } let grouped_examples = Mutex::new(group_examples_by_repo(examples)); let finished_examples = Mutex::new(Vec::new()); @@ -958,7 +983,9 @@ fn main() { let should_write = !failed || args.failed == FailedHandling::Keep; if should_write { - if let Some(ref markdown_dir) = markdown_output_dir { + if args.markdown { + let markdown_dir = + output.as_ref().expect("--markdown requires -o"); let filename = format!("{}.md", example.spec.filename()); let path = markdown_dir.join(&filename); let markdown = example.spec.to_markdown(); @@ -1044,7 +1071,8 @@ fn main() { }; // For --in-place, atomically rename temp file to original - if let (Some(temp_path), Some(final_path)) = (&in_place_temp_path, &output) { + if let Some(temp_path) = &in_place_temp_path { + let final_path = output.as_ref().expect("in_place_temp_path requires output"); std::fs::rename(temp_path, final_path) .expect("Failed to rename temp file to final output"); } diff --git a/crates/edit_prediction_cli/src/pull_examples.rs b/crates/edit_prediction_cli/src/pull_examples.rs index ce6886fd87fd2464940076d475ce3bc6f0061d9b..953f0edacebf8ccf20450b7b4de2c5ba75baf2bb 100644 --- a/crates/edit_prediction_cli/src/pull_examples.rs +++ b/crates/edit_prediction_cli/src/pull_examples.rs @@ -8,6 +8,7 @@ use serde_json::{Value as JsonValue, json}; use std::io::Read; use std::sync::Arc; use std::time::Duration; +use telemetry_events::EditPredictionRating; use zeta_prompt::ZetaPromptInput; @@ -24,6 +25,7 @@ const SNOWFLAKE_ASYNC_IN_PROGRESS_CODE: &str = "333334"; const EDIT_PREDICTION_EXAMPLE_CAPTURED_EVENT: &str = "Edit Prediction Example Captured"; const PREDICTIVE_EDIT_REQUESTED_EVENT: &str = "Predictive Edit Requested"; const PREDICTIVE_EDIT_REJECTED_EVENT: &str = "Predictive Edit Rejected"; +const EDIT_PREDICTION_RATED_EVENT: &str = "Edit Prediction Rated"; const DEFAULT_STATEMENT_TIMEOUT_SECONDS: u64 = 120; const POLL_INTERVAL: Duration = Duration::from_secs(2); @@ -44,6 +46,21 @@ pub fn parse_requested_after_input(input: &str) -> Option<&str> { input.strip_prefix("requested-after:") } +/// Parse an input token of the form `rated-after:{timestamp}`, `rated-positive-after:{timestamp}`, +/// or `rated-negative-after:{timestamp}`. +/// Returns `(timestamp, Option)` where `None` means all ratings. +pub fn parse_rated_after_input(input: &str) -> Option<(&str, Option)> { + if let Some(timestamp) = input.strip_prefix("rated-positive-after:") { + Some((timestamp, Some(EditPredictionRating::Positive))) + } else if let Some(timestamp) = input.strip_prefix("rated-negative-after:") { + Some((timestamp, Some(EditPredictionRating::Negative))) + } else if let Some(timestamp) = input.strip_prefix("rated-after:") { + Some((timestamp, None)) + } else { + None + } +} + pub async fn fetch_captured_examples_after( http_client: Arc, after_timestamps: &[String], @@ -120,7 +137,21 @@ pub async fn fetch_captured_examples_after( step_progress.set_info(format!("{} rows", total_rows), InfoStyle::Normal); step_progress.set_substatus("parsing"); - all_examples.extend(examples_from_response(&response)?); + let example_index = response + .result_set_meta_data + .as_ref() + .and_then(|m| { + m.row_type.iter().enumerate().find_map(|(index, col)| { + if col.name.eq_ignore_ascii_case("example") { + Some(index) + } else { + None + } + }) + }) + .unwrap_or(0); + + all_examples.extend(examples_from_response(&response, example_index)?); if num_partitions > 1 { let statement_handle = response @@ -144,7 +175,7 @@ pub async fn fetch_captured_examples_after( ) .await?; - all_examples.extend(examples_from_response(&partition_response)?); + all_examples.extend(examples_from_response(&partition_response, example_index)?); } } @@ -192,6 +223,7 @@ struct SnowflakeColumnMeta { fn examples_from_response( response: &SnowflakeStatementResponse, + example_index: usize, ) -> Result + '_> { if let Some(code) = &response.code { if code != SNOWFLAKE_SUCCESS_CODE { @@ -202,20 +234,6 @@ fn examples_from_response( } } - let example_index = response - .result_set_meta_data - .as_ref() - .and_then(|m| { - m.row_type.iter().enumerate().find_map(|(index, col)| { - if col.name.eq_ignore_ascii_case("example") { - Some(index) - } else { - None - } - }) - }) - .unwrap_or(0); - let iter = response.data.iter().enumerate().filter_map(move |(row_index, data_row)| { let Some(example_value) = data_row.get(example_index) else { return None; @@ -527,7 +545,21 @@ pub async fn fetch_rejected_examples_after( step_progress.set_info(format!("{} rows", total_rows), InfoStyle::Normal); step_progress.set_substatus("parsing"); - all_examples.extend(rejected_examples_from_response(&response)?); + let column_indices = get_column_indices( + &response.result_set_meta_data, + &[ + "request_id", + "device_id", + "time", + "input", + "prompt", + "output", + "was_shown", + "reason", + ], + ); + + all_examples.extend(rejected_examples_from_response(&response, &column_indices)?); if num_partitions > 1 { let statement_handle = response @@ -551,7 +583,10 @@ pub async fn fetch_rejected_examples_after( ) .await?; - all_examples.extend(rejected_examples_from_response(&partition_response)?); + all_examples.extend(rejected_examples_from_response( + &partition_response, + &column_indices, + )?); } } @@ -686,6 +721,282 @@ pub async fn fetch_requested_examples_after( Ok(all_examples) } +pub async fn fetch_rated_examples_after( + http_client: Arc, + inputs: &[(String, Option)], + max_rows_per_timestamp: usize, + background_executor: BackgroundExecutor, +) -> Result> { + if inputs.is_empty() { + return Ok(Vec::new()); + } + + let progress = Progress::global(); + + let token = std::env::var("EP_SNOWFLAKE_API_KEY") + .context("missing required environment variable EP_SNOWFLAKE_API_KEY")?; + let base_url = std::env::var("EP_SNOWFLAKE_BASE_URL").context( + "missing required environment variable EP_SNOWFLAKE_BASE_URL (e.g. https://.snowflakecomputing.com)", + )?; + let role = std::env::var("EP_SNOWFLAKE_ROLE").ok(); + + let mut all_examples = Vec::new(); + + for (after_date, rating_filter) in inputs.iter() { + let filter_label = match rating_filter { + None => "", + Some(EditPredictionRating::Positive) => ":positive", + Some(EditPredictionRating::Negative) => ":negative", + }; + let step_progress_name = format!("rated{filter_label}>{after_date}"); + let step_progress = progress.start(Step::PullExamples, &step_progress_name); + step_progress.set_substatus("querying"); + + let rating_value = rating_filter.as_ref().map(|r| match r { + EditPredictionRating::Positive => "Positive", + EditPredictionRating::Negative => "Negative", + }); + + let statement = indoc! {r#" + SELECT + event_properties:inputs AS inputs, + event_properties:output::string AS output, + event_properties:rating::string AS rating, + event_properties:feedback::string AS feedback, + device_id::string AS device_id, + time::string AS time + FROM events + WHERE event_type = ? + AND (? IS NULL OR event_properties:rating::string = ?) + AND time > TRY_TO_TIMESTAMP_NTZ(?) + AND event_properties:inputs IS NOT NULL + AND event_properties:inputs:cursor_excerpt IS NOT NULL + AND event_properties:output IS NOT NULL + ORDER BY time ASC + LIMIT ? + "#}; + + let bindings = json!({ + "1": { "type": "TEXT", "value": EDIT_PREDICTION_RATED_EVENT }, + "2": { "type": "TEXT", "value": rating_value }, + "3": { "type": "TEXT", "value": rating_value }, + "4": { "type": "TEXT", "value": after_date }, + "5": { "type": "FIXED", "value": max_rows_per_timestamp.to_string() } + }); + + let request = json!({ + "statement": statement, + "timeout": DEFAULT_STATEMENT_TIMEOUT_SECONDS, + "database": "EVENTS", + "schema": "PUBLIC", + "warehouse": "DBT", + "role": role, + "bindings": bindings + }); + + let response = run_sql_with_polling( + http_client.clone(), + &base_url, + &token, + &request, + &step_progress, + background_executor.clone(), + ) + .await?; + + let total_rows = response + .result_set_meta_data + .as_ref() + .and_then(|m| m.num_rows) + .unwrap_or(response.data.len() as i64); + + let num_partitions = response + .result_set_meta_data + .as_ref() + .map(|m| m.partition_info.len()) + .unwrap_or(1) + .max(1); + + step_progress.set_info(format!("{} rows", total_rows), InfoStyle::Normal); + step_progress.set_substatus("parsing"); + + let column_indices = get_column_indices( + &response.result_set_meta_data, + &[ + "inputs", + "output", + "rating", + "feedback", + "device_id", + "time", + ], + ); + + all_examples.extend(rated_examples_from_response(&response, &column_indices)?); + + if num_partitions > 1 { + let statement_handle = response + .statement_handle + .as_ref() + .context("response has multiple partitions but no statementHandle")?; + + for partition in 1..num_partitions { + step_progress.set_substatus(format!( + "fetching partition {}/{}", + partition + 1, + num_partitions + )); + + let partition_response = fetch_partition( + http_client.clone(), + &base_url, + &token, + statement_handle, + partition, + ) + .await?; + + all_examples.extend(rated_examples_from_response( + &partition_response, + &column_indices, + )?); + } + } + + step_progress.set_substatus("done"); + } + + Ok(all_examples) +} + +fn rated_examples_from_response<'a>( + response: &'a SnowflakeStatementResponse, + column_indices: &'a std::collections::HashMap, +) -> Result + 'a> { + if let Some(code) = &response.code { + if code != SNOWFLAKE_SUCCESS_CODE { + anyhow::bail!( + "snowflake sql api returned error code={code} message={}", + response.message.as_deref().unwrap_or("") + ); + } + } + + let iter = response + .data + .iter() + .enumerate() + .filter_map(move |(row_index, data_row)| { + let get_string = |name: &str| -> Option { + let index = column_indices.get(name).copied()?; + match data_row.get(index)? { + JsonValue::String(s) => Some(s.clone()), + JsonValue::Null => None, + other => Some(other.to_string()), + } + }; + + let get_json = |name: &str| -> Option { + let index = column_indices.get(name).copied()?; + let value = data_row.get(index)?; + if value.is_null() { + return None; + } + match value { + JsonValue::String(s) => serde_json::from_str(s).ok(), + other => Some(other.clone()), + } + }; + + let inputs_json = get_json("inputs"); + let inputs: Option = match &inputs_json { + Some(v) => match serde_json::from_value(v.clone()) { + Ok(parsed) => Some(parsed), + Err(e) => { + log::warn!( + "skipping row {row_index}: failed to parse inputs - {e}", + ); + return None; + } + }, + None => None, + }; + let output = get_string("output"); + let rating = get_string("rating"); + let feedback = get_string("feedback").unwrap_or_default(); + let device_id = get_string("device_id"); + let time = get_string("time"); + + match (inputs, output.clone(), rating.clone(), device_id.clone(), time.clone()) { + (Some(inputs), Some(output), Some(rating), Some(device_id), Some(time)) => { + Some(build_rated_example( + device_id, + time, + inputs, + output, + rating, + feedback, + )) + } + _ => { + log::warn!( + "skipping row {row_index}: missing fields - inputs={:?} output={:?} rating={:?} device_id={:?} time={:?}", + inputs_json.is_some(), + output.is_some(), + rating.is_some(), + device_id.is_some(), + time.is_some(), + ); + None + } + } + }); + + Ok(iter) +} + +fn build_rated_example( + device_id: String, + time: String, + input: ZetaPromptInput, + output: String, + rating: String, + feedback: String, +) -> Example { + let parsed_rating = if rating == "Positive" { + EditPredictionRating::Positive + } else { + EditPredictionRating::Negative + }; + let is_positive = parsed_rating == EditPredictionRating::Positive; + let request_id = format!("rated-{}-{}", device_id, time); + + let tags = if is_positive { + vec!["rated:positive".to_string()] + } else { + vec!["rated:negative".to_string()] + }; + + let mut example = build_example_from_snowflake(request_id, device_id, time, input, tags, None); + + example.spec.rating = Some(parsed_rating); + + if !feedback.is_empty() { + example + .spec + .human_feedback + .push(edit_prediction::example_spec::HumanFeedback { message: feedback }); + } + + if is_positive { + example.spec.expected_patches = vec![output]; + } else { + example.spec.rejected_patch = Some(output); + } + + example +} + fn requested_examples_from_response<'a>( response: &'a SnowflakeStatementResponse, column_indices: &'a std::collections::HashMap, @@ -759,9 +1070,10 @@ fn requested_examples_from_response<'a>( Ok(iter) } -fn rejected_examples_from_response( - response: &SnowflakeStatementResponse, -) -> Result + '_> { +fn rejected_examples_from_response<'a>( + response: &'a SnowflakeStatementResponse, + column_indices: &'a std::collections::HashMap, +) -> Result + 'a> { if let Some(code) = &response.code { if code != SNOWFLAKE_SUCCESS_CODE { anyhow::bail!( @@ -771,20 +1083,6 @@ fn rejected_examples_from_response( } } - let column_indices = get_column_indices( - &response.result_set_meta_data, - &[ - "request_id", - "device_id", - "time", - "input", - "prompt", - "output", - "was_shown", - "reason", - ], - ); - let iter = response .data .iter() @@ -981,6 +1279,8 @@ fn build_example_from_snowflake( rejection_reason, was_shown, }), + human_feedback: Vec::new(), + rating: None, }; Example { diff --git a/crates/edit_prediction_cli/src/split_commit.rs b/crates/edit_prediction_cli/src/split_commit.rs index 8d00fd2f0846bfaf20c84a71a0f00a3e77cbc54e..b54da11c2dfa77bb531da06c581bf02a5fd615d0 100644 --- a/crates/edit_prediction_cli/src/split_commit.rs +++ b/crates/edit_prediction_cli/src/split_commit.rs @@ -372,6 +372,8 @@ pub fn generate_evaluation_example_from_ordered_commit( rejected_patch: None, captured_prompt_input: None, telemetry: None, + human_feedback: Vec::new(), + rating: None, }) } @@ -1404,6 +1406,8 @@ Date: Mon Jan 1 00:00:00 2024 rejected_patch: None, captured_prompt_input: None, telemetry: None, + human_feedback: Vec::new(), + rating: None, }; let json = serde_json::to_string(&case).unwrap(); diff --git a/crates/edit_prediction_cli/src/synthesize.rs b/crates/edit_prediction_cli/src/synthesize.rs index 603ee8de65663256246a5413dab6906c641d3de6..31bae0af63858b4aaf45670bb543d8cf810bb1a1 100644 --- a/crates/edit_prediction_cli/src/synthesize.rs +++ b/crates/edit_prediction_cli/src/synthesize.rs @@ -794,6 +794,8 @@ async fn build_example( rejected_patch: None, captured_prompt_input: None, telemetry: None, + human_feedback: Vec::new(), + rating: None, }; spec.set_cursor_excerpt(&excerpt, cursor_offset, comment_prefix); diff --git a/crates/telemetry_events/src/telemetry_events.rs b/crates/telemetry_events/src/telemetry_events.rs index 83ec2c064499804463ada5eabfcbd48d77369bb6..e98a2c025fd26ecbc6414831d0cf479bd33b9ff9 100644 --- a/crates/telemetry_events/src/telemetry_events.rs +++ b/crates/telemetry_events/src/telemetry_events.rs @@ -101,7 +101,7 @@ pub struct FlexibleEvent { pub event_properties: HashMap, } -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] pub enum EditPredictionRating { Positive, Negative,