Cargo.lock 🔗
@@ -5399,6 +5399,7 @@ dependencies = [
"smol",
"sqlez",
"sqlez_macros",
+ "telemetry_events",
"tempfile",
"terminal_view",
"toml 0.8.23",
ep_cli: `rated-after:` query (#47906)
Ben Kunkle created
Closes #ISSUE
Release Notes:
- N/A *or* Added/Fixed/Improved ...Change summary
Cargo.lock | 1
crates/edit_prediction/src/capture_example.rs | 4
crates/edit_prediction/src/example_spec.rs | 16
crates/edit_prediction_cli/Cargo.toml | 1
crates/edit_prediction_cli/src/main.rs | 134 ++++--
crates/edit_prediction_cli/src/pull_examples.rs | 370 +++++++++++++++++-
crates/edit_prediction_cli/src/split_commit.rs | 4
crates/edit_prediction_cli/src/synthesize.rs | 2
crates/telemetry_events/src/telemetry_events.rs | 2
9 files changed, 445 insertions(+), 89 deletions(-)
Detailed changes
crates/edit_prediction/src/capture_example.rs 🔗
@@ -175,6 +175,8 @@ pub fn capture_example(
rejected_patch,
captured_prompt_input: prompt_input,
telemetry: None,
+ human_feedback: Vec::new(),
+ rating: None,
};
spec.set_cursor_excerpt(
&cursor_excerpt,
@@ -606,6 +608,8 @@ mod tests {
),
captured_prompt_input: example.captured_prompt_input.clone(),
telemetry: None,
+ human_feedback: Vec::new(),
+ rating: None,
}
);
crates/edit_prediction/src/example_spec.rs 🔗
@@ -1,6 +1,7 @@
use anyhow::{Context as _, Result};
use serde::{Deserialize, Serialize};
use std::{borrow::Cow, fmt::Write as _, mem, ops::Range, path::Path, sync::Arc};
+use telemetry_events::EditPredictionRating;
pub const CURSOR_POSITION_MARKER: &str = "[CURSOR_POSITION]";
pub const INLINE_CURSOR_MARKER: &str = "<|user_cursor|>";
@@ -32,6 +33,15 @@ pub struct ExampleSpec {
pub captured_prompt_input: Option<CapturedPromptInput>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub telemetry: Option<TelemetrySource>,
+ #[serde(default, skip_serializing_if = "Vec::is_empty")]
+ pub human_feedback: Vec<HumanFeedback>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub rating: Option<EditPredictionRating>,
+}
+
+#[derive(Clone, Debug, PartialEq, Hash, Serialize, Deserialize)]
+pub struct HumanFeedback {
+ pub message: String,
}
/// Metadata for examples sourced from production telemetry (rejected predictions).
@@ -252,6 +262,8 @@ impl ExampleSpec {
rejected_patch: None,
captured_prompt_input: None,
telemetry: None,
+ human_feedback: Vec::new(),
+ rating: None,
};
if let Some(rest) = input.strip_prefix("+++\n")
@@ -500,6 +512,8 @@ mod tests {
rejected_patch: None,
captured_prompt_input: None,
telemetry: None,
+ human_feedback: Vec::new(),
+ rating: None,
};
// Cursor before `42`
@@ -635,6 +649,8 @@ mod tests {
rejected_patch: None,
captured_prompt_input: None,
telemetry: None,
+ human_feedback: Vec::new(),
+ rating: None,
};
// Cursor before `42` using inline marker
crates/edit_prediction_cli/Cargo.toml 🔗
@@ -55,6 +55,7 @@ terminal_view.workspace = true
util.workspace = true
watch.workspace = true
edit_prediction = { workspace = true, features = ["cli-support"] }
+telemetry_events.workspace = true
wasmtime.workspace = true
zeta_prompt.workspace = true
rand.workspace = true
crates/edit_prediction_cli/src/main.rs 🔗
@@ -118,6 +118,19 @@ Inputs can be file paths or special specifiers:
Fetch rejected edit predictions from Snowflake after the given RFC3339 timestamp.
These are predictions that were shown to users but rejected (useful for DPO training).
+ rated-after:{timestamp}
+ Fetch user-rated edit predictions from Snowflake after the given RFC3339 timestamp.
+ These are predictions that users explicitly rated as positive or negative via the
+ rate completions modal. Only zeta2 predictions are included.
+ - Positive ratings: output becomes expected_patches
+ - Negative ratings: output becomes rejected_patch
+
+ rated-positive-after:{timestamp}
+ Same as rated-after, but only fetches positively rated predictions.
+
+ rated-negative-after:{timestamp}
+ Same as rated-after, but only fetches negatively rated predictions.
+
Required environment variables to connect to Snowflake:
EP_SNOWFLAKE_API_KEY
EP_SNOWFLAKE_BASE_URL
@@ -136,6 +149,15 @@ Examples:
# Read rejected predictions for DPO training
ep read rejected-after:2025-01-01T00:00:00Z -o rejected.jsonl
+ # Read user-rated predictions
+ ep read rated-after:2025-01-01T00:00:00Z -o rated.jsonl
+
+ # Read only positively rated predictions
+ ep read rated-positive-after:2025-01-01T00:00:00Z -o positive.jsonl
+
+ # Read only negatively rated predictions
+ ep read rated-negative-after:2025-01-01T00:00:00Z -o negative.jsonl
+
# Mix multiple input sources
ep predict examples.jsonl captured-after:2025-01-01T00:00:00Z
"#;
@@ -436,6 +458,8 @@ async fn load_examples(
let mut captured_after_timestamps = Vec::new();
let mut rejected_after_timestamps = Vec::new();
let mut requested_after_timestamps = Vec::new();
+ let mut rated_after_inputs: Vec<(String, Option<telemetry_events::EditPredictionRating>)> =
+ Vec::new();
let mut file_inputs = Vec::new();
for input in &args.inputs {
@@ -450,6 +474,10 @@ async fn load_examples(
pull_examples::parse_requested_after_input(input_string.as_ref())
{
requested_after_timestamps.push(timestamp.to_string());
+ } else if let Some((timestamp, rating_filter)) =
+ pull_examples::parse_rated_after_input(input_string.as_ref())
+ {
+ rated_after_inputs.push((timestamp.to_string(), rating_filter));
} else {
file_inputs.push(input.clone());
}
@@ -499,14 +527,27 @@ async fn load_examples(
requested_after_timestamps.sort();
let mut requested_examples = pull_examples::fetch_requested_examples_after(
- http_client,
+ http_client.clone(),
&requested_after_timestamps,
max_rows_per_timestamp,
- background_executor,
+ background_executor.clone(),
)
.await?;
examples.append(&mut requested_examples);
}
+
+ if !rated_after_inputs.is_empty() {
+ rated_after_inputs.sort();
+
+ let mut rated_examples = pull_examples::fetch_rated_examples_after(
+ http_client,
+ &rated_after_inputs,
+ max_rows_per_timestamp,
+ background_executor,
+ )
+ .await?;
+ examples.append(&mut rated_examples);
+ }
}
crate::example::sort_examples_by_repo_and_rev(&mut examples);
@@ -785,63 +826,47 @@ fn main() {
let failfast_on_single_example = examples.len() == 1;
// For --markdown mode, create the output directory if it doesn't exist
- let markdown_output_dir = if args.markdown {
+ if args.markdown {
let dir = output.as_ref().expect("--markdown requires -o");
if !dir.exists() {
std::fs::create_dir_all(dir)
.expect("Failed to create markdown output directory");
}
- Some(dir.clone())
- } else {
- None
- };
-
- // For --in-place, write to a temp file and rename at the end to avoid data loss on interruption
- let in_place_temp_path = if args.in_place {
- output.as_ref().map(|path| {
- let mut temp_path = path.clone();
- temp_path.set_extension("jsonl.tmp");
- temp_path
- })
- } else {
- None
- };
+ }
- let output_sender: Option<mpsc::UnboundedSender<String>> = if !args.markdown
- && (args.output.is_some() || !matches!(command, Command::Eval(_)))
+ // Set up JSONL output writer (not used in markdown mode)
+ let mut output_sender: Option<mpsc::UnboundedSender<String>> = None;
+ let mut in_place_temp_path: Option<PathBuf> = None;
+ if !args.markdown
+ && let Some(output_path) = output.as_ref()
{
- let write_path = in_place_temp_path.as_ref().or(output.as_ref());
- write_path.map(|path| {
- let file = if args.in_place {
- // For --in-place, write to temp file (truncate if exists)
- OpenOptions::new()
- .create(true)
- .write(true)
- .truncate(true)
- .open(path)
- .expect("Failed to open temp output file")
- } else {
- // For regular output, append to support resuming
- OpenOptions::new()
- .create(true)
- .append(true)
- .open(path)
- .expect("Failed to open output file")
- };
- let mut writer = BufWriter::new(file);
- let (sender, mut receiver) = mpsc::unbounded::<String>();
- cx.background_spawn(async move {
- while let Some(line) = receiver.next().await {
- writeln!(writer, "{}", line).expect("Failed to write example");
- writer.flush().expect("Failed to flush output");
- }
- })
- .detach();
- sender
+ let write_path = if args.in_place {
+ let temp = output_path.with_extension("jsonl.tmp");
+ in_place_temp_path = Some(temp.clone());
+ temp
+ } else {
+ output_path.clone()
+ };
+
+ let file = OpenOptions::new()
+ .create(true)
+ .write(true)
+ .truncate(args.in_place)
+ .append(!args.in_place)
+ .open(&write_path)
+ .expect("Failed to open output file");
+
+ let mut writer = BufWriter::new(file);
+ let (sender, mut receiver) = mpsc::unbounded::<String>();
+ cx.background_spawn(async move {
+ while let Some(line) = receiver.next().await {
+ writeln!(writer, "{}", line).expect("Failed to write example");
+ writer.flush().expect("Failed to flush output");
+ }
})
- } else {
- None
- };
+ .detach();
+ output_sender = Some(sender);
+ }
let grouped_examples = Mutex::new(group_examples_by_repo(examples));
let finished_examples = Mutex::new(Vec::new());
@@ -958,7 +983,9 @@ fn main() {
let should_write = !failed || args.failed == FailedHandling::Keep;
if should_write {
- if let Some(ref markdown_dir) = markdown_output_dir {
+ if args.markdown {
+ let markdown_dir =
+ output.as_ref().expect("--markdown requires -o");
let filename = format!("{}.md", example.spec.filename());
let path = markdown_dir.join(&filename);
let markdown = example.spec.to_markdown();
@@ -1044,7 +1071,8 @@ fn main() {
};
// For --in-place, atomically rename temp file to original
- if let (Some(temp_path), Some(final_path)) = (&in_place_temp_path, &output) {
+ if let Some(temp_path) = &in_place_temp_path {
+ let final_path = output.as_ref().expect("in_place_temp_path requires output");
std::fs::rename(temp_path, final_path)
.expect("Failed to rename temp file to final output");
}
crates/edit_prediction_cli/src/pull_examples.rs 🔗
@@ -8,6 +8,7 @@ use serde_json::{Value as JsonValue, json};
use std::io::Read;
use std::sync::Arc;
use std::time::Duration;
+use telemetry_events::EditPredictionRating;
use zeta_prompt::ZetaPromptInput;
@@ -24,6 +25,7 @@ const SNOWFLAKE_ASYNC_IN_PROGRESS_CODE: &str = "333334";
const EDIT_PREDICTION_EXAMPLE_CAPTURED_EVENT: &str = "Edit Prediction Example Captured";
const PREDICTIVE_EDIT_REQUESTED_EVENT: &str = "Predictive Edit Requested";
const PREDICTIVE_EDIT_REJECTED_EVENT: &str = "Predictive Edit Rejected";
+const EDIT_PREDICTION_RATED_EVENT: &str = "Edit Prediction Rated";
const DEFAULT_STATEMENT_TIMEOUT_SECONDS: u64 = 120;
const POLL_INTERVAL: Duration = Duration::from_secs(2);
@@ -44,6 +46,21 @@ pub fn parse_requested_after_input(input: &str) -> Option<&str> {
input.strip_prefix("requested-after:")
}
+/// Parse an input token of the form `rated-after:{timestamp}`, `rated-positive-after:{timestamp}`,
+/// or `rated-negative-after:{timestamp}`.
+/// Returns `(timestamp, Option<EditPredictionRating>)` where `None` means all ratings.
+pub fn parse_rated_after_input(input: &str) -> Option<(&str, Option<EditPredictionRating>)> {
+ if let Some(timestamp) = input.strip_prefix("rated-positive-after:") {
+ Some((timestamp, Some(EditPredictionRating::Positive)))
+ } else if let Some(timestamp) = input.strip_prefix("rated-negative-after:") {
+ Some((timestamp, Some(EditPredictionRating::Negative)))
+ } else if let Some(timestamp) = input.strip_prefix("rated-after:") {
+ Some((timestamp, None))
+ } else {
+ None
+ }
+}
+
pub async fn fetch_captured_examples_after(
http_client: Arc<dyn HttpClient>,
after_timestamps: &[String],
@@ -120,7 +137,21 @@ pub async fn fetch_captured_examples_after(
step_progress.set_info(format!("{} rows", total_rows), InfoStyle::Normal);
step_progress.set_substatus("parsing");
- all_examples.extend(examples_from_response(&response)?);
+ let example_index = response
+ .result_set_meta_data
+ .as_ref()
+ .and_then(|m| {
+ m.row_type.iter().enumerate().find_map(|(index, col)| {
+ if col.name.eq_ignore_ascii_case("example") {
+ Some(index)
+ } else {
+ None
+ }
+ })
+ })
+ .unwrap_or(0);
+
+ all_examples.extend(examples_from_response(&response, example_index)?);
if num_partitions > 1 {
let statement_handle = response
@@ -144,7 +175,7 @@ pub async fn fetch_captured_examples_after(
)
.await?;
- all_examples.extend(examples_from_response(&partition_response)?);
+ all_examples.extend(examples_from_response(&partition_response, example_index)?);
}
}
@@ -192,6 +223,7 @@ struct SnowflakeColumnMeta {
fn examples_from_response(
response: &SnowflakeStatementResponse,
+ example_index: usize,
) -> Result<impl Iterator<Item = Example> + '_> {
if let Some(code) = &response.code {
if code != SNOWFLAKE_SUCCESS_CODE {
@@ -202,20 +234,6 @@ fn examples_from_response(
}
}
- let example_index = response
- .result_set_meta_data
- .as_ref()
- .and_then(|m| {
- m.row_type.iter().enumerate().find_map(|(index, col)| {
- if col.name.eq_ignore_ascii_case("example") {
- Some(index)
- } else {
- None
- }
- })
- })
- .unwrap_or(0);
-
let iter = response.data.iter().enumerate().filter_map(move |(row_index, data_row)| {
let Some(example_value) = data_row.get(example_index) else {
return None;
@@ -527,7 +545,21 @@ pub async fn fetch_rejected_examples_after(
step_progress.set_info(format!("{} rows", total_rows), InfoStyle::Normal);
step_progress.set_substatus("parsing");
- all_examples.extend(rejected_examples_from_response(&response)?);
+ let column_indices = get_column_indices(
+ &response.result_set_meta_data,
+ &[
+ "request_id",
+ "device_id",
+ "time",
+ "input",
+ "prompt",
+ "output",
+ "was_shown",
+ "reason",
+ ],
+ );
+
+ all_examples.extend(rejected_examples_from_response(&response, &column_indices)?);
if num_partitions > 1 {
let statement_handle = response
@@ -551,7 +583,10 @@ pub async fn fetch_rejected_examples_after(
)
.await?;
- all_examples.extend(rejected_examples_from_response(&partition_response)?);
+ all_examples.extend(rejected_examples_from_response(
+ &partition_response,
+ &column_indices,
+ )?);
}
}
@@ -686,6 +721,282 @@ pub async fn fetch_requested_examples_after(
Ok(all_examples)
}
+pub async fn fetch_rated_examples_after(
+ http_client: Arc<dyn HttpClient>,
+ inputs: &[(String, Option<EditPredictionRating>)],
+ max_rows_per_timestamp: usize,
+ background_executor: BackgroundExecutor,
+) -> Result<Vec<Example>> {
+ if inputs.is_empty() {
+ return Ok(Vec::new());
+ }
+
+ let progress = Progress::global();
+
+ let token = std::env::var("EP_SNOWFLAKE_API_KEY")
+ .context("missing required environment variable EP_SNOWFLAKE_API_KEY")?;
+ let base_url = std::env::var("EP_SNOWFLAKE_BASE_URL").context(
+ "missing required environment variable EP_SNOWFLAKE_BASE_URL (e.g. https://<account>.snowflakecomputing.com)",
+ )?;
+ let role = std::env::var("EP_SNOWFLAKE_ROLE").ok();
+
+ let mut all_examples = Vec::new();
+
+ for (after_date, rating_filter) in inputs.iter() {
+ let filter_label = match rating_filter {
+ None => "",
+ Some(EditPredictionRating::Positive) => ":positive",
+ Some(EditPredictionRating::Negative) => ":negative",
+ };
+ let step_progress_name = format!("rated{filter_label}>{after_date}");
+ let step_progress = progress.start(Step::PullExamples, &step_progress_name);
+ step_progress.set_substatus("querying");
+
+ let rating_value = rating_filter.as_ref().map(|r| match r {
+ EditPredictionRating::Positive => "Positive",
+ EditPredictionRating::Negative => "Negative",
+ });
+
+ let statement = indoc! {r#"
+ SELECT
+ event_properties:inputs AS inputs,
+ event_properties:output::string AS output,
+ event_properties:rating::string AS rating,
+ event_properties:feedback::string AS feedback,
+ device_id::string AS device_id,
+ time::string AS time
+ FROM events
+ WHERE event_type = ?
+ AND (? IS NULL OR event_properties:rating::string = ?)
+ AND time > TRY_TO_TIMESTAMP_NTZ(?)
+ AND event_properties:inputs IS NOT NULL
+ AND event_properties:inputs:cursor_excerpt IS NOT NULL
+ AND event_properties:output IS NOT NULL
+ ORDER BY time ASC
+ LIMIT ?
+ "#};
+
+ let bindings = json!({
+ "1": { "type": "TEXT", "value": EDIT_PREDICTION_RATED_EVENT },
+ "2": { "type": "TEXT", "value": rating_value },
+ "3": { "type": "TEXT", "value": rating_value },
+ "4": { "type": "TEXT", "value": after_date },
+ "5": { "type": "FIXED", "value": max_rows_per_timestamp.to_string() }
+ });
+
+ let request = json!({
+ "statement": statement,
+ "timeout": DEFAULT_STATEMENT_TIMEOUT_SECONDS,
+ "database": "EVENTS",
+ "schema": "PUBLIC",
+ "warehouse": "DBT",
+ "role": role,
+ "bindings": bindings
+ });
+
+ let response = run_sql_with_polling(
+ http_client.clone(),
+ &base_url,
+ &token,
+ &request,
+ &step_progress,
+ background_executor.clone(),
+ )
+ .await?;
+
+ let total_rows = response
+ .result_set_meta_data
+ .as_ref()
+ .and_then(|m| m.num_rows)
+ .unwrap_or(response.data.len() as i64);
+
+ let num_partitions = response
+ .result_set_meta_data
+ .as_ref()
+ .map(|m| m.partition_info.len())
+ .unwrap_or(1)
+ .max(1);
+
+ step_progress.set_info(format!("{} rows", total_rows), InfoStyle::Normal);
+ step_progress.set_substatus("parsing");
+
+ let column_indices = get_column_indices(
+ &response.result_set_meta_data,
+ &[
+ "inputs",
+ "output",
+ "rating",
+ "feedback",
+ "device_id",
+ "time",
+ ],
+ );
+
+ all_examples.extend(rated_examples_from_response(&response, &column_indices)?);
+
+ if num_partitions > 1 {
+ let statement_handle = response
+ .statement_handle
+ .as_ref()
+ .context("response has multiple partitions but no statementHandle")?;
+
+ for partition in 1..num_partitions {
+ step_progress.set_substatus(format!(
+ "fetching partition {}/{}",
+ partition + 1,
+ num_partitions
+ ));
+
+ let partition_response = fetch_partition(
+ http_client.clone(),
+ &base_url,
+ &token,
+ statement_handle,
+ partition,
+ )
+ .await?;
+
+ all_examples.extend(rated_examples_from_response(
+ &partition_response,
+ &column_indices,
+ )?);
+ }
+ }
+
+ step_progress.set_substatus("done");
+ }
+
+ Ok(all_examples)
+}
+
+fn rated_examples_from_response<'a>(
+ response: &'a SnowflakeStatementResponse,
+ column_indices: &'a std::collections::HashMap<String, usize>,
+) -> Result<impl Iterator<Item = Example> + 'a> {
+ if let Some(code) = &response.code {
+ if code != SNOWFLAKE_SUCCESS_CODE {
+ anyhow::bail!(
+ "snowflake sql api returned error code={code} message={}",
+ response.message.as_deref().unwrap_or("<no message>")
+ );
+ }
+ }
+
+ let iter = response
+ .data
+ .iter()
+ .enumerate()
+ .filter_map(move |(row_index, data_row)| {
+ let get_string = |name: &str| -> Option<String> {
+ let index = column_indices.get(name).copied()?;
+ match data_row.get(index)? {
+ JsonValue::String(s) => Some(s.clone()),
+ JsonValue::Null => None,
+ other => Some(other.to_string()),
+ }
+ };
+
+ let get_json = |name: &str| -> Option<JsonValue> {
+ let index = column_indices.get(name).copied()?;
+ let value = data_row.get(index)?;
+ if value.is_null() {
+ return None;
+ }
+ match value {
+ JsonValue::String(s) => serde_json::from_str(s).ok(),
+ other => Some(other.clone()),
+ }
+ };
+
+ let inputs_json = get_json("inputs");
+ let inputs: Option<ZetaPromptInput> = match &inputs_json {
+ Some(v) => match serde_json::from_value(v.clone()) {
+ Ok(parsed) => Some(parsed),
+ Err(e) => {
+ log::warn!(
+ "skipping row {row_index}: failed to parse inputs - {e}",
+ );
+ return None;
+ }
+ },
+ None => None,
+ };
+ let output = get_string("output");
+ let rating = get_string("rating");
+ let feedback = get_string("feedback").unwrap_or_default();
+ let device_id = get_string("device_id");
+ let time = get_string("time");
+
+ match (inputs, output.clone(), rating.clone(), device_id.clone(), time.clone()) {
+ (Some(inputs), Some(output), Some(rating), Some(device_id), Some(time)) => {
+ Some(build_rated_example(
+ device_id,
+ time,
+ inputs,
+ output,
+ rating,
+ feedback,
+ ))
+ }
+ _ => {
+ log::warn!(
+ "skipping row {row_index}: missing fields - inputs={:?} output={:?} rating={:?} device_id={:?} time={:?}",
+ inputs_json.is_some(),
+ output.is_some(),
+ rating.is_some(),
+ device_id.is_some(),
+ time.is_some(),
+ );
+ None
+ }
+ }
+ });
+
+ Ok(iter)
+}
+
+fn build_rated_example(
+ device_id: String,
+ time: String,
+ input: ZetaPromptInput,
+ output: String,
+ rating: String,
+ feedback: String,
+) -> Example {
+ let parsed_rating = if rating == "Positive" {
+ EditPredictionRating::Positive
+ } else {
+ EditPredictionRating::Negative
+ };
+ let is_positive = parsed_rating == EditPredictionRating::Positive;
+ let request_id = format!("rated-{}-{}", device_id, time);
+
+ let tags = if is_positive {
+ vec!["rated:positive".to_string()]
+ } else {
+ vec!["rated:negative".to_string()]
+ };
+
+ let mut example = build_example_from_snowflake(request_id, device_id, time, input, tags, None);
+
+ example.spec.rating = Some(parsed_rating);
+
+ if !feedback.is_empty() {
+ example
+ .spec
+ .human_feedback
+ .push(edit_prediction::example_spec::HumanFeedback { message: feedback });
+ }
+
+ if is_positive {
+ example.spec.expected_patches = vec![output];
+ } else {
+ example.spec.rejected_patch = Some(output);
+ }
+
+ example
+}
+
fn requested_examples_from_response<'a>(
response: &'a SnowflakeStatementResponse,
column_indices: &'a std::collections::HashMap<String, usize>,
@@ -759,9 +1070,10 @@ fn requested_examples_from_response<'a>(
Ok(iter)
}
-fn rejected_examples_from_response(
- response: &SnowflakeStatementResponse,
-) -> Result<impl Iterator<Item = Example> + '_> {
+fn rejected_examples_from_response<'a>(
+ response: &'a SnowflakeStatementResponse,
+ column_indices: &'a std::collections::HashMap<String, usize>,
+) -> Result<impl Iterator<Item = Example> + 'a> {
if let Some(code) = &response.code {
if code != SNOWFLAKE_SUCCESS_CODE {
anyhow::bail!(
@@ -771,20 +1083,6 @@ fn rejected_examples_from_response(
}
}
- let column_indices = get_column_indices(
- &response.result_set_meta_data,
- &[
- "request_id",
- "device_id",
- "time",
- "input",
- "prompt",
- "output",
- "was_shown",
- "reason",
- ],
- );
-
let iter = response
.data
.iter()
@@ -981,6 +1279,8 @@ fn build_example_from_snowflake(
rejection_reason,
was_shown,
}),
+ human_feedback: Vec::new(),
+ rating: None,
};
Example {
crates/edit_prediction_cli/src/split_commit.rs 🔗
@@ -372,6 +372,8 @@ pub fn generate_evaluation_example_from_ordered_commit(
rejected_patch: None,
captured_prompt_input: None,
telemetry: None,
+ human_feedback: Vec::new(),
+ rating: None,
})
}
@@ -1404,6 +1406,8 @@ Date: Mon Jan 1 00:00:00 2024
rejected_patch: None,
captured_prompt_input: None,
telemetry: None,
+ human_feedback: Vec::new(),
+ rating: None,
};
let json = serde_json::to_string(&case).unwrap();
crates/edit_prediction_cli/src/synthesize.rs 🔗
@@ -794,6 +794,8 @@ async fn build_example(
rejected_patch: None,
captured_prompt_input: None,
telemetry: None,
+ human_feedback: Vec::new(),
+ rating: None,
};
spec.set_cursor_excerpt(&excerpt, cursor_offset, comment_prefix);
crates/telemetry_events/src/telemetry_events.rs 🔗
@@ -101,7 +101,7 @@ pub struct FlexibleEvent {
pub event_properties: HashMap<String, serde_json::Value>,
}
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
pub enum EditPredictionRating {
Positive,
Negative,