reliability.rs

  1use crate::stdout_is_a_pty;
  2use anyhow::{Context as _, Result};
  3use backtrace::{self, Backtrace};
  4use chrono::Utc;
  5use client::{
  6    TelemetrySettings,
  7    telemetry::{self, MINIDUMP_ENDPOINT},
  8};
  9use db::kvp::KEY_VALUE_STORE;
 10use futures::AsyncReadExt;
 11use gpui::{App, AppContext as _, SemanticVersion};
 12use http_client::{self, HttpClient, HttpClientWithUrl, HttpRequestExt, Method};
 13use paths::{crashes_dir, crashes_retired_dir};
 14use project::Project;
 15use proto::{CrashReport, GetCrashFilesResponse};
 16use release_channel::{AppCommitSha, RELEASE_CHANNEL, ReleaseChannel};
 17use reqwest::multipart::{Form, Part};
 18use settings::Settings;
 19use smol::stream::StreamExt;
 20use std::{
 21    env,
 22    ffi::{OsStr, c_void},
 23    fs,
 24    io::Write,
 25    panic,
 26    sync::{
 27        Arc,
 28        atomic::{AtomicU32, Ordering},
 29    },
 30    thread,
 31};
 32use telemetry_events::{LocationData, Panic, PanicRequest};
 33use url::Url;
 34use util::ResultExt;
 35
 36static PANIC_COUNT: AtomicU32 = AtomicU32::new(0);
 37
 38pub fn init_panic_hook(
 39    app_version: SemanticVersion,
 40    app_commit_sha: Option<AppCommitSha>,
 41    system_id: Option<String>,
 42    installation_id: Option<String>,
 43    session_id: String,
 44) {
 45    let is_pty = stdout_is_a_pty();
 46
 47    panic::set_hook(Box::new(move |info| {
 48        let prior_panic_count = PANIC_COUNT.fetch_add(1, Ordering::SeqCst);
 49        if prior_panic_count > 0 {
 50            // Give the panic-ing thread time to write the panic file
 51            loop {
 52                thread::yield_now();
 53            }
 54        }
 55
 56        let payload = info
 57            .payload()
 58            .downcast_ref::<&str>()
 59            .map(|s| s.to_string())
 60            .or_else(|| info.payload().downcast_ref::<String>().cloned())
 61            .unwrap_or_else(|| "Box<Any>".to_string());
 62
 63        if *release_channel::RELEASE_CHANNEL != ReleaseChannel::Dev
 64            || env::var("ZED_GENERATE_MINIDUMPS").is_ok()
 65        {
 66            crashes::handle_panic(payload.clone(), info.location());
 67        }
 68
 69        let thread = thread::current();
 70        let thread_name = thread.name().unwrap_or("<unnamed>");
 71
 72        if *release_channel::RELEASE_CHANNEL == ReleaseChannel::Dev {
 73            let location = info.location().unwrap();
 74            let backtrace = Backtrace::new();
 75            eprintln!(
 76                "Thread {:?} panicked with {:?} at {}:{}:{}\n{}{:?}",
 77                thread_name,
 78                payload,
 79                location.file(),
 80                location.line(),
 81                location.column(),
 82                match app_commit_sha.as_ref() {
 83                    Some(commit_sha) => format!(
 84                        "https://github.com/zed-industries/zed/blob/{}/{}#L{} \
 85                        (may not be uploaded, line may be incorrect if files modified)\n",
 86                        commit_sha.full(),
 87                        location.file(),
 88                        location.line()
 89                    ),
 90                    None => "".to_string(),
 91                },
 92                backtrace,
 93            );
 94            if MINIDUMP_ENDPOINT.is_none() {
 95                std::process::exit(-1);
 96            }
 97        }
 98        let main_module_base_address = get_main_module_base_address();
 99
100        let backtrace = Backtrace::new();
101        let mut symbols = backtrace
102            .frames()
103            .iter()
104            .flat_map(|frame| {
105                let base = frame
106                    .module_base_address()
107                    .unwrap_or(main_module_base_address);
108                frame.symbols().iter().map(move |symbol| {
109                    format!(
110                        "{}+{}",
111                        symbol
112                            .name()
113                            .as_ref()
114                            .map_or("<unknown>".to_owned(), <_>::to_string),
115                        (frame.ip() as isize).saturating_sub(base as isize)
116                    )
117                })
118            })
119            .collect::<Vec<_>>();
120
121        // Strip out leading stack frames for rust panic-handling.
122        if let Some(ix) = symbols
123            .iter()
124            .position(|name| name == "rust_begin_unwind" || name == "_rust_begin_unwind")
125        {
126            symbols.drain(0..=ix);
127        }
128
129        let panic_data = telemetry_events::Panic {
130            thread: thread_name.into(),
131            payload,
132            location_data: info.location().map(|location| LocationData {
133                file: location.file().into(),
134                line: location.line(),
135            }),
136            app_version: app_version.to_string(),
137            app_commit_sha: app_commit_sha.as_ref().map(|sha| sha.full()),
138            release_channel: RELEASE_CHANNEL.dev_name().into(),
139            target: env!("TARGET").to_owned().into(),
140            os_name: telemetry::os_name(),
141            os_version: Some(telemetry::os_version()),
142            architecture: env::consts::ARCH.into(),
143            panicked_on: Utc::now().timestamp_millis(),
144            backtrace: symbols,
145            system_id: system_id.clone(),
146            installation_id: installation_id.clone(),
147            session_id: session_id.clone(),
148        };
149
150        if let Some(panic_data_json) = serde_json::to_string_pretty(&panic_data).log_err() {
151            log::error!("{}", panic_data_json);
152        }
153        zlog::flush();
154
155        if (!is_pty || MINIDUMP_ENDPOINT.is_some())
156            && let Some(panic_data_json) = serde_json::to_string(&panic_data).log_err()
157        {
158            let timestamp = chrono::Utc::now().format("%Y_%m_%d %H_%M_%S").to_string();
159            let panic_file_path = paths::logs_dir().join(format!("zed-{timestamp}.panic"));
160            let panic_file = fs::OpenOptions::new()
161                .write(true)
162                .create_new(true)
163                .open(&panic_file_path)
164                .log_err();
165            if let Some(mut panic_file) = panic_file {
166                writeln!(&mut panic_file, "{panic_data_json}").log_err();
167                panic_file.flush().log_err();
168            }
169        }
170
171        std::process::abort();
172    }));
173}
174
175#[cfg(not(target_os = "windows"))]
176fn get_main_module_base_address() -> *mut c_void {
177    let mut dl_info = libc::Dl_info {
178        dli_fname: std::ptr::null(),
179        dli_fbase: std::ptr::null_mut(),
180        dli_sname: std::ptr::null(),
181        dli_saddr: std::ptr::null_mut(),
182    };
183    unsafe {
184        libc::dladdr(get_main_module_base_address as _, &mut dl_info);
185    }
186    dl_info.dli_fbase
187}
188
189#[cfg(target_os = "windows")]
190fn get_main_module_base_address() -> *mut c_void {
191    std::ptr::null_mut()
192}
193
194pub fn init(
195    http_client: Arc<HttpClientWithUrl>,
196    system_id: Option<String>,
197    installation_id: Option<String>,
198    session_id: String,
199    cx: &mut App,
200) {
201    #[cfg(target_os = "macos")]
202    monitor_main_thread_hangs(http_client.clone(), installation_id.clone(), cx);
203
204    let Some(panic_report_url) = http_client
205        .build_zed_api_url("/telemetry/panics", &[])
206        .log_err()
207    else {
208        return;
209    };
210
211    upload_panics_and_crashes(
212        http_client.clone(),
213        panic_report_url.clone(),
214        installation_id.clone(),
215        cx,
216    );
217
218    cx.observe_new(move |project: &mut Project, _, cx| {
219        let http_client = http_client.clone();
220        let panic_report_url = panic_report_url.clone();
221        let session_id = session_id.clone();
222        let installation_id = installation_id.clone();
223        let system_id = system_id.clone();
224
225        let Some(remote_client) = project.remote_client() else {
226            return;
227        };
228        remote_client.update(cx, |client, cx| {
229            if !TelemetrySettings::get_global(cx).diagnostics {
230                return;
231            }
232            let request = client.proto_client().request(proto::GetCrashFiles {});
233            cx.background_spawn(async move {
234                let GetCrashFilesResponse {
235                    legacy_panics,
236                    crashes,
237                } = request.await?;
238
239                for panic in legacy_panics {
240                    if let Some(mut panic) = serde_json::from_str::<Panic>(&panic).log_err() {
241                        panic.session_id = session_id.clone();
242                        panic.system_id = system_id.clone();
243                        panic.installation_id = installation_id.clone();
244                        upload_panic(&http_client, &panic_report_url, panic, &mut None).await?;
245                    }
246                }
247
248                let Some(endpoint) = MINIDUMP_ENDPOINT.as_ref() else {
249                    return Ok(());
250                };
251                for CrashReport {
252                    metadata,
253                    minidump_contents,
254                } in crashes
255                {
256                    if let Some(metadata) = serde_json::from_str(&metadata).log_err() {
257                        upload_minidump(
258                            http_client.clone(),
259                            endpoint,
260                            minidump_contents,
261                            &metadata,
262                            installation_id.clone(),
263                        )
264                        .await
265                        .log_err();
266                    }
267                }
268
269                anyhow::Ok(())
270            })
271            .detach_and_log_err(cx);
272        })
273    })
274    .detach();
275}
276
277#[cfg(target_os = "macos")]
278pub fn monitor_main_thread_hangs(
279    http_client: Arc<HttpClientWithUrl>,
280    installation_id: Option<String>,
281    cx: &App,
282) {
283    // This is too noisy to ship to stable for now.
284    if !matches!(
285        ReleaseChannel::global(cx),
286        ReleaseChannel::Dev | ReleaseChannel::Nightly | ReleaseChannel::Preview
287    ) {
288        return;
289    }
290
291    use nix::sys::signal::{
292        SaFlags, SigAction, SigHandler, SigSet,
293        Signal::{self, SIGUSR2},
294        sigaction,
295    };
296
297    use parking_lot::Mutex;
298
299    use http_client::Method;
300    use std::{
301        ffi::c_int,
302        sync::{OnceLock, mpsc},
303        time::Duration,
304    };
305    use telemetry_events::{BacktraceFrame, HangReport};
306
307    use nix::sys::pthread;
308
309    let foreground_executor = cx.foreground_executor();
310    let background_executor = cx.background_executor();
311    let telemetry_settings = *client::TelemetrySettings::get_global(cx);
312
313    // Initialize SIGUSR2 handler to send a backtrace to a channel.
314    let (backtrace_tx, backtrace_rx) = mpsc::channel();
315    static BACKTRACE: Mutex<Vec<backtrace::Frame>> = Mutex::new(Vec::new());
316    static BACKTRACE_SENDER: OnceLock<mpsc::Sender<()>> = OnceLock::new();
317    BACKTRACE_SENDER.get_or_init(|| backtrace_tx);
318    BACKTRACE.lock().reserve(100);
319
320    fn handle_backtrace_signal() {
321        unsafe {
322            extern "C" fn handle_sigusr2(_i: c_int) {
323                unsafe {
324                    // ASYNC SIGNAL SAFETY: This lock is only accessed one other time,
325                    // which can only be triggered by This signal handler. In addition,
326                    // this signal handler is immediately removed by SA_RESETHAND, and this
327                    // signal handler cannot be re-entrant due to the SIGUSR2 mask defined
328                    // below
329                    let mut bt = BACKTRACE.lock();
330                    bt.clear();
331                    backtrace::trace_unsynchronized(|frame| {
332                        if bt.len() < bt.capacity() {
333                            bt.push(frame.clone());
334                            true
335                        } else {
336                            false
337                        }
338                    });
339                }
340
341                BACKTRACE_SENDER.get().unwrap().send(()).ok();
342            }
343
344            let mut mask = SigSet::empty();
345            mask.add(SIGUSR2);
346            sigaction(
347                Signal::SIGUSR2,
348                &SigAction::new(
349                    SigHandler::Handler(handle_sigusr2),
350                    SaFlags::SA_RESTART | SaFlags::SA_RESETHAND,
351                    mask,
352                ),
353            )
354            .log_err();
355        }
356    }
357
358    handle_backtrace_signal();
359    let main_thread = pthread::pthread_self();
360
361    let (mut tx, mut rx) = futures::channel::mpsc::channel(3);
362    foreground_executor
363        .spawn(async move { while (rx.next().await).is_some() {} })
364        .detach();
365
366    background_executor
367        .spawn({
368            let background_executor = background_executor.clone();
369            async move {
370                loop {
371                    background_executor.timer(Duration::from_secs(1)).await;
372                    match tx.try_send(()) {
373                        Ok(_) => continue,
374                        Err(e) => {
375                            if e.into_send_error().is_full() {
376                                pthread::pthread_kill(main_thread, SIGUSR2).log_err();
377                            }
378                            // Only detect the first hang
379                            break;
380                        }
381                    }
382                }
383            }
384        })
385        .detach();
386
387    let app_version = release_channel::AppVersion::global(cx);
388    let os_name = client::telemetry::os_name();
389
390    background_executor
391        .clone()
392        .spawn(async move {
393            let os_version = client::telemetry::os_version();
394
395            loop {
396                while backtrace_rx.recv().is_ok() {
397                    if !telemetry_settings.diagnostics {
398                        return;
399                    }
400
401                    // ASYNC SIGNAL SAFETY: This lock is only accessed _after_
402                    // the backtrace transmitter has fired, which itself is only done
403                    // by the signal handler. And due to SA_RESETHAND  the signal handler
404                    // will not run again until `handle_backtrace_signal` is called.
405                    let raw_backtrace = BACKTRACE.lock().drain(..).collect::<Vec<_>>();
406                    let backtrace: Vec<_> = raw_backtrace
407                        .into_iter()
408                        .map(|frame| {
409                            let mut btf = BacktraceFrame {
410                                ip: frame.ip() as usize,
411                                symbol_addr: frame.symbol_address() as usize,
412                                base: frame.module_base_address().map(|addr| addr as usize),
413                                symbols: vec![],
414                            };
415
416                            backtrace::resolve_frame(&frame, |symbol| {
417                                if let Some(name) = symbol.name() {
418                                    btf.symbols.push(name.to_string());
419                                }
420                            });
421
422                            btf
423                        })
424                        .collect();
425
426                    // IMPORTANT: Don't move this to before `BACKTRACE.lock()`
427                    handle_backtrace_signal();
428
429                    log::error!(
430                        "Suspected hang on main thread:\n{}",
431                        backtrace
432                            .iter()
433                            .flat_map(|bt| bt.symbols.first().as_ref().map(|s| s.as_str()))
434                            .collect::<Vec<_>>()
435                            .join("\n")
436                    );
437
438                    let report = HangReport {
439                        backtrace,
440                        app_version: Some(app_version),
441                        os_name: os_name.clone(),
442                        os_version: Some(os_version.clone()),
443                        architecture: env::consts::ARCH.into(),
444                        installation_id: installation_id.clone(),
445                    };
446
447                    let Some(json_bytes) = serde_json::to_vec(&report).log_err() else {
448                        continue;
449                    };
450
451                    let Some(checksum) = client::telemetry::calculate_json_checksum(&json_bytes)
452                    else {
453                        continue;
454                    };
455
456                    let Ok(url) = http_client.build_zed_api_url("/telemetry/hangs", &[]) else {
457                        continue;
458                    };
459
460                    let Ok(request) = http_client::Request::builder()
461                        .method(Method::POST)
462                        .uri(url.as_ref())
463                        .header("x-zed-checksum", checksum)
464                        .body(json_bytes.into())
465                    else {
466                        continue;
467                    };
468
469                    if let Some(response) = http_client.send(request).await.log_err()
470                        && response.status() != 200
471                    {
472                        log::error!("Failed to send hang report: HTTP {:?}", response.status());
473                    }
474                }
475            }
476        })
477        .detach()
478}
479
480fn upload_panics_and_crashes(
481    http: Arc<HttpClientWithUrl>,
482    panic_report_url: Url,
483    installation_id: Option<String>,
484    cx: &App,
485) {
486    if !client::TelemetrySettings::get_global(cx).diagnostics {
487        return;
488    }
489    cx.background_spawn(async move {
490        upload_previous_minidumps(http.clone(), installation_id.clone())
491            .await
492            .warn_on_err();
493        let most_recent_panic = upload_previous_panics(http.clone(), &panic_report_url)
494            .await
495            .log_err()
496            .flatten();
497        upload_previous_crashes(http, most_recent_panic, installation_id)
498            .await
499            .log_err();
500    })
501    .detach()
502}
503
504/// Uploads panics via `zed.dev`.
505async fn upload_previous_panics(
506    http: Arc<HttpClientWithUrl>,
507    panic_report_url: &Url,
508) -> anyhow::Result<Option<(i64, String)>> {
509    let mut children = smol::fs::read_dir(paths::logs_dir()).await?;
510
511    let mut most_recent_panic = None;
512
513    while let Some(child) = children.next().await {
514        let child = child?;
515        let child_path = child.path();
516
517        if child_path.extension() != Some(OsStr::new("panic")) {
518            continue;
519        }
520        let filename = if let Some(filename) = child_path.file_name() {
521            filename.to_string_lossy()
522        } else {
523            continue;
524        };
525
526        if !filename.starts_with("zed") {
527            continue;
528        }
529
530        let panic_file_content = smol::fs::read_to_string(&child_path)
531            .await
532            .context("error reading panic file")?;
533
534        let panic: Option<Panic> = serde_json::from_str(&panic_file_content)
535            .log_err()
536            .or_else(|| {
537                panic_file_content
538                    .lines()
539                    .next()
540                    .and_then(|line| serde_json::from_str(line).ok())
541            })
542            .unwrap_or_else(|| {
543                log::error!("failed to deserialize panic file {:?}", panic_file_content);
544                None
545            });
546
547        if let Some(panic) = panic
548            && upload_panic(&http, panic_report_url, panic, &mut most_recent_panic).await?
549        {
550            // We've done what we can, delete the file
551            fs::remove_file(child_path)
552                .context("error removing panic")
553                .log_err();
554        }
555    }
556
557    Ok(most_recent_panic)
558}
559
560pub async fn upload_previous_minidumps(
561    http: Arc<HttpClientWithUrl>,
562    installation_id: Option<String>,
563) -> anyhow::Result<()> {
564    let Some(minidump_endpoint) = MINIDUMP_ENDPOINT.as_ref() else {
565        log::warn!("Minidump endpoint not set");
566        return Ok(());
567    };
568
569    let mut children = smol::fs::read_dir(paths::logs_dir()).await?;
570    while let Some(child) = children.next().await {
571        let child = child?;
572        let child_path = child.path();
573        if child_path.extension() != Some(OsStr::new("dmp")) {
574            continue;
575        }
576        let mut json_path = child_path.clone();
577        json_path.set_extension("json");
578        if let Ok(metadata) = serde_json::from_slice(&smol::fs::read(&json_path).await?)
579            && upload_minidump(
580                http.clone(),
581                minidump_endpoint,
582                smol::fs::read(&child_path)
583                    .await
584                    .context("Failed to read minidump")?,
585                &metadata,
586                installation_id.clone(),
587            )
588            .await
589            .log_err()
590            .is_some()
591        {
592            fs::remove_file(child_path).ok();
593            fs::remove_file(json_path).ok();
594        }
595    }
596    Ok(())
597}
598
599async fn upload_minidump(
600    http: Arc<HttpClientWithUrl>,
601    endpoint: &str,
602    minidump: Vec<u8>,
603    metadata: &crashes::CrashInfo,
604    installation_id: Option<String>,
605) -> Result<()> {
606    let mut form = Form::new()
607        .part(
608            "upload_file_minidump",
609            Part::bytes(minidump)
610                .file_name("minidump.dmp")
611                .mime_str("application/octet-stream")?,
612        )
613        .text(
614            "sentry[tags][channel]",
615            metadata.init.release_channel.clone(),
616        )
617        .text("sentry[tags][version]", metadata.init.zed_version.clone())
618        .text("sentry[release]", metadata.init.commit_sha.clone())
619        .text("platform", "rust");
620    let mut panic_message = "".to_owned();
621    if let Some(panic_info) = metadata.panic.as_ref() {
622        panic_message = panic_info.message.clone();
623        form = form
624            .text("sentry[logentry][formatted]", panic_info.message.clone())
625            .text("span", panic_info.span.clone());
626    }
627    if let Some(minidump_error) = metadata.minidump_error.clone() {
628        form = form.text("minidump_error", minidump_error);
629    }
630    if let Some(id) = installation_id.clone() {
631        form = form.text("sentry[user][id]", id)
632    }
633
634    ::telemetry::event!(
635        "Minidump Uploaded",
636        panic_message = panic_message,
637        crashed_version = metadata.init.zed_version.clone(),
638        commit_sha = metadata.init.commit_sha.clone(),
639    );
640
641    let gpu_count = metadata.gpus.len();
642    for (index, gpu) in metadata.gpus.iter().cloned().enumerate() {
643        let system_specs::GpuInfo {
644            device_name,
645            device_pci_id,
646            vendor_name,
647            vendor_pci_id,
648            driver_version,
649            driver_name,
650        } = gpu;
651        let num = if gpu_count == 1 && metadata.active_gpu.is_none() {
652            String::new()
653        } else {
654            index.to_string()
655        };
656        let name = format!("gpu{num}");
657        let root = format!("sentry[contexts][{name}]");
658        form = form
659            .text(
660                format!("{root}[Description]"),
661                "A GPU found on the users system. May or may not be the GPU Zed is running on",
662            )
663            .text(format!("{root}[type]"), "gpu")
664            .text(format!("{root}[name]"), device_name.unwrap_or(name))
665            .text(format!("{root}[id]"), format!("{:#06x}", device_pci_id))
666            .text(
667                format!("{root}[vendor_id]"),
668                format!("{:#06x}", vendor_pci_id),
669            )
670            .text_if_some(format!("{root}[vendor_name]"), vendor_name)
671            .text_if_some(format!("{root}[driver_version]"), driver_version)
672            .text_if_some(format!("{root}[driver_name]"), driver_name);
673    }
674    if let Some(active_gpu) = metadata.active_gpu.clone() {
675        form = form
676            .text(
677                "sentry[contexts][Active_GPU][Description]",
678                "The GPU Zed is running on",
679            )
680            .text("sentry[contexts][Active_GPU][type]", "gpu")
681            .text("sentry[contexts][Active_GPU][name]", active_gpu.device_name)
682            .text(
683                "sentry[contexts][Active_GPU][driver_version]",
684                active_gpu.driver_info,
685            )
686            .text(
687                "sentry[contexts][Active_GPU][driver_name]",
688                active_gpu.driver_name,
689            )
690            .text(
691                "sentry[contexts][Active_GPU][is_software_emulated]",
692                active_gpu.is_software_emulated.to_string(),
693            );
694    }
695
696    // TODO: feature-flag-context, and more of device-context like screen resolution, available ram, device model, etc
697
698    let mut response_text = String::new();
699    let mut response = http.send_multipart_form(endpoint, form).await?;
700    response
701        .body_mut()
702        .read_to_string(&mut response_text)
703        .await?;
704    if !response.status().is_success() {
705        anyhow::bail!("failed to upload minidump: {response_text}");
706    }
707    log::info!("Uploaded minidump. event id: {response_text}");
708    Ok(())
709}
710
711trait FormExt {
712    fn text_if_some(
713        self,
714        label: impl Into<std::borrow::Cow<'static, str>>,
715        value: Option<impl Into<std::borrow::Cow<'static, str>>>,
716    ) -> Self;
717}
718
719impl FormExt for Form {
720    fn text_if_some(
721        self,
722        label: impl Into<std::borrow::Cow<'static, str>>,
723        value: Option<impl Into<std::borrow::Cow<'static, str>>>,
724    ) -> Self {
725        match value {
726            Some(value) => self.text(label.into(), value.into()),
727            None => self,
728        }
729    }
730}
731
732async fn upload_panic(
733    http: &Arc<HttpClientWithUrl>,
734    panic_report_url: &Url,
735    panic: telemetry_events::Panic,
736    most_recent_panic: &mut Option<(i64, String)>,
737) -> Result<bool> {
738    *most_recent_panic = Some((panic.panicked_on, panic.payload.clone()));
739
740    let json_bytes = serde_json::to_vec(&PanicRequest { panic }).unwrap();
741
742    let Some(checksum) = client::telemetry::calculate_json_checksum(&json_bytes) else {
743        return Ok(false);
744    };
745
746    let Ok(request) = http_client::Request::builder()
747        .method(Method::POST)
748        .uri(panic_report_url.as_ref())
749        .header("x-zed-checksum", checksum)
750        .body(json_bytes.into())
751    else {
752        return Ok(false);
753    };
754
755    let response = http.send(request).await.context("error sending panic")?;
756    if !response.status().is_success() {
757        log::error!("Error uploading panic to server: {}", response.status());
758    }
759
760    Ok(true)
761}
762const LAST_CRASH_UPLOADED: &str = "LAST_CRASH_UPLOADED";
763
764/// upload crashes from apple's diagnostic reports to our server.
765/// (only if telemetry is enabled)
766async fn upload_previous_crashes(
767    http: Arc<HttpClientWithUrl>,
768    most_recent_panic: Option<(i64, String)>,
769    installation_id: Option<String>,
770) -> Result<()> {
771    let last_uploaded = KEY_VALUE_STORE
772        .read_kvp(LAST_CRASH_UPLOADED)?
773        .unwrap_or("zed-2024-01-17-221900.ips".to_string()); // don't upload old crash reports from before we had this.
774    let mut uploaded = last_uploaded.clone();
775
776    let crash_report_url = http.build_zed_api_url("/telemetry/crashes", &[])?;
777
778    // Crash directories are only set on macOS.
779    for dir in [crashes_dir(), crashes_retired_dir()]
780        .iter()
781        .filter_map(|d| d.as_deref())
782    {
783        let mut children = smol::fs::read_dir(&dir).await?;
784        while let Some(child) = children.next().await {
785            let child = child?;
786            let Some(filename) = child
787                .path()
788                .file_name()
789                .map(|f| f.to_string_lossy().to_lowercase())
790            else {
791                continue;
792            };
793
794            if !filename.starts_with("zed-") || !filename.ends_with(".ips") {
795                continue;
796            }
797
798            if filename <= last_uploaded {
799                continue;
800            }
801
802            let body = smol::fs::read_to_string(&child.path())
803                .await
804                .context("error reading crash file")?;
805
806            let mut request = http_client::Request::post(&crash_report_url.to_string())
807                .follow_redirects(http_client::RedirectPolicy::FollowAll)
808                .header("Content-Type", "text/plain");
809
810            if let Some((panicked_on, payload)) = most_recent_panic.as_ref() {
811                request = request
812                    .header("x-zed-panicked-on", format!("{panicked_on}"))
813                    .header("x-zed-panic", payload)
814            }
815            if let Some(installation_id) = installation_id.as_ref() {
816                request = request.header("x-zed-installation-id", installation_id);
817            }
818
819            let request = request.body(body.into())?;
820
821            let response = http.send(request).await.context("error sending crash")?;
822            if !response.status().is_success() {
823                log::error!("Error uploading crash to server: {}", response.status());
824            }
825
826            if uploaded < filename {
827                uploaded.clone_from(&filename);
828                KEY_VALUE_STORE
829                    .write_kvp(LAST_CRASH_UPLOADED.to_string(), filename)
830                    .await?;
831            }
832        }
833    }
834
835    Ok(())
836}