reliability.rs

  1use crate::stdout_is_a_pty;
  2use anyhow::{Context as _, Result};
  3use backtrace::{self, Backtrace};
  4use chrono::Utc;
  5use client::{
  6    TelemetrySettings,
  7    telemetry::{self, MINIDUMP_ENDPOINT},
  8};
  9use db::kvp::KEY_VALUE_STORE;
 10use futures::AsyncReadExt;
 11use gpui::{App, AppContext as _, SemanticVersion};
 12use http_client::{self, HttpClient, HttpClientWithUrl, HttpRequestExt, Method};
 13use paths::{crashes_dir, crashes_retired_dir};
 14use project::Project;
 15use proto::{CrashReport, GetCrashFilesResponse};
 16use release_channel::{AppCommitSha, RELEASE_CHANNEL, ReleaseChannel};
 17use reqwest::multipart::{Form, Part};
 18use settings::Settings;
 19use smol::stream::StreamExt;
 20use std::{
 21    env,
 22    ffi::{OsStr, c_void},
 23    fs,
 24    io::Write,
 25    panic,
 26    sync::{
 27        Arc,
 28        atomic::{AtomicU32, Ordering},
 29    },
 30    thread,
 31};
 32use telemetry_events::{LocationData, Panic, PanicRequest};
 33use url::Url;
 34use util::ResultExt;
 35
 36static PANIC_COUNT: AtomicU32 = AtomicU32::new(0);
 37
 38pub fn init_panic_hook(
 39    app_version: SemanticVersion,
 40    app_commit_sha: Option<AppCommitSha>,
 41    system_id: Option<String>,
 42    installation_id: Option<String>,
 43    session_id: String,
 44) {
 45    let is_pty = stdout_is_a_pty();
 46
 47    panic::set_hook(Box::new(move |info| {
 48        let prior_panic_count = PANIC_COUNT.fetch_add(1, Ordering::SeqCst);
 49        if prior_panic_count > 0 {
 50            // Give the panic-ing thread time to write the panic file
 51            loop {
 52                thread::yield_now();
 53            }
 54        }
 55
 56        let payload = info
 57            .payload()
 58            .downcast_ref::<&str>()
 59            .map(|s| s.to_string())
 60            .or_else(|| info.payload().downcast_ref::<String>().cloned())
 61            .unwrap_or_else(|| "Box<Any>".to_string());
 62
 63        if *release_channel::RELEASE_CHANNEL != ReleaseChannel::Dev {
 64            crashes::handle_panic(payload.clone(), info.location());
 65        }
 66
 67        let thread = thread::current();
 68        let thread_name = thread.name().unwrap_or("<unnamed>");
 69
 70        if *release_channel::RELEASE_CHANNEL == ReleaseChannel::Dev {
 71            let location = info.location().unwrap();
 72            let backtrace = Backtrace::new();
 73            eprintln!(
 74                "Thread {:?} panicked with {:?} at {}:{}:{}\n{}{:?}",
 75                thread_name,
 76                payload,
 77                location.file(),
 78                location.line(),
 79                location.column(),
 80                match app_commit_sha.as_ref() {
 81                    Some(commit_sha) => format!(
 82                        "https://github.com/zed-industries/zed/blob/{}/{}#L{} \
 83                        (may not be uploaded, line may be incorrect if files modified)\n",
 84                        commit_sha.full(),
 85                        location.file(),
 86                        location.line()
 87                    ),
 88                    None => "".to_string(),
 89                },
 90                backtrace,
 91            );
 92            if MINIDUMP_ENDPOINT.is_none() {
 93                std::process::exit(-1);
 94            }
 95        }
 96        let main_module_base_address = get_main_module_base_address();
 97
 98        let backtrace = Backtrace::new();
 99        let mut symbols = backtrace
100            .frames()
101            .iter()
102            .flat_map(|frame| {
103                let base = frame
104                    .module_base_address()
105                    .unwrap_or(main_module_base_address);
106                frame.symbols().iter().map(move |symbol| {
107                    format!(
108                        "{}+{}",
109                        symbol
110                            .name()
111                            .as_ref()
112                            .map_or("<unknown>".to_owned(), <_>::to_string),
113                        (frame.ip() as isize).saturating_sub(base as isize)
114                    )
115                })
116            })
117            .collect::<Vec<_>>();
118
119        // Strip out leading stack frames for rust panic-handling.
120        if let Some(ix) = symbols
121            .iter()
122            .position(|name| name == "rust_begin_unwind" || name == "_rust_begin_unwind")
123        {
124            symbols.drain(0..=ix);
125        }
126
127        let panic_data = telemetry_events::Panic {
128            thread: thread_name.into(),
129            payload,
130            location_data: info.location().map(|location| LocationData {
131                file: location.file().into(),
132                line: location.line(),
133            }),
134            app_version: app_version.to_string(),
135            app_commit_sha: app_commit_sha.as_ref().map(|sha| sha.full()),
136            release_channel: RELEASE_CHANNEL.dev_name().into(),
137            target: env!("TARGET").to_owned().into(),
138            os_name: telemetry::os_name(),
139            os_version: Some(telemetry::os_version()),
140            architecture: env::consts::ARCH.into(),
141            panicked_on: Utc::now().timestamp_millis(),
142            backtrace: symbols,
143            system_id: system_id.clone(),
144            installation_id: installation_id.clone(),
145            session_id: session_id.clone(),
146        };
147
148        if let Some(panic_data_json) = serde_json::to_string_pretty(&panic_data).log_err() {
149            log::error!("{}", panic_data_json);
150        }
151        zlog::flush();
152
153        if (!is_pty || MINIDUMP_ENDPOINT.is_some())
154            && let Some(panic_data_json) = serde_json::to_string(&panic_data).log_err()
155        {
156            let timestamp = chrono::Utc::now().format("%Y_%m_%d %H_%M_%S").to_string();
157            let panic_file_path = paths::logs_dir().join(format!("zed-{timestamp}.panic"));
158            let panic_file = fs::OpenOptions::new()
159                .write(true)
160                .create_new(true)
161                .open(&panic_file_path)
162                .log_err();
163            if let Some(mut panic_file) = panic_file {
164                writeln!(&mut panic_file, "{panic_data_json}").log_err();
165                panic_file.flush().log_err();
166            }
167        }
168
169        std::process::abort();
170    }));
171}
172
173#[cfg(not(target_os = "windows"))]
174fn get_main_module_base_address() -> *mut c_void {
175    let mut dl_info = libc::Dl_info {
176        dli_fname: std::ptr::null(),
177        dli_fbase: std::ptr::null_mut(),
178        dli_sname: std::ptr::null(),
179        dli_saddr: std::ptr::null_mut(),
180    };
181    unsafe {
182        libc::dladdr(get_main_module_base_address as _, &mut dl_info);
183    }
184    dl_info.dli_fbase
185}
186
187#[cfg(target_os = "windows")]
188fn get_main_module_base_address() -> *mut c_void {
189    std::ptr::null_mut()
190}
191
192pub fn init(
193    http_client: Arc<HttpClientWithUrl>,
194    system_id: Option<String>,
195    installation_id: Option<String>,
196    session_id: String,
197    cx: &mut App,
198) {
199    #[cfg(target_os = "macos")]
200    monitor_main_thread_hangs(http_client.clone(), installation_id.clone(), cx);
201
202    let Some(panic_report_url) = http_client
203        .build_zed_api_url("/telemetry/panics", &[])
204        .log_err()
205    else {
206        return;
207    };
208
209    upload_panics_and_crashes(
210        http_client.clone(),
211        panic_report_url.clone(),
212        installation_id.clone(),
213        cx,
214    );
215
216    cx.observe_new(move |project: &mut Project, _, cx| {
217        let http_client = http_client.clone();
218        let panic_report_url = panic_report_url.clone();
219        let session_id = session_id.clone();
220        let installation_id = installation_id.clone();
221        let system_id = system_id.clone();
222
223        let Some(remote_client) = project.remote_client() else {
224            return;
225        };
226        remote_client.update(cx, |client, cx| {
227            if !TelemetrySettings::get_global(cx).diagnostics {
228                return;
229            }
230            let request = client.proto_client().request(proto::GetCrashFiles {});
231            cx.background_spawn(async move {
232                let GetCrashFilesResponse {
233                    legacy_panics,
234                    crashes,
235                } = request.await?;
236
237                for panic in legacy_panics {
238                    if let Some(mut panic) = serde_json::from_str::<Panic>(&panic).log_err() {
239                        panic.session_id = session_id.clone();
240                        panic.system_id = system_id.clone();
241                        panic.installation_id = installation_id.clone();
242                        upload_panic(&http_client, &panic_report_url, panic, &mut None).await?;
243                    }
244                }
245
246                let Some(endpoint) = MINIDUMP_ENDPOINT.as_ref() else {
247                    return Ok(());
248                };
249                for CrashReport {
250                    metadata,
251                    minidump_contents,
252                } in crashes
253                {
254                    if let Some(metadata) = serde_json::from_str(&metadata).log_err() {
255                        upload_minidump(
256                            http_client.clone(),
257                            endpoint,
258                            minidump_contents,
259                            &metadata,
260                            installation_id.clone(),
261                        )
262                        .await
263                        .log_err();
264                    }
265                }
266
267                anyhow::Ok(())
268            })
269            .detach_and_log_err(cx);
270        })
271    })
272    .detach();
273}
274
275#[cfg(target_os = "macos")]
276pub fn monitor_main_thread_hangs(
277    http_client: Arc<HttpClientWithUrl>,
278    installation_id: Option<String>,
279    cx: &App,
280) {
281    // This is too noisy to ship to stable for now.
282    if !matches!(
283        ReleaseChannel::global(cx),
284        ReleaseChannel::Dev | ReleaseChannel::Nightly | ReleaseChannel::Preview
285    ) {
286        return;
287    }
288
289    use nix::sys::signal::{
290        SaFlags, SigAction, SigHandler, SigSet,
291        Signal::{self, SIGUSR2},
292        sigaction,
293    };
294
295    use parking_lot::Mutex;
296
297    use http_client::Method;
298    use std::{
299        ffi::c_int,
300        sync::{OnceLock, mpsc},
301        time::Duration,
302    };
303    use telemetry_events::{BacktraceFrame, HangReport};
304
305    use nix::sys::pthread;
306
307    let foreground_executor = cx.foreground_executor();
308    let background_executor = cx.background_executor();
309    let telemetry_settings = *client::TelemetrySettings::get_global(cx);
310
311    // Initialize SIGUSR2 handler to send a backtrace to a channel.
312    let (backtrace_tx, backtrace_rx) = mpsc::channel();
313    static BACKTRACE: Mutex<Vec<backtrace::Frame>> = Mutex::new(Vec::new());
314    static BACKTRACE_SENDER: OnceLock<mpsc::Sender<()>> = OnceLock::new();
315    BACKTRACE_SENDER.get_or_init(|| backtrace_tx);
316    BACKTRACE.lock().reserve(100);
317
318    fn handle_backtrace_signal() {
319        unsafe {
320            extern "C" fn handle_sigusr2(_i: c_int) {
321                unsafe {
322                    // ASYNC SIGNAL SAFETY: This lock is only accessed one other time,
323                    // which can only be triggered by This signal handler. In addition,
324                    // this signal handler is immediately removed by SA_RESETHAND, and this
325                    // signal handler cannot be re-entrant due to the SIGUSR2 mask defined
326                    // below
327                    let mut bt = BACKTRACE.lock();
328                    bt.clear();
329                    backtrace::trace_unsynchronized(|frame| {
330                        if bt.len() < bt.capacity() {
331                            bt.push(frame.clone());
332                            true
333                        } else {
334                            false
335                        }
336                    });
337                }
338
339                BACKTRACE_SENDER.get().unwrap().send(()).ok();
340            }
341
342            let mut mask = SigSet::empty();
343            mask.add(SIGUSR2);
344            sigaction(
345                Signal::SIGUSR2,
346                &SigAction::new(
347                    SigHandler::Handler(handle_sigusr2),
348                    SaFlags::SA_RESTART | SaFlags::SA_RESETHAND,
349                    mask,
350                ),
351            )
352            .log_err();
353        }
354    }
355
356    handle_backtrace_signal();
357    let main_thread = pthread::pthread_self();
358
359    let (mut tx, mut rx) = futures::channel::mpsc::channel(3);
360    foreground_executor
361        .spawn(async move { while (rx.next().await).is_some() {} })
362        .detach();
363
364    background_executor
365        .spawn({
366            let background_executor = background_executor.clone();
367            async move {
368                loop {
369                    background_executor.timer(Duration::from_secs(1)).await;
370                    match tx.try_send(()) {
371                        Ok(_) => continue,
372                        Err(e) => {
373                            if e.into_send_error().is_full() {
374                                pthread::pthread_kill(main_thread, SIGUSR2).log_err();
375                            }
376                            // Only detect the first hang
377                            break;
378                        }
379                    }
380                }
381            }
382        })
383        .detach();
384
385    let app_version = release_channel::AppVersion::global(cx);
386    let os_name = client::telemetry::os_name();
387
388    background_executor
389        .clone()
390        .spawn(async move {
391            let os_version = client::telemetry::os_version();
392
393            loop {
394                while backtrace_rx.recv().is_ok() {
395                    if !telemetry_settings.diagnostics {
396                        return;
397                    }
398
399                    // ASYNC SIGNAL SAFETY: This lock is only accessed _after_
400                    // the backtrace transmitter has fired, which itself is only done
401                    // by the signal handler. And due to SA_RESETHAND  the signal handler
402                    // will not run again until `handle_backtrace_signal` is called.
403                    let raw_backtrace = BACKTRACE.lock().drain(..).collect::<Vec<_>>();
404                    let backtrace: Vec<_> = raw_backtrace
405                        .into_iter()
406                        .map(|frame| {
407                            let mut btf = BacktraceFrame {
408                                ip: frame.ip() as usize,
409                                symbol_addr: frame.symbol_address() as usize,
410                                base: frame.module_base_address().map(|addr| addr as usize),
411                                symbols: vec![],
412                            };
413
414                            backtrace::resolve_frame(&frame, |symbol| {
415                                if let Some(name) = symbol.name() {
416                                    btf.symbols.push(name.to_string());
417                                }
418                            });
419
420                            btf
421                        })
422                        .collect();
423
424                    // IMPORTANT: Don't move this to before `BACKTRACE.lock()`
425                    handle_backtrace_signal();
426
427                    log::error!(
428                        "Suspected hang on main thread:\n{}",
429                        backtrace
430                            .iter()
431                            .flat_map(|bt| bt.symbols.first().as_ref().map(|s| s.as_str()))
432                            .collect::<Vec<_>>()
433                            .join("\n")
434                    );
435
436                    let report = HangReport {
437                        backtrace,
438                        app_version: Some(app_version),
439                        os_name: os_name.clone(),
440                        os_version: Some(os_version.clone()),
441                        architecture: env::consts::ARCH.into(),
442                        installation_id: installation_id.clone(),
443                    };
444
445                    let Some(json_bytes) = serde_json::to_vec(&report).log_err() else {
446                        continue;
447                    };
448
449                    let Some(checksum) = client::telemetry::calculate_json_checksum(&json_bytes)
450                    else {
451                        continue;
452                    };
453
454                    let Ok(url) = http_client.build_zed_api_url("/telemetry/hangs", &[]) else {
455                        continue;
456                    };
457
458                    let Ok(request) = http_client::Request::builder()
459                        .method(Method::POST)
460                        .uri(url.as_ref())
461                        .header("x-zed-checksum", checksum)
462                        .body(json_bytes.into())
463                    else {
464                        continue;
465                    };
466
467                    if let Some(response) = http_client.send(request).await.log_err()
468                        && response.status() != 200
469                    {
470                        log::error!("Failed to send hang report: HTTP {:?}", response.status());
471                    }
472                }
473            }
474        })
475        .detach()
476}
477
478fn upload_panics_and_crashes(
479    http: Arc<HttpClientWithUrl>,
480    panic_report_url: Url,
481    installation_id: Option<String>,
482    cx: &App,
483) {
484    if !client::TelemetrySettings::get_global(cx).diagnostics {
485        return;
486    }
487    cx.background_spawn(async move {
488        upload_previous_minidumps(http.clone(), installation_id.clone())
489            .await
490            .warn_on_err();
491        let most_recent_panic = upload_previous_panics(http.clone(), &panic_report_url)
492            .await
493            .log_err()
494            .flatten();
495        upload_previous_crashes(http, most_recent_panic, installation_id)
496            .await
497            .log_err();
498    })
499    .detach()
500}
501
502/// Uploads panics via `zed.dev`.
503async fn upload_previous_panics(
504    http: Arc<HttpClientWithUrl>,
505    panic_report_url: &Url,
506) -> anyhow::Result<Option<(i64, String)>> {
507    let mut children = smol::fs::read_dir(paths::logs_dir()).await?;
508
509    let mut most_recent_panic = None;
510
511    while let Some(child) = children.next().await {
512        let child = child?;
513        let child_path = child.path();
514
515        if child_path.extension() != Some(OsStr::new("panic")) {
516            continue;
517        }
518        let filename = if let Some(filename) = child_path.file_name() {
519            filename.to_string_lossy()
520        } else {
521            continue;
522        };
523
524        if !filename.starts_with("zed") {
525            continue;
526        }
527
528        let panic_file_content = smol::fs::read_to_string(&child_path)
529            .await
530            .context("error reading panic file")?;
531
532        let panic: Option<Panic> = serde_json::from_str(&panic_file_content)
533            .log_err()
534            .or_else(|| {
535                panic_file_content
536                    .lines()
537                    .next()
538                    .and_then(|line| serde_json::from_str(line).ok())
539            })
540            .unwrap_or_else(|| {
541                log::error!("failed to deserialize panic file {:?}", panic_file_content);
542                None
543            });
544
545        if let Some(panic) = panic
546            && upload_panic(&http, panic_report_url, panic, &mut most_recent_panic).await?
547        {
548            // We've done what we can, delete the file
549            fs::remove_file(child_path)
550                .context("error removing panic")
551                .log_err();
552        }
553    }
554
555    Ok(most_recent_panic)
556}
557
558pub async fn upload_previous_minidumps(
559    http: Arc<HttpClientWithUrl>,
560    installation_id: Option<String>,
561) -> anyhow::Result<()> {
562    let Some(minidump_endpoint) = MINIDUMP_ENDPOINT.as_ref() else {
563        log::warn!("Minidump endpoint not set");
564        return Ok(());
565    };
566
567    let mut children = smol::fs::read_dir(paths::logs_dir()).await?;
568    while let Some(child) = children.next().await {
569        let child = child?;
570        let child_path = child.path();
571        if child_path.extension() != Some(OsStr::new("dmp")) {
572            continue;
573        }
574        let mut json_path = child_path.clone();
575        json_path.set_extension("json");
576        if let Ok(metadata) = serde_json::from_slice(&smol::fs::read(&json_path).await?)
577            && upload_minidump(
578                http.clone(),
579                minidump_endpoint,
580                smol::fs::read(&child_path)
581                    .await
582                    .context("Failed to read minidump")?,
583                &metadata,
584                installation_id.clone(),
585            )
586            .await
587            .log_err()
588            .is_some()
589        {
590            fs::remove_file(child_path).ok();
591            fs::remove_file(json_path).ok();
592        }
593    }
594    Ok(())
595}
596
597async fn upload_minidump(
598    http: Arc<HttpClientWithUrl>,
599    endpoint: &str,
600    minidump: Vec<u8>,
601    metadata: &crashes::CrashInfo,
602    installation_id: Option<String>,
603) -> Result<()> {
604    let mut form = Form::new()
605        .part(
606            "upload_file_minidump",
607            Part::bytes(minidump)
608                .file_name("minidump.dmp")
609                .mime_str("application/octet-stream")?,
610        )
611        .text(
612            "sentry[tags][channel]",
613            metadata.init.release_channel.clone(),
614        )
615        .text("sentry[tags][version]", metadata.init.zed_version.clone())
616        .text("sentry[release]", metadata.init.commit_sha.clone())
617        .text("platform", "rust");
618    let mut panic_message = "".to_owned();
619    if let Some(panic_info) = metadata.panic.as_ref() {
620        panic_message = panic_info.message.clone();
621        form = form
622            .text("sentry[logentry][formatted]", panic_info.message.clone())
623            .text("span", panic_info.span.clone());
624    }
625    if let Some(minidump_error) = metadata.minidump_error.clone() {
626        form = form.text("minidump_error", minidump_error);
627    }
628    if let Some(id) = installation_id.clone() {
629        form = form.text("sentry[user][id]", id)
630    }
631
632    ::telemetry::event!(
633        "Minidump Uploaded",
634        panic_message = panic_message,
635        crashed_version = metadata.init.zed_version.clone(),
636        commit_sha = metadata.init.commit_sha.clone(),
637    );
638
639    let gpu_count = metadata.gpus.len();
640    for (index, gpu) in metadata.gpus.iter().cloned().enumerate() {
641        let system_specs::GpuInfo {
642            device_name,
643            device_pci_id,
644            vendor_name,
645            vendor_pci_id,
646            driver_version,
647            driver_name,
648        } = gpu;
649        let num = if gpu_count == 1 && metadata.active_gpu.is_none() {
650            String::new()
651        } else {
652            index.to_string()
653        };
654        let name = format!("gpu{num}");
655        let root = format!("sentry[contexts][{name}]");
656        form = form
657            .text(
658                format!("{root}[Description]"),
659                "A GPU found on the users system. May or may not be the GPU Zed is running on",
660            )
661            .text(format!("{root}[type]"), "gpu")
662            .text(format!("{root}[name]"), device_name.unwrap_or(name))
663            .text(format!("{root}[id]"), format!("{:#06x}", device_pci_id))
664            .text(
665                format!("{root}[vendor_id]"),
666                format!("{:#06x}", vendor_pci_id),
667            )
668            .text_if_some(format!("{root}[vendor_name]"), vendor_name)
669            .text_if_some(format!("{root}[driver_version]"), driver_version)
670            .text_if_some(format!("{root}[driver_name]"), driver_name);
671    }
672    if let Some(active_gpu) = metadata.active_gpu.clone() {
673        form = form
674            .text(
675                "sentry[contexts][Active_GPU][Description]",
676                "The GPU Zed is running on",
677            )
678            .text("sentry[contexts][Active_GPU][type]", "gpu")
679            .text("sentry[contexts][Active_GPU][name]", active_gpu.device_name)
680            .text(
681                "sentry[contexts][Active_GPU][driver_version]",
682                active_gpu.driver_info,
683            )
684            .text(
685                "sentry[contexts][Active_GPU][driver_name]",
686                active_gpu.driver_name,
687            )
688            .text(
689                "sentry[contexts][Active_GPU][is_software_emulated]",
690                active_gpu.is_software_emulated.to_string(),
691            );
692    }
693
694    // TODO: feature-flag-context, and more of device-context like screen resolution, available ram, device model, etc
695
696    let mut response_text = String::new();
697    let mut response = http.send_multipart_form(endpoint, form).await?;
698    response
699        .body_mut()
700        .read_to_string(&mut response_text)
701        .await?;
702    if !response.status().is_success() {
703        anyhow::bail!("failed to upload minidump: {response_text}");
704    }
705    log::info!("Uploaded minidump. event id: {response_text}");
706    Ok(())
707}
708
709trait FormExt {
710    fn text_if_some(
711        self,
712        label: impl Into<std::borrow::Cow<'static, str>>,
713        value: Option<impl Into<std::borrow::Cow<'static, str>>>,
714    ) -> Self;
715}
716
717impl FormExt for Form {
718    fn text_if_some(
719        self,
720        label: impl Into<std::borrow::Cow<'static, str>>,
721        value: Option<impl Into<std::borrow::Cow<'static, str>>>,
722    ) -> Self {
723        match value {
724            Some(value) => self.text(label.into(), value.into()),
725            None => self,
726        }
727    }
728}
729
730async fn upload_panic(
731    http: &Arc<HttpClientWithUrl>,
732    panic_report_url: &Url,
733    panic: telemetry_events::Panic,
734    most_recent_panic: &mut Option<(i64, String)>,
735) -> Result<bool> {
736    *most_recent_panic = Some((panic.panicked_on, panic.payload.clone()));
737
738    let json_bytes = serde_json::to_vec(&PanicRequest { panic }).unwrap();
739
740    let Some(checksum) = client::telemetry::calculate_json_checksum(&json_bytes) else {
741        return Ok(false);
742    };
743
744    let Ok(request) = http_client::Request::builder()
745        .method(Method::POST)
746        .uri(panic_report_url.as_ref())
747        .header("x-zed-checksum", checksum)
748        .body(json_bytes.into())
749    else {
750        return Ok(false);
751    };
752
753    let response = http.send(request).await.context("error sending panic")?;
754    if !response.status().is_success() {
755        log::error!("Error uploading panic to server: {}", response.status());
756    }
757
758    Ok(true)
759}
760const LAST_CRASH_UPLOADED: &str = "LAST_CRASH_UPLOADED";
761
762/// upload crashes from apple's diagnostic reports to our server.
763/// (only if telemetry is enabled)
764async fn upload_previous_crashes(
765    http: Arc<HttpClientWithUrl>,
766    most_recent_panic: Option<(i64, String)>,
767    installation_id: Option<String>,
768) -> Result<()> {
769    let last_uploaded = KEY_VALUE_STORE
770        .read_kvp(LAST_CRASH_UPLOADED)?
771        .unwrap_or("zed-2024-01-17-221900.ips".to_string()); // don't upload old crash reports from before we had this.
772    let mut uploaded = last_uploaded.clone();
773
774    let crash_report_url = http.build_zed_api_url("/telemetry/crashes", &[])?;
775
776    // Crash directories are only set on macOS.
777    for dir in [crashes_dir(), crashes_retired_dir()]
778        .iter()
779        .filter_map(|d| d.as_deref())
780    {
781        let mut children = smol::fs::read_dir(&dir).await?;
782        while let Some(child) = children.next().await {
783            let child = child?;
784            let Some(filename) = child
785                .path()
786                .file_name()
787                .map(|f| f.to_string_lossy().to_lowercase())
788            else {
789                continue;
790            };
791
792            if !filename.starts_with("zed-") || !filename.ends_with(".ips") {
793                continue;
794            }
795
796            if filename <= last_uploaded {
797                continue;
798            }
799
800            let body = smol::fs::read_to_string(&child.path())
801                .await
802                .context("error reading crash file")?;
803
804            let mut request = http_client::Request::post(&crash_report_url.to_string())
805                .follow_redirects(http_client::RedirectPolicy::FollowAll)
806                .header("Content-Type", "text/plain");
807
808            if let Some((panicked_on, payload)) = most_recent_panic.as_ref() {
809                request = request
810                    .header("x-zed-panicked-on", format!("{panicked_on}"))
811                    .header("x-zed-panic", payload)
812            }
813            if let Some(installation_id) = installation_id.as_ref() {
814                request = request.header("x-zed-installation-id", installation_id);
815            }
816
817            let request = request.body(body.into())?;
818
819            let response = http.send(request).await.context("error sending crash")?;
820            if !response.status().is_success() {
821                log::error!("Error uploading crash to server: {}", response.status());
822            }
823
824            if uploaded < filename {
825                uploaded.clone_from(&filename);
826                KEY_VALUE_STORE
827                    .write_kvp(LAST_CRASH_UPLOADED.to_string(), filename)
828                    .await?;
829            }
830        }
831    }
832
833    Ok(())
834}