ssh_session.rs

  1use crate::{
  2    json_log::LogRecord,
  3    protocol::{
  4        message_len_from_buffer, read_message_with_len, write_message, MessageId, MESSAGE_LEN_SIZE,
  5    },
  6};
  7use anyhow::{anyhow, Context as _, Result};
  8use collections::HashMap;
  9use futures::{
 10    channel::{mpsc, oneshot},
 11    future::BoxFuture,
 12    select_biased, AsyncReadExt as _, AsyncWriteExt as _, Future, FutureExt as _, StreamExt as _,
 13};
 14use gpui::{AppContext, AsyncAppContext, Model, SemanticVersion};
 15use parking_lot::Mutex;
 16use rpc::proto::{
 17    self, build_typed_envelope, EntityMessageSubscriber, Envelope, EnvelopedMessage, PeerId,
 18    ProtoClient, ProtoMessageHandlerSet, RequestMessage,
 19};
 20use smol::{
 21    fs,
 22    process::{self, Stdio},
 23};
 24use std::{
 25    any::TypeId,
 26    ffi::OsStr,
 27    path::{Path, PathBuf},
 28    sync::{
 29        atomic::{AtomicU32, Ordering::SeqCst},
 30        Arc,
 31    },
 32    time::Instant,
 33};
 34use tempfile::TempDir;
 35
 36#[derive(Clone)]
 37pub struct SshSocket {
 38    connection_options: SshConnectionOptions,
 39    socket_path: PathBuf,
 40}
 41
 42pub struct SshSession {
 43    next_message_id: AtomicU32,
 44    response_channels: ResponseChannels,
 45    outgoing_tx: mpsc::UnboundedSender<Envelope>,
 46    spawn_process_tx: mpsc::UnboundedSender<SpawnRequest>,
 47    client_socket: Option<SshSocket>,
 48    state: Mutex<ProtoMessageHandlerSet>,
 49}
 50
 51struct SshClientState {
 52    socket: SshSocket,
 53    _master_process: process::Child,
 54    _temp_dir: TempDir,
 55}
 56
 57#[derive(Debug, Clone, PartialEq, Eq)]
 58pub struct SshConnectionOptions {
 59    pub host: String,
 60    pub username: Option<String>,
 61    pub port: Option<u16>,
 62    pub password: Option<String>,
 63}
 64
 65impl SshConnectionOptions {
 66    pub fn ssh_url(&self) -> String {
 67        let mut result = String::from("ssh://");
 68        if let Some(username) = &self.username {
 69            result.push_str(username);
 70            result.push('@');
 71        }
 72        result.push_str(&self.host);
 73        if let Some(port) = self.port {
 74            result.push(':');
 75            result.push_str(&port.to_string());
 76        }
 77        result
 78    }
 79
 80    fn scp_url(&self) -> String {
 81        if let Some(username) = &self.username {
 82            format!("{}@{}", username, self.host)
 83        } else {
 84            self.host.clone()
 85        }
 86    }
 87
 88    pub fn connection_string(&self) -> String {
 89        let host = if let Some(username) = &self.username {
 90            format!("{}@{}", username, self.host)
 91        } else {
 92            self.host.clone()
 93        };
 94        if let Some(port) = &self.port {
 95            format!("{}:{}", host, port)
 96        } else {
 97            host
 98        }
 99    }
100}
101
102struct SpawnRequest {
103    command: String,
104    process_tx: oneshot::Sender<process::Child>,
105}
106
107#[derive(Copy, Clone, Debug)]
108pub struct SshPlatform {
109    pub os: &'static str,
110    pub arch: &'static str,
111}
112
113pub trait SshClientDelegate {
114    fn ask_password(
115        &self,
116        prompt: String,
117        cx: &mut AsyncAppContext,
118    ) -> oneshot::Receiver<Result<String>>;
119    fn remote_server_binary_path(&self, cx: &mut AsyncAppContext) -> Result<PathBuf>;
120    fn get_server_binary(
121        &self,
122        platform: SshPlatform,
123        cx: &mut AsyncAppContext,
124    ) -> oneshot::Receiver<Result<(PathBuf, SemanticVersion)>>;
125    fn set_status(&self, status: Option<&str>, cx: &mut AsyncAppContext);
126}
127
128type ResponseChannels = Mutex<HashMap<MessageId, oneshot::Sender<(Envelope, oneshot::Sender<()>)>>>;
129
130impl SshSession {
131    pub async fn client(
132        connection_options: SshConnectionOptions,
133        delegate: Arc<dyn SshClientDelegate>,
134        cx: &mut AsyncAppContext,
135    ) -> Result<Arc<Self>> {
136        let client_state = SshClientState::new(connection_options, delegate.clone(), cx).await?;
137
138        let platform = client_state.query_platform().await?;
139        let (local_binary_path, version) = delegate.get_server_binary(platform, cx).await??;
140        let remote_binary_path = delegate.remote_server_binary_path(cx)?;
141        client_state
142            .ensure_server_binary(
143                &delegate,
144                &local_binary_path,
145                &remote_binary_path,
146                version,
147                cx,
148            )
149            .await?;
150
151        let (spawn_process_tx, mut spawn_process_rx) = mpsc::unbounded::<SpawnRequest>();
152        let (outgoing_tx, mut outgoing_rx) = mpsc::unbounded::<Envelope>();
153        let (incoming_tx, incoming_rx) = mpsc::unbounded::<Envelope>();
154
155        let socket = client_state.socket.clone();
156        run_cmd(socket.ssh_command(&remote_binary_path).arg("version")).await?;
157
158        let mut remote_server_child = socket
159            .ssh_command(format!(
160                "RUST_LOG={} {:?} run",
161                std::env::var("RUST_LOG").unwrap_or_default(),
162                remote_binary_path,
163            ))
164            .spawn()
165            .context("failed to spawn remote server")?;
166        let mut child_stderr = remote_server_child.stderr.take().unwrap();
167        let mut child_stdout = remote_server_child.stdout.take().unwrap();
168        let mut child_stdin = remote_server_child.stdin.take().unwrap();
169
170        let executor = cx.background_executor().clone();
171        executor.clone().spawn(async move {
172            let mut stdin_buffer = Vec::new();
173            let mut stdout_buffer = Vec::new();
174            let mut stderr_buffer = Vec::new();
175            let mut stderr_offset = 0;
176
177            loop {
178                stdout_buffer.resize(MESSAGE_LEN_SIZE, 0);
179                stderr_buffer.resize(stderr_offset + 1024, 0);
180
181                select_biased! {
182                    outgoing = outgoing_rx.next().fuse() => {
183                        let Some(outgoing) = outgoing else {
184                            return anyhow::Ok(());
185                        };
186
187                        write_message(&mut child_stdin, &mut stdin_buffer, outgoing).await?;
188                    }
189
190                    request = spawn_process_rx.next().fuse() => {
191                        let Some(request) = request else {
192                            return Ok(());
193                        };
194
195                        log::info!("spawn process: {:?}", request.command);
196                        let child = client_state.socket
197                            .ssh_command(&request.command)
198                            .spawn()
199                            .context("failed to create channel")?;
200                        request.process_tx.send(child).ok();
201                    }
202
203                    result = child_stdout.read(&mut stdout_buffer).fuse() => {
204                        match result {
205                            Ok(len) => {
206                                if len == 0 {
207                                    child_stdin.close().await?;
208                                    let status = remote_server_child.status().await?;
209                                    if !status.success() {
210                                        log::info!("channel exited with status: {status:?}");
211                                    }
212                                    return Ok(());
213                                }
214
215                                if len < stdout_buffer.len() {
216                                    child_stdout.read_exact(&mut stdout_buffer[len..]).await?;
217                                }
218
219                                let message_len = message_len_from_buffer(&stdout_buffer);
220                                match read_message_with_len(&mut child_stdout, &mut stdout_buffer, message_len).await {
221                                    Ok(envelope) => {
222                                        incoming_tx.unbounded_send(envelope).ok();
223                                    }
224                                    Err(error) => {
225                                        log::error!("error decoding message {error:?}");
226                                    }
227                                }
228                            }
229                            Err(error) => {
230                                Err(anyhow!("error reading stdout: {error:?}"))?;
231                            }
232                        }
233                    }
234
235                    result = child_stderr.read(&mut stderr_buffer[stderr_offset..]).fuse() => {
236                        match result {
237                            Ok(len) => {
238                                stderr_offset += len;
239                                let mut start_ix = 0;
240                                while let Some(ix) = stderr_buffer[start_ix..stderr_offset].iter().position(|b| b == &b'\n') {
241                                    let line_ix = start_ix + ix;
242                                    let content = &stderr_buffer[start_ix..line_ix];
243                                    start_ix = line_ix + 1;
244                                    if let Ok(record) = serde_json::from_slice::<LogRecord>(content) {
245                                        record.log(log::logger())
246                                    } else {
247                                        eprintln!("(remote) {}", String::from_utf8_lossy(content));
248                                    }
249                                }
250                                stderr_buffer.drain(0..start_ix);
251                                stderr_offset -= start_ix;
252                            }
253                            Err(error) => {
254                                Err(anyhow!("error reading stderr: {error:?}"))?;
255                            }
256                        }
257                    }
258                }
259            }
260        }).detach();
261
262        cx.update(|cx| Self::new(incoming_rx, outgoing_tx, spawn_process_tx, Some(socket), cx))
263    }
264
265    pub fn server(
266        incoming_rx: mpsc::UnboundedReceiver<Envelope>,
267        outgoing_tx: mpsc::UnboundedSender<Envelope>,
268        cx: &AppContext,
269    ) -> Arc<SshSession> {
270        let (tx, _rx) = mpsc::unbounded();
271        Self::new(incoming_rx, outgoing_tx, tx, None, cx)
272    }
273
274    #[cfg(any(test, feature = "test-support"))]
275    pub fn fake(
276        client_cx: &mut gpui::TestAppContext,
277        server_cx: &mut gpui::TestAppContext,
278    ) -> (Arc<Self>, Arc<Self>) {
279        let (server_to_client_tx, server_to_client_rx) = mpsc::unbounded();
280        let (client_to_server_tx, client_to_server_rx) = mpsc::unbounded();
281        let (tx, _rx) = mpsc::unbounded();
282        (
283            client_cx.update(|cx| {
284                Self::new(
285                    server_to_client_rx,
286                    client_to_server_tx,
287                    tx.clone(),
288                    None, // todo()
289                    cx,
290                )
291            }),
292            server_cx.update(|cx| {
293                Self::new(
294                    client_to_server_rx,
295                    server_to_client_tx,
296                    tx.clone(),
297                    None,
298                    cx,
299                )
300            }),
301        )
302    }
303
304    fn new(
305        mut incoming_rx: mpsc::UnboundedReceiver<Envelope>,
306        outgoing_tx: mpsc::UnboundedSender<Envelope>,
307        spawn_process_tx: mpsc::UnboundedSender<SpawnRequest>,
308        client_socket: Option<SshSocket>,
309        cx: &AppContext,
310    ) -> Arc<SshSession> {
311        let this = Arc::new(Self {
312            next_message_id: AtomicU32::new(0),
313            response_channels: ResponseChannels::default(),
314            outgoing_tx,
315            spawn_process_tx,
316            client_socket,
317            state: Default::default(),
318        });
319
320        cx.spawn(|cx| {
321            let this = this.clone();
322            async move {
323                let peer_id = PeerId { owner_id: 0, id: 0 };
324                while let Some(incoming) = incoming_rx.next().await {
325                    if let Some(request_id) = incoming.responding_to {
326                        let request_id = MessageId(request_id);
327                        let sender = this.response_channels.lock().remove(&request_id);
328                        if let Some(sender) = sender {
329                            let (tx, rx) = oneshot::channel();
330                            if incoming.payload.is_some() {
331                                sender.send((incoming, tx)).ok();
332                            }
333                            rx.await.ok();
334                        }
335                    } else if let Some(envelope) =
336                        build_typed_envelope(peer_id, Instant::now(), incoming)
337                    {
338                        let type_name = envelope.payload_type_name();
339                        if let Some(future) = ProtoMessageHandlerSet::handle_message(
340                            &this.state,
341                            envelope,
342                            this.clone().into(),
343                            cx.clone(),
344                        ) {
345                            log::debug!("ssh message received. name:{type_name}");
346                            match future.await {
347                                Ok(_) => {
348                                    log::debug!("ssh message handled. name:{type_name}");
349                                }
350                                Err(error) => {
351                                    log::error!(
352                                        "error handling message. type:{type_name}, error:{error:?}",
353                                    );
354                                }
355                            }
356                        } else {
357                            log::error!("unhandled ssh message name:{type_name}");
358                        }
359                    }
360                }
361                anyhow::Ok(())
362            }
363        })
364        .detach();
365
366        this
367    }
368
369    pub fn request<T: RequestMessage>(
370        &self,
371        payload: T,
372    ) -> impl 'static + Future<Output = Result<T::Response>> {
373        log::debug!("ssh request start. name:{}", T::NAME);
374        let response = self.request_dynamic(payload.into_envelope(0, None, None), "");
375        async move {
376            let response = response.await?;
377            log::debug!("ssh request finish. name:{}", T::NAME);
378            T::Response::from_envelope(response)
379                .ok_or_else(|| anyhow!("received a response of the wrong type"))
380        }
381    }
382
383    pub fn send<T: EnvelopedMessage>(&self, payload: T) -> Result<()> {
384        log::debug!("ssh send name:{}", T::NAME);
385        self.send_dynamic(payload.into_envelope(0, None, None))
386    }
387
388    pub fn request_dynamic(
389        &self,
390        mut envelope: proto::Envelope,
391        _request_type: &'static str,
392    ) -> impl 'static + Future<Output = Result<proto::Envelope>> {
393        envelope.id = self.next_message_id.fetch_add(1, SeqCst);
394        let (tx, rx) = oneshot::channel();
395        self.response_channels
396            .lock()
397            .insert(MessageId(envelope.id), tx);
398        self.outgoing_tx.unbounded_send(envelope).ok();
399        async move { Ok(rx.await.context("connection lost")?.0) }
400    }
401
402    pub fn send_dynamic(&self, mut envelope: proto::Envelope) -> Result<()> {
403        envelope.id = self.next_message_id.fetch_add(1, SeqCst);
404        self.outgoing_tx.unbounded_send(envelope)?;
405        Ok(())
406    }
407
408    pub fn subscribe_to_entity<E: 'static>(&self, remote_id: u64, entity: &Model<E>) {
409        let id = (TypeId::of::<E>(), remote_id);
410
411        let mut state = self.state.lock();
412        if state.entities_by_type_and_remote_id.contains_key(&id) {
413            panic!("already subscribed to entity");
414        }
415
416        state.entities_by_type_and_remote_id.insert(
417            id,
418            EntityMessageSubscriber::Entity {
419                handle: entity.downgrade().into(),
420            },
421        );
422    }
423
424    pub async fn spawn_process(&self, command: String) -> process::Child {
425        let (process_tx, process_rx) = oneshot::channel();
426        self.spawn_process_tx
427            .unbounded_send(SpawnRequest {
428                command,
429                process_tx,
430            })
431            .ok();
432        process_rx.await.unwrap()
433    }
434
435    pub fn ssh_args(&self) -> Vec<String> {
436        self.client_socket.as_ref().unwrap().ssh_args()
437    }
438}
439
440impl ProtoClient for SshSession {
441    fn request(
442        &self,
443        envelope: proto::Envelope,
444        request_type: &'static str,
445    ) -> BoxFuture<'static, Result<proto::Envelope>> {
446        self.request_dynamic(envelope, request_type).boxed()
447    }
448
449    fn send(&self, envelope: proto::Envelope, _message_type: &'static str) -> Result<()> {
450        self.send_dynamic(envelope)
451    }
452
453    fn send_response(&self, envelope: Envelope, _message_type: &'static str) -> anyhow::Result<()> {
454        self.send_dynamic(envelope)
455    }
456
457    fn message_handler_set(&self) -> &Mutex<ProtoMessageHandlerSet> {
458        &self.state
459    }
460}
461
462impl SshClientState {
463    #[cfg(not(unix))]
464    async fn new(
465        _connection_options: SshConnectionOptions,
466        _delegate: Arc<dyn SshClientDelegate>,
467        _cx: &mut AsyncAppContext,
468    ) -> Result<Self> {
469        Err(anyhow!("ssh is not supported on this platform"))
470    }
471
472    #[cfg(unix)]
473    async fn new(
474        connection_options: SshConnectionOptions,
475        delegate: Arc<dyn SshClientDelegate>,
476        cx: &mut AsyncAppContext,
477    ) -> Result<Self> {
478        use futures::{io::BufReader, AsyncBufReadExt as _};
479        use smol::{fs::unix::PermissionsExt as _, net::unix::UnixListener};
480        use util::ResultExt as _;
481
482        delegate.set_status(Some("connecting"), cx);
483
484        let url = connection_options.ssh_url();
485        let temp_dir = tempfile::Builder::new()
486            .prefix("zed-ssh-session")
487            .tempdir()?;
488
489        // Create a domain socket listener to handle requests from the askpass program.
490        let askpass_socket = temp_dir.path().join("askpass.sock");
491        let listener =
492            UnixListener::bind(&askpass_socket).context("failed to create askpass socket")?;
493
494        let askpass_task = cx.spawn(|mut cx| async move {
495            while let Ok((mut stream, _)) = listener.accept().await {
496                let mut buffer = Vec::new();
497                let mut reader = BufReader::new(&mut stream);
498                if reader.read_until(b'\0', &mut buffer).await.is_err() {
499                    buffer.clear();
500                }
501                let password_prompt = String::from_utf8_lossy(&buffer);
502                if let Some(password) = delegate
503                    .ask_password(password_prompt.to_string(), &mut cx)
504                    .await
505                    .context("failed to get ssh password")
506                    .and_then(|p| p)
507                    .log_err()
508                {
509                    stream.write_all(password.as_bytes()).await.log_err();
510                }
511            }
512        });
513
514        // Create an askpass script that communicates back to this process.
515        let askpass_script = format!(
516            "{shebang}\n{print_args} | nc -U {askpass_socket} 2> /dev/null \n",
517            askpass_socket = askpass_socket.display(),
518            print_args = "printf '%s\\0' \"$@\"",
519            shebang = "#!/bin/sh",
520        );
521        let askpass_script_path = temp_dir.path().join("askpass.sh");
522        fs::write(&askpass_script_path, askpass_script).await?;
523        fs::set_permissions(&askpass_script_path, std::fs::Permissions::from_mode(0o755)).await?;
524
525        // Start the master SSH process, which does not do anything except for establish
526        // the connection and keep it open, allowing other ssh commands to reuse it
527        // via a control socket.
528        let socket_path = temp_dir.path().join("ssh.sock");
529        let mut master_process = process::Command::new("ssh")
530            .stdin(Stdio::null())
531            .stdout(Stdio::piped())
532            .stderr(Stdio::piped())
533            .env("SSH_ASKPASS_REQUIRE", "force")
534            .env("SSH_ASKPASS", &askpass_script_path)
535            .args(["-N", "-o", "ControlMaster=yes", "-o"])
536            .arg(format!("ControlPath={}", socket_path.display()))
537            .arg(&url)
538            .spawn()?;
539
540        // Wait for this ssh process to close its stdout, indicating that authentication
541        // has completed.
542        let stdout = master_process.stdout.as_mut().unwrap();
543        let mut output = Vec::new();
544        stdout.read_to_end(&mut output).await?;
545        drop(askpass_task);
546
547        if master_process.try_status()?.is_some() {
548            output.clear();
549            let mut stderr = master_process.stderr.take().unwrap();
550            stderr.read_to_end(&mut output).await?;
551            Err(anyhow!(
552                "failed to connect: {}",
553                String::from_utf8_lossy(&output)
554            ))?;
555        }
556
557        Ok(Self {
558            socket: SshSocket {
559                connection_options,
560                socket_path,
561            },
562            _master_process: master_process,
563            _temp_dir: temp_dir,
564        })
565    }
566
567    async fn ensure_server_binary(
568        &self,
569        delegate: &Arc<dyn SshClientDelegate>,
570        src_path: &Path,
571        dst_path: &Path,
572        version: SemanticVersion,
573        cx: &mut AsyncAppContext,
574    ) -> Result<()> {
575        let mut dst_path_gz = dst_path.to_path_buf();
576        dst_path_gz.set_extension("gz");
577
578        if let Some(parent) = dst_path.parent() {
579            run_cmd(self.socket.ssh_command("mkdir").arg("-p").arg(parent)).await?;
580        }
581
582        let mut server_binary_exists = false;
583        if cfg!(not(debug_assertions)) {
584            if let Ok(installed_version) =
585                run_cmd(self.socket.ssh_command(dst_path).arg("version")).await
586            {
587                if installed_version.trim() == version.to_string() {
588                    server_binary_exists = true;
589                }
590            }
591        }
592
593        if server_binary_exists {
594            log::info!("remote development server already present",);
595            return Ok(());
596        }
597
598        let src_stat = fs::metadata(src_path).await?;
599        let size = src_stat.len();
600        let server_mode = 0o755;
601
602        let t0 = Instant::now();
603        delegate.set_status(Some("uploading remote development server"), cx);
604        log::info!("uploading remote development server ({}kb)", size / 1024);
605        self.upload_file(src_path, &dst_path_gz)
606            .await
607            .context("failed to upload server binary")?;
608        log::info!("uploaded remote development server in {:?}", t0.elapsed());
609
610        delegate.set_status(Some("extracting remote development server"), cx);
611        run_cmd(
612            self.socket
613                .ssh_command("gunzip")
614                .arg("--force")
615                .arg(&dst_path_gz),
616        )
617        .await?;
618
619        delegate.set_status(Some("unzipping remote development server"), cx);
620        run_cmd(
621            self.socket
622                .ssh_command("chmod")
623                .arg(format!("{:o}", server_mode))
624                .arg(dst_path),
625        )
626        .await?;
627
628        Ok(())
629    }
630
631    async fn query_platform(&self) -> Result<SshPlatform> {
632        let os = run_cmd(self.socket.ssh_command("uname").arg("-s")).await?;
633        let arch = run_cmd(self.socket.ssh_command("uname").arg("-m")).await?;
634
635        let os = match os.trim() {
636            "Darwin" => "macos",
637            "Linux" => "linux",
638            _ => Err(anyhow!("unknown uname os {os:?}"))?,
639        };
640        let arch = if arch.starts_with("arm") || arch.starts_with("aarch64") {
641            "aarch64"
642        } else if arch.starts_with("x86") || arch.starts_with("i686") {
643            "x86_64"
644        } else {
645            Err(anyhow!("unknown uname architecture {arch:?}"))?
646        };
647
648        Ok(SshPlatform { os, arch })
649    }
650
651    async fn upload_file(&self, src_path: &Path, dest_path: &Path) -> Result<()> {
652        let mut command = process::Command::new("scp");
653        let output = self
654            .socket
655            .ssh_options(&mut command)
656            .args(
657                self.socket
658                    .connection_options
659                    .port
660                    .map(|port| vec!["-P".to_string(), port.to_string()])
661                    .unwrap_or_default(),
662            )
663            .arg(src_path)
664            .arg(format!(
665                "{}:{}",
666                self.socket.connection_options.scp_url(),
667                dest_path.display()
668            ))
669            .output()
670            .await?;
671
672        if output.status.success() {
673            Ok(())
674        } else {
675            Err(anyhow!(
676                "failed to upload file {} -> {}: {}",
677                src_path.display(),
678                dest_path.display(),
679                String::from_utf8_lossy(&output.stderr)
680            ))
681        }
682    }
683}
684
685impl SshSocket {
686    fn ssh_command<S: AsRef<OsStr>>(&self, program: S) -> process::Command {
687        let mut command = process::Command::new("ssh");
688        self.ssh_options(&mut command)
689            .arg(self.connection_options.ssh_url())
690            .arg(program);
691        command
692    }
693
694    fn ssh_options<'a>(&self, command: &'a mut process::Command) -> &'a mut process::Command {
695        command
696            .stdin(Stdio::piped())
697            .stdout(Stdio::piped())
698            .stderr(Stdio::piped())
699            .args(["-o", "ControlMaster=no", "-o"])
700            .arg(format!("ControlPath={}", self.socket_path.display()))
701    }
702
703    fn ssh_args(&self) -> Vec<String> {
704        vec![
705            "-o".to_string(),
706            "ControlMaster=no".to_string(),
707            "-o".to_string(),
708            format!("ControlPath={}", self.socket_path.display()),
709            self.connection_options.ssh_url(),
710        ]
711    }
712}
713
714async fn run_cmd(command: &mut process::Command) -> Result<String> {
715    let output = command.output().await?;
716    if output.status.success() {
717        Ok(String::from_utf8_lossy(&output.stdout).to_string())
718    } else {
719        Err(anyhow!(
720            "failed to run command: {}",
721            String::from_utf8_lossy(&output.stderr)
722        ))
723    }
724}