From 2e6fa889ea8659ad01f7c7ca0217000539f1eeff Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 10 Jun 2022 13:32:56 -0700 Subject: [PATCH 1/3] Add OpenMetrics endpoint exposing the basic RPC store metrics as guages Co-authored-by: Antonio Scandurra --- Cargo.lock | 22 ++++++++++++++++++++++ crates/collab/Cargo.toml | 1 + crates/collab/src/rpc.rs | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 55 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 119ad4ea96fc901dfe83de64dedf42c22ee2b8d8..5d83f52ccfdd57521a3f8f59d665dce83134ae05 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -870,6 +870,7 @@ dependencies = [ "nanoid", "parking_lot 0.11.2", "project", + "prometheus", "rand 0.8.5", "reqwest", "rpc", @@ -3401,6 +3402,21 @@ dependencies = [ "workspace", ] +[[package]] +name = "prometheus" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cface98dfa6d645ea4c789839f176e4b072265d085bfcc48eaa8d137f58d3c39" +dependencies = [ + "cfg-if 1.0.0", + "fnv", + "lazy_static", + "memchr", + "parking_lot 0.12.1", + "protobuf", + "thiserror", +] + [[package]] name = "prost" version = "0.8.0" @@ -3477,6 +3493,12 @@ dependencies = [ "prost 0.9.0", ] +[[package]] +name = "protobuf" +version = "2.27.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf7e6d18738ecd0902d30d1ad232c9125985a3422929b16c65517b38adc14f96" + [[package]] name = "pulldown-cmark" version = "0.9.1" diff --git a/crates/collab/Cargo.toml b/crates/collab/Cargo.toml index a3c9e689ecca753c00f21ac93b7f66e452e4496a..4f2b505aacfa5af3961f62c63936695f8178ba82 100644 --- a/crates/collab/Cargo.toml +++ b/crates/collab/Cargo.toml @@ -31,6 +31,7 @@ lazy_static = "1.4" lipsum = { version = "0.8", optional = true } nanoid = "0.4" parking_lot = "0.11.1" +prometheus = "0.13" rand = "0.8" reqwest = { version = "0.11", features = ["json"], optional = true } scrypt = "0.7" diff --git a/crates/collab/src/rpc.rs b/crates/collab/src/rpc.rs index 40ce95bce61fb7181fae8f4cd644234281b6cacf..0184554ccac85fdeafc449716bce79e3f58415a3 100644 --- a/crates/collab/src/rpc.rs +++ b/crates/collab/src/rpc.rs @@ -29,6 +29,7 @@ use futures::{ FutureExt, SinkExt, StreamExt, TryStreamExt, }; use lazy_static::lazy_static; +use prometheus::{register_int_gauge, IntGauge}; use rpc::{ proto::{self, AnyTypedEnvelope, EntityMessage, EnvelopedMessage, RequestMessage}, Connection, ConnectionId, Peer, Receipt, TypedEnvelope, @@ -57,6 +58,18 @@ use tracing::{info_span, instrument, Instrument}; pub use store::{Store, Worktree}; +lazy_static! { + static ref METRIC_CONNECTIONS: IntGauge = + register_int_gauge!("connections", "number of connections").unwrap(); + static ref METRIC_PROJECTS: IntGauge = + register_int_gauge!("projects", "number of open projects").unwrap(); + static ref METRIC_SHARED_PROJECTS: IntGauge = register_int_gauge!( + "shared_projects", + "number of open projects with one or more guests" + ) + .unwrap(); +} + type MessageHandler = Box, Box) -> BoxFuture<'static, ()>>; @@ -1534,6 +1547,11 @@ impl<'a> Drop for StoreWriteGuard<'a> { self.check_invariants(); let metrics = self.metrics(); + + METRIC_CONNECTIONS.set(metrics.connections as _); + METRIC_PROJECTS.set(metrics.registered_projects as _); + METRIC_SHARED_PROJECTS.set(metrics.shared_projects as _); + tracing::info!( connections = metrics.connections, registered_projects = metrics.registered_projects, @@ -1609,6 +1627,7 @@ pub fn routes(server: Arc) -> Router { .layer(middleware::from_fn(auth::validate_header)) .layer(Extension(server)), ) + .route("/metrics", get(handle_metrics)) } pub async fn handle_websocket_request( @@ -1642,6 +1661,19 @@ pub async fn handle_websocket_request( }) } +pub async fn handle_metrics() -> axum::response::Response { + let encoder = prometheus::TextEncoder::new(); + let metric_families = prometheus::gather(); + match encoder.encode_to_string(&metric_families) { + Ok(string) => (StatusCode::OK, string).into_response(), + Err(error) => ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("failed to encode metrics {:?}", error), + ) + .into_response(), + } +} + fn to_axum_message(message: TungsteniteMessage) -> AxumMessage { match message { TungsteniteMessage::Text(payload) => AxumMessage::Text(payload), From 2311534c3c46f58c3ff4b59d7363a02c2be39318 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 10 Jun 2022 13:35:35 -0700 Subject: [PATCH 2/3] Add DataDog OpenMetrics annotations to collab k8s deployment Co-authored-by: Antonio Scandurra --- crates/collab/k8s/manifest.template.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/crates/collab/k8s/manifest.template.yml b/crates/collab/k8s/manifest.template.yml index 9387f232d1034d44f96b3e50faefa1ba356bf78a..36fa32a441543e3a6bcde5ccc4dfa48527b2bfeb 100644 --- a/crates/collab/k8s/manifest.template.yml +++ b/crates/collab/k8s/manifest.template.yml @@ -27,6 +27,20 @@ kind: Deployment metadata: namespace: ${ZED_KUBE_NAMESPACE} name: collab + annotations: + ad.datadoghq.com/collab.check_names: | + ["openmetrics"] + ad.datadoghq.com/collab.init_configs: | + [{}] + ad.datadoghq.com/collab.instances: | + [ + { + "openmetrics_endpoint": "http://%%host%%:%%port%%/metrics", + "namespace": "collab_${ZED_KUBE_NAMESPACE}", + "metrics": [".*"] + } + ] + spec: replicas: 1 selector: From e2935100db17dc284921a382911495adae44c7a3 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 10 Jun 2022 14:32:36 -0700 Subject: [PATCH 3/3] Move prometheus annotations from deployment to pod spec --- crates/collab/k8s/manifest.template.yml | 26 ++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/crates/collab/k8s/manifest.template.yml b/crates/collab/k8s/manifest.template.yml index 36fa32a441543e3a6bcde5ccc4dfa48527b2bfeb..6f3eb26df454c6ae34e4cbdc614daf8441ef2e99 100644 --- a/crates/collab/k8s/manifest.template.yml +++ b/crates/collab/k8s/manifest.template.yml @@ -27,19 +27,6 @@ kind: Deployment metadata: namespace: ${ZED_KUBE_NAMESPACE} name: collab - annotations: - ad.datadoghq.com/collab.check_names: | - ["openmetrics"] - ad.datadoghq.com/collab.init_configs: | - [{}] - ad.datadoghq.com/collab.instances: | - [ - { - "openmetrics_endpoint": "http://%%host%%:%%port%%/metrics", - "namespace": "collab_${ZED_KUBE_NAMESPACE}", - "metrics": [".*"] - } - ] spec: replicas: 1 @@ -50,6 +37,19 @@ spec: metadata: labels: app: collab + annotations: + ad.datadoghq.com/collab.check_names: | + ["openmetrics"] + ad.datadoghq.com/collab.init_configs: | + [{}] + ad.datadoghq.com/collab.instances: | + [ + { + "openmetrics_endpoint": "http://%%host%%:%%port%%/metrics", + "namespace": "collab_${ZED_KUBE_NAMESPACE}", + "metrics": [".*"] + } + ] spec: containers: - name: collab