From 969cf76a1d0dfeb368b0ee115a83f27b954b4850 Mon Sep 17 00:00:00 2001 From: Carlos Alexandro Becker Date: Wed, 31 May 2023 16:19:49 -0300 Subject: [PATCH] fix: metric cardinality et al (#315) --- server/cmd/cmd.go | 23 +++++++++++--- server/ssh/session.go | 32 +++++++++++++------ server/ssh/ssh.go | 73 +++++++++++++++++++++++++++++++------------ 3 files changed, 94 insertions(+), 34 deletions(-) diff --git a/server/cmd/cmd.go b/server/cmd/cmd.go index c30dd8b29c69a3cc6a284b4655cb1a78699a39ac..16d89d1639fd2153899a90da431e7e7fbe8fa448 100644 --- a/server/cmd/cmd.go +++ b/server/cmd/cmd.go @@ -15,13 +15,20 @@ import ( "github.com/charmbracelet/soft-serve/server/utils" "github.com/charmbracelet/ssh" "github.com/charmbracelet/wish" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" "github.com/spf13/cobra" ) -var ( - // sessionCtxKey is the key for the session in the context. - sessionCtxKey = &struct{ string }{"session"} -) +// sessionCtxKey is the key for the session in the context. +var sessionCtxKey = &struct{ string }{"session"} + +var cliCommandCounter = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: "soft_serve", + Subsystem: "cli", + Name: "commands_total", + Help: "Total times each command was called", +}, []string{"command"}) var templateFuncs = template.FuncMap{ "trim": strings.TrimSpace, @@ -75,8 +82,16 @@ func rpad(s string, padding int) string { return fmt.Sprintf(template, s) } +func cmdName(args []string) string { + if len(args) == 0 { + return "" + } + return args[0] +} + // rootCommand is the root command for the server. func rootCommand(cfg *config.Config, s ssh.Session) *cobra.Command { + cliCommandCounter.WithLabelValues(cmdName(s.Command())).Inc() rootCmd := &cobra.Command{ Short: "Soft Serve is a self-hostable Git server for the command line.", SilenceUsage: true, diff --git a/server/ssh/session.go b/server/ssh/session.go index b39e15e24cc305c14eac1f59c777c0baa5f20f52..26ee3a5032b881c9f8f2670bd3daf7accc850bb1 100644 --- a/server/ssh/session.go +++ b/server/ssh/session.go @@ -2,6 +2,7 @@ package ssh import ( "strings" + "time" tea "github.com/charmbracelet/bubbletea" "github.com/charmbracelet/log" @@ -19,19 +20,23 @@ import ( "github.com/prometheus/client_golang/prometheus/promauto" ) -var ( - tuiSessionCounter = promauto.NewCounterVec(prometheus.CounterOpts{ - Namespace: "soft_serve", - Subsystem: "ssh", - Name: "tui_session_total", - Help: "The total number of TUI sessions", - }, []string{"key", "user", "repo", "term"}) -) +var tuiSessionCounter = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: "soft_serve", + Subsystem: "ssh", + Name: "tui_session_total", + Help: "The total number of TUI sessions", +}, []string{"repo", "term"}) + +var tuiSessionDuration = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: "soft_serve", + Subsystem: "ssh", + Name: "tui_session_seconds_total", + Help: "The total number of TUI sessions", +}, []string{"repo", "term"}) // SessionHandler is the soft-serve bubbletea ssh session handler. func SessionHandler(cfg *config.Config) bm.ProgramHandler { return func(s ssh.Session) *tea.Program { - ak := backend.MarshalAuthorizedKey(s.PublicKey()) pty, _, active := s.Pty() if !active { return nil @@ -61,9 +66,16 @@ func SessionHandler(cfg *config.Config) bm.ProgramHandler { tea.WithAltScreen(), tea.WithoutCatchPanics(), tea.WithMouseCellMotion(), + tea.WithContext(ctx), ) - tuiSessionCounter.WithLabelValues(ak, s.User(), initialRepo, pty.Term).Inc() + tuiSessionCounter.WithLabelValues(initialRepo, pty.Term).Inc() + + start := time.Now() + go func() { + <-ctx.Done() + tuiSessionDuration.WithLabelValues(initialRepo, pty.Term).Add(time.Since(start).Seconds()) + }() return p } diff --git a/server/ssh/ssh.go b/server/ssh/ssh.go index 6e727fbd2431fa2d81631f44f025266e4808215f..42cd7f5f543044881d79af9725e9c4a55211d277 100644 --- a/server/ssh/ssh.go +++ b/server/ssh/ssh.go @@ -35,42 +35,63 @@ var ( Subsystem: "ssh", Name: "public_key_auth_total", Help: "The total number of public key auth requests", - }, []string{"key", "user", "allowed"}) + }, []string{"allowed"}) keyboardInteractiveCounter = promauto.NewCounterVec(prometheus.CounterOpts{ Namespace: "soft_serve", Subsystem: "ssh", Name: "keyboard_interactive_auth_total", Help: "The total number of keyboard interactive auth requests", - }, []string{"user", "allowed"}) + }, []string{"allowed"}) uploadPackCounter = promauto.NewCounterVec(prometheus.CounterOpts{ Namespace: "soft_serve", - Subsystem: "ssh", - Name: "git_upload_pack_total", + Subsystem: "git", + Name: "upload_pack_total", Help: "The total number of git-upload-pack requests", - }, []string{"key", "user", "repo"}) + }, []string{"repo"}) receivePackCounter = promauto.NewCounterVec(prometheus.CounterOpts{ Namespace: "soft_serve", - Subsystem: "ssh", - Name: "git_receive_pack_total", + Subsystem: "git", + Name: "receive_pack_total", Help: "The total number of git-receive-pack requests", - }, []string{"key", "user", "repo"}) + }, []string{"repo"}) uploadArchiveCounter = promauto.NewCounterVec(prometheus.CounterOpts{ Namespace: "soft_serve", - Subsystem: "ssh", - Name: "git_upload_archive_total", + Subsystem: "git", + Name: "upload_archive_total", Help: "The total number of git-upload-archive requests", - }, []string{"key", "user", "repo"}) + }, []string{"repo"}) + + uploadPackSeconds = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: "soft_serve", + Subsystem: "git", + Name: "upload_pack_seconds_total", + Help: "The total time spent on git-upload-pack requests", + }, []string{"repo"}) + + receivePackSeconds = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: "soft_serve", + Subsystem: "git", + Name: "receive_pack_seconds_total", + Help: "The total time spent on git-receive-pack requests", + }, []string{"repo"}) + + uploadArchiveSeconds = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: "soft_serve", + Subsystem: "git", + Name: "upload_archive_seconds_total", + Help: "The total time spent on git-upload-archive requests", + }, []string{"repo"}) createRepoCounter = promauto.NewCounterVec(prometheus.CounterOpts{ Namespace: "soft_serve", Subsystem: "ssh", Name: "create_repo_total", Help: "The total number of create repo requests", - }, []string{"key", "user", "repo"}) + }, []string{"repo"}) ) // SSHServer is a SSH server that implements the git protocol. @@ -168,7 +189,7 @@ func (s *SSHServer) PublicKeyHandler(ctx ssh.Context, pk ssh.PublicKey) (allowed ak := backend.MarshalAuthorizedKey(pk) defer func(allowed *bool) { - publicKeyCounter.WithLabelValues(ak, ctx.User(), strconv.FormatBool(*allowed)).Inc() + publicKeyCounter.WithLabelValues(strconv.FormatBool(*allowed)).Inc() }(&allowed) ac := s.cfg.Backend.AccessLevelByPublicKey("", pk) @@ -181,7 +202,7 @@ func (s *SSHServer) PublicKeyHandler(ctx ssh.Context, pk ssh.PublicKey) (allowed // This is used after all public key authentication has failed. func (s *SSHServer) KeyboardInteractiveHandler(ctx ssh.Context, _ gossh.KeyboardInteractiveChallenge) bool { ac := s.cfg.Backend.AllowKeyless() - keyboardInteractiveCounter.WithLabelValues(ctx.User(), strconv.FormatBool(ac)).Inc() + keyboardInteractiveCounter.WithLabelValues(strconv.FormatBool(ac)).Inc() return ac } @@ -194,6 +215,7 @@ func (ss *SSHServer) Middleware(cfg *config.Config) wish.Middleware { return func(sh ssh.Handler) ssh.Handler { return func(s ssh.Session) { func() { + start := time.Now() cmd := s.Command() ctx := s.Context() be := ss.be.WithContext(ctx) @@ -224,6 +246,10 @@ func (ss *SSHServer) Middleware(cfg *config.Config) wish.Middleware { repoDir := filepath.Join(reposDir, repo) switch gc { case git.ReceivePackBin: + receivePackCounter.WithLabelValues(name).Inc() + defer func() { + receivePackSeconds.WithLabelValues(name).Add(time.Since(start).Seconds()) + }() if access < backend.ReadWriteAccess { sshFatal(s, git.ErrNotAuthed) return @@ -234,12 +260,11 @@ func (ss *SSHServer) Middleware(cfg *config.Config) wish.Middleware { sshFatal(s, err) return } - createRepoCounter.WithLabelValues(ak, s.User(), name).Inc() + createRepoCounter.WithLabelValues(name).Inc() } if err := git.ReceivePack(s.Context(), s, s, s.Stderr(), repoDir, envs...); err != nil { sshFatal(s, git.ErrSystemMalfunction) } - receivePackCounter.WithLabelValues(ak, s.User(), name).Inc() return case git.UploadPackBin, git.UploadArchiveBin: if access < backend.ReadOnlyAccess { @@ -248,10 +273,19 @@ func (ss *SSHServer) Middleware(cfg *config.Config) wish.Middleware { } gitPack := git.UploadPack - counter := uploadPackCounter - if gc == git.UploadArchiveBin { + switch gc { + case git.UploadArchiveBin: gitPack = git.UploadArchive - counter = uploadArchiveCounter + uploadArchiveCounter.WithLabelValues(name).Inc() + defer func() { + uploadArchiveSeconds.WithLabelValues(name).Add(time.Since(start).Seconds()) + }() + default: + uploadPackCounter.WithLabelValues(name).Inc() + defer func() { + uploadPackSeconds.WithLabelValues(name).Add(time.Since(start).Seconds()) + }() + } err := gitPack(ctx, s, s, s.Stderr(), repoDir, envs...) @@ -261,7 +295,6 @@ func (ss *SSHServer) Middleware(cfg *config.Config) wish.Middleware { sshFatal(s, git.ErrSystemMalfunction) } - counter.WithLabelValues(ak, s.User(), name).Inc() } } }()