From 01f7c278e9994e71ec71b0d1b2f8d0358c326566 Mon Sep 17 00:00:00 2001 From: Ayman Bagabas Date: Mon, 10 Apr 2023 12:58:31 -0400 Subject: [PATCH] refactor(server): move into modules --- server/{ => daemon}/daemon.go | 45 ++++++++++++++++++------------ server/{ => daemon}/daemon_test.go | 14 ++++++---- server/{ => git}/git.go | 44 ++++++++++++++--------------- server/server.go | 22 +++++++++------ server/server_test.go | 12 ++------ server/{ => ssh}/session.go | 2 +- server/{ => ssh}/session_test.go | 5 ++-- server/{ => ssh}/ssh.go | 35 +++++++++++++---------- server/{ => stats}/stats.go | 2 +- server/test/test.go | 9 ++++++ server/{ => web}/http.go | 7 ++++- 11 files changed, 113 insertions(+), 84 deletions(-) rename server/{ => daemon}/daemon.go (89%) rename server/{ => daemon}/daemon_test.go (83%) rename server/{ => git}/git.go (70%) rename server/{ => ssh}/session.go (99%) rename server/{ => ssh}/session_test.go (95%) rename server/{ => ssh}/ssh.go (90%) rename server/{ => stats}/stats.go (98%) create mode 100644 server/test/test.go rename server/{ => web}/http.go (98%) diff --git a/server/daemon.go b/server/daemon/daemon.go similarity index 89% rename from server/daemon.go rename to server/daemon/daemon.go index 79b94d88418307fedd16858afa6c13842067b4f9..1a10c8c3bc204183e9362f74a0da948e8e816621 100644 --- a/server/daemon.go +++ b/server/daemon/daemon.go @@ -1,4 +1,4 @@ -package server +package daemon import ( "bytes" @@ -9,14 +9,20 @@ import ( "sync" "time" + "github.com/charmbracelet/log" "github.com/charmbracelet/soft-serve/server/backend" "github.com/charmbracelet/soft-serve/server/config" + "github.com/charmbracelet/soft-serve/server/git" "github.com/charmbracelet/soft-serve/server/utils" "github.com/go-git/go-git/v5/plumbing/format/pktline" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" ) +var ( + logger = log.WithPrefix("server.daemon") +) + var ( uploadPackGitCounter = promauto.NewCounterVec(prometheus.CounterOpts{ Namespace: "soft_serve", @@ -33,8 +39,11 @@ var ( }, []string{"repo"}) ) -// ErrServerClosed indicates that the server has been closed. -var ErrServerClosed = fmt.Errorf("git: %w", net.ErrClosed) +var ( + + // ErrServerClosed indicates that the server has been closed. + ErrServerClosed = fmt.Errorf("git: %w", net.ErrClosed) +) // connections synchronizes access to to a net.Conn pool. type connections struct { @@ -133,7 +142,7 @@ func (d *GitDaemon) Start() error { // Close connection if there are too many open connections. if d.conns.Size()+1 >= d.cfg.Git.MaxConnections { logger.Debugf("git: max connections reached, closing %s", conn.RemoteAddr()) - fatal(conn, ErrMaxConnections) + fatal(conn, git.ErrMaxConnections) continue } @@ -146,7 +155,7 @@ func (d *GitDaemon) Start() error { } func fatal(c net.Conn, err error) { - writePktline(c, err) + git.WritePktline(c, err) if err := c.Close(); err != nil { logger.Debugf("git: error closing connection: %v", err) } @@ -176,10 +185,10 @@ func (d *GitDaemon) handleClient(conn net.Conn) { if !s.Scan() { if err := s.Err(); err != nil { if nerr, ok := err.(net.Error); ok && nerr.Timeout() { - fatal(c, ErrTimeout) + fatal(c, git.ErrTimeout) } else { logger.Debugf("git: error scanning pktline: %v", err) - fatal(c, ErrSystemMalfunction) + fatal(c, git.ErrSystemMalfunction) } } return @@ -197,32 +206,32 @@ func (d *GitDaemon) handleClient(conn net.Conn) { line := s.Bytes() split := bytes.SplitN(line, []byte{' '}, 2) if len(split) != 2 { - fatal(c, ErrInvalidRequest) + fatal(c, git.ErrInvalidRequest) return } - gitPack := uploadPack + gitPack := git.UploadPack counter := uploadPackGitCounter cmd := string(split[0]) switch cmd { - case uploadPackBin: - gitPack = uploadPack - case uploadArchiveBin: - gitPack = uploadArchive + case git.UploadPackBin: + gitPack = git.UploadPack + case git.UploadArchiveBin: + gitPack = git.UploadArchive counter = uploadArchiveGitCounter default: - fatal(c, ErrInvalidRequest) + fatal(c, git.ErrInvalidRequest) return } opts := bytes.Split(split[1], []byte{'\x00'}) if len(opts) == 0 { - fatal(c, ErrInvalidRequest) + fatal(c, git.ErrInvalidRequest) return } if !d.cfg.Backend.AllowKeyless() { - fatal(c, ErrNotAuthed) + fatal(c, git.ErrNotAuthed) return } @@ -233,14 +242,14 @@ func (d *GitDaemon) handleClient(conn net.Conn) { // https://git-scm.com/docs/gitrepository-layout repo := name + ".git" reposDir := filepath.Join(d.cfg.DataPath, "repos") - if err := ensureWithin(reposDir, repo); err != nil { + if err := git.EnsureWithin(reposDir, repo); err != nil { fatal(c, err) return } auth := d.cfg.Backend.AccessLevel(name, "") if auth < backend.ReadOnlyAccess { - fatal(c, ErrNotAuthed) + fatal(c, git.ErrNotAuthed) return } diff --git a/server/daemon_test.go b/server/daemon/daemon_test.go similarity index 83% rename from server/daemon_test.go rename to server/daemon/daemon_test.go index 765e62c9a84cc7240529768e04744a982f46e573..a87cdbfb92cac21f6cf9d70c7f9dfe2af7e0ddc0 100644 --- a/server/daemon_test.go +++ b/server/daemon/daemon_test.go @@ -1,4 +1,4 @@ -package server +package daemon import ( "bytes" @@ -14,6 +14,8 @@ import ( "github.com/charmbracelet/soft-serve/server/backend/sqlite" "github.com/charmbracelet/soft-serve/server/config" + "github.com/charmbracelet/soft-serve/server/git" + "github.com/charmbracelet/soft-serve/server/test" "github.com/go-git/go-git/v5/plumbing/format/pktline" ) @@ -29,7 +31,7 @@ func TestMain(m *testing.M) { os.Setenv("SOFT_SERVE_GIT_MAX_CONNECTIONS", "3") os.Setenv("SOFT_SERVE_GIT_MAX_TIMEOUT", "100") os.Setenv("SOFT_SERVE_GIT_IDLE_TIMEOUT", "1") - os.Setenv("SOFT_SERVE_GIT_LISTEN_ADDR", fmt.Sprintf(":%d", randomPort())) + os.Setenv("SOFT_SERVE_GIT_LISTEN_ADDR", fmt.Sprintf(":%d", test.RandomPort())) cfg := config.DefaultConfig() d, err := NewGitDaemon(cfg) if err != nil { @@ -67,8 +69,8 @@ func TestIdleTimeout(t *testing.T) { if err != nil && !errors.Is(err, io.EOF) { t.Fatalf("expected nil, got error: %v", err) } - if out != ErrTimeout.Error() || out == "" { - t.Fatalf("expected %q error, got %q", ErrTimeout, out) + if out != git.ErrTimeout.Error() || out == "" { + t.Fatalf("expected %q error, got %q", git.ErrTimeout, out) } } @@ -84,8 +86,8 @@ func TestInvalidRepo(t *testing.T) { if err != nil { t.Fatalf("expected nil, got error: %v", err) } - if out != ErrInvalidRepo.Error() { - t.Fatalf("expected %q error, got %q", ErrInvalidRepo, out) + if out != git.ErrInvalidRepo.Error() { + t.Fatalf("expected %q error, got %q", git.ErrInvalidRepo, out) } } diff --git a/server/git.go b/server/git/git.go similarity index 70% rename from server/git.go rename to server/git/git.go index 41a1609f1710051b7b1f46ebe0a789386c59a54d..53c192d5468cbecf84ea8d740945e9cc8c0d5ce0 100644 --- a/server/git.go +++ b/server/git/git.go @@ -1,4 +1,4 @@ -package server +package git import ( "errors" @@ -36,13 +36,13 @@ var ( // Git protocol commands. const ( - receivePackBin = "git-receive-pack" - uploadPackBin = "git-upload-pack" - uploadArchiveBin = "git-upload-archive" + ReceivePackBin = "git-receive-pack" + UploadPackBin = "git-upload-pack" + UploadArchiveBin = "git-upload-archive" ) -// uploadPack runs the git upload-pack protocol against the provided repo. -func uploadPack(in io.Reader, out io.Writer, er io.Writer, repoDir string) error { +// UploadPack runs the git upload-pack protocol against the provided repo. +func UploadPack(in io.Reader, out io.Writer, er io.Writer, repoDir string) error { exists, err := fileExists(repoDir) if !exists { return ErrInvalidRepo @@ -50,11 +50,11 @@ func uploadPack(in io.Reader, out io.Writer, er io.Writer, repoDir string) error if err != nil { return err } - return runGit(in, out, er, "", uploadPackBin[4:], repoDir) + return RunGit(in, out, er, "", UploadPackBin[4:], repoDir) } -// uploadArchive runs the git upload-archive protocol against the provided repo. -func uploadArchive(in io.Reader, out io.Writer, er io.Writer, repoDir string) error { +// UploadArchive runs the git upload-archive protocol against the provided repo. +func UploadArchive(in io.Reader, out io.Writer, er io.Writer, repoDir string) error { exists, err := fileExists(repoDir) if !exists { return ErrInvalidRepo @@ -62,19 +62,19 @@ func uploadArchive(in io.Reader, out io.Writer, er io.Writer, repoDir string) er if err != nil { return err } - return runGit(in, out, er, "", uploadArchiveBin[4:], repoDir) + return RunGit(in, out, er, "", UploadArchiveBin[4:], repoDir) } -// receivePack runs the git receive-pack protocol against the provided repo. -func receivePack(in io.Reader, out io.Writer, er io.Writer, repoDir string) error { - if err := runGit(in, out, er, "", receivePackBin[4:], repoDir); err != nil { +// ReceivePack runs the git receive-pack protocol against the provided repo. +func ReceivePack(in io.Reader, out io.Writer, er io.Writer, repoDir string) error { + if err := RunGit(in, out, er, "", ReceivePackBin[4:], repoDir); err != nil { return err } - return ensureDefaultBranch(in, out, er, repoDir) + return EnsureDefaultBranch(in, out, er, repoDir) } -// runGit runs a git command in the given repo. -func runGit(in io.Reader, out io.Writer, err io.Writer, dir string, args ...string) error { +// RunGit runs a git command in the given repo. +func RunGit(in io.Reader, out io.Writer, err io.Writer, dir string, args ...string) error { c := git.NewCommand(args...) return c.RunInDirWithOptions(dir, git.RunInDirOptions{ Stdin: in, @@ -83,8 +83,8 @@ func runGit(in io.Reader, out io.Writer, err io.Writer, dir string, args ...stri }) } -// writePktline encodes and writes a pktline to the given writer. -func writePktline(w io.Writer, v ...interface{}) { +// WritePktline encodes and writes a pktline to the given writer. +func WritePktline(w io.Writer, v ...interface{}) { msg := fmt.Sprintln(v...) pkt := pktline.NewEncoder(w) if err := pkt.EncodeString(msg); err != nil { @@ -95,8 +95,8 @@ func writePktline(w io.Writer, v ...interface{}) { } } -// ensureWithin ensures the given repo is within the repos directory. -func ensureWithin(reposDir string, repo string) error { +// EnsureWithin ensures the given repo is within the repos directory. +func EnsureWithin(reposDir string, repo string) error { repoDir := filepath.Join(reposDir, repo) absRepos, err := filepath.Abs(reposDir) if err != nil { @@ -129,7 +129,7 @@ func fileExists(path string) (bool, error) { return true, err } -func ensureDefaultBranch(in io.Reader, out io.Writer, er io.Writer, repoPath string) error { +func EnsureDefaultBranch(in io.Reader, out io.Writer, er io.Writer, repoPath string) error { r, err := git.Open(repoPath) if err != nil { return err @@ -144,7 +144,7 @@ func ensureDefaultBranch(in io.Reader, out io.Writer, er io.Writer, repoPath str // Rename the default branch to the first branch available _, err = r.HEAD() if err == git.ErrReferenceNotExist { - err = runGit(in, out, er, repoPath, "branch", "-M", brs[0]) + err = RunGit(in, out, er, repoPath, "branch", "-M", brs[0]) if err != nil { return err } diff --git a/server/server.go b/server/server.go index b4645f3cab1262667c37a07bd6e2c15763085f8e..d8be8069717ad7b0175b82ed4c8bf37c4797f617 100644 --- a/server/server.go +++ b/server/server.go @@ -13,6 +13,10 @@ import ( "github.com/charmbracelet/soft-serve/server/backend/sqlite" "github.com/charmbracelet/soft-serve/server/config" "github.com/charmbracelet/soft-serve/server/cron" + "github.com/charmbracelet/soft-serve/server/daemon" + sshsrv "github.com/charmbracelet/soft-serve/server/ssh" + "github.com/charmbracelet/soft-serve/server/stats" + "github.com/charmbracelet/soft-serve/server/web" "github.com/charmbracelet/ssh" "golang.org/x/sync/errgroup" ) @@ -23,10 +27,10 @@ var ( // Server is the Soft Serve server. type Server struct { - SSHServer *SSHServer - GitDaemon *GitDaemon - HTTPServer *HTTPServer - StatsServer *StatsServer + SSHServer *sshsrv.SSHServer + GitDaemon *daemon.GitDaemon + HTTPServer *web.HTTPServer + StatsServer *stats.StatsServer Cron *cron.CronScheduler Config *config.Config Backend backend.Backend @@ -81,22 +85,22 @@ func NewServer(ctx context.Context, cfg *config.Config) (*Server, error) { // Add cron jobs. srv.Cron.AddFunc(jobSpecs["mirror"], mirrorJob(cfg)) - srv.SSHServer, err = NewSSHServer(cfg, srv) + srv.SSHServer, err = sshsrv.NewSSHServer(cfg, srv) if err != nil { return nil, err } - srv.GitDaemon, err = NewGitDaemon(cfg) + srv.GitDaemon, err = daemon.NewGitDaemon(cfg) if err != nil { return nil, err } - srv.HTTPServer, err = NewHTTPServer(cfg) + srv.HTTPServer, err = web.NewHTTPServer(cfg) if err != nil { return nil, err } - srv.StatsServer, err = NewStatsServer(cfg) + srv.StatsServer, err = stats.NewStatsServer(cfg) if err != nil { return nil, err } @@ -124,7 +128,7 @@ func (s *Server) Start() error { errg, ctx := errgroup.WithContext(s.ctx) errg.Go(func() error { logger.Print("Starting Git daemon", "addr", s.Config.Git.ListenAddr) - if err := start(ctx, s.GitDaemon.Start); !errors.Is(err, ErrServerClosed) { + if err := start(ctx, s.GitDaemon.Start); !errors.Is(err, daemon.ErrServerClosed) { return err } return nil diff --git a/server/server_test.go b/server/server_test.go index 3992794e936426dcce2a477921552811666512bc..55c614bf8f39ca669561f4b4c399bc66e2382644 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -3,34 +3,28 @@ package server import ( "context" "fmt" - "net" "path/filepath" "strings" "testing" "github.com/charmbracelet/keygen" "github.com/charmbracelet/soft-serve/server/config" + "github.com/charmbracelet/soft-serve/server/test" "github.com/charmbracelet/ssh" "github.com/matryer/is" gossh "golang.org/x/crypto/ssh" ) -func randomPort() int { - addr, _ := net.Listen("tcp", ":0") //nolint:gosec - _ = addr.Close() - return addr.Addr().(*net.TCPAddr).Port -} - func setupServer(tb testing.TB) (*Server, *config.Config, string) { tb.Helper() tb.Log("creating keypair") pub, pkPath := createKeyPair(tb) dp := tb.TempDir() - sshPort := fmt.Sprintf(":%d", randomPort()) + sshPort := fmt.Sprintf(":%d", test.RandomPort()) tb.Setenv("SOFT_SERVE_DATA_PATH", dp) tb.Setenv("SOFT_SERVE_INITIAL_ADMIN_KEY", authorizedKey(pub)) tb.Setenv("SOFT_SERVE_SSH_LISTEN_ADDR", sshPort) - tb.Setenv("SOFT_SERVE_GIT_LISTEN_ADDR", fmt.Sprintf(":%d", randomPort())) + tb.Setenv("SOFT_SERVE_GIT_LISTEN_ADDR", fmt.Sprintf(":%d", test.RandomPort())) cfg := config.DefaultConfig() tb.Log("configuring server") ctx := context.TODO() diff --git a/server/session.go b/server/ssh/session.go similarity index 99% rename from server/session.go rename to server/ssh/session.go index cd7ddd272c08f2264666d761ac0ae6c92e649344..754fdbe68aeaeef79c379127339da9450d51d509 100644 --- a/server/session.go +++ b/server/ssh/session.go @@ -1,4 +1,4 @@ -package server +package ssh import ( "fmt" diff --git a/server/session_test.go b/server/ssh/session_test.go similarity index 95% rename from server/session_test.go rename to server/ssh/session_test.go index 673e85a6099668009d6499ee4bfe70c812507214..a39d500d25823a1b07e496ea75f111076f209a17 100644 --- a/server/session_test.go +++ b/server/ssh/session_test.go @@ -1,4 +1,4 @@ -package server +package ssh import ( "context" @@ -11,6 +11,7 @@ import ( "github.com/charmbracelet/soft-serve/server/backend/sqlite" "github.com/charmbracelet/soft-serve/server/config" + "github.com/charmbracelet/soft-serve/server/test" "github.com/charmbracelet/ssh" bm "github.com/charmbracelet/wish/bubbletea" "github.com/charmbracelet/wish/testsession" @@ -49,7 +50,7 @@ func setup(tb testing.TB) (*gossh.Session, func() error) { dp := tb.TempDir() is.NoErr(os.Setenv("SOFT_SERVE_DATA_PATH", dp)) is.NoErr(os.Setenv("SOFT_SERVE_GIT_LISTEN_ADDR", ":9418")) - is.NoErr(os.Setenv("SOFT_SERVE_SSH_LISTEN_ADDR", fmt.Sprintf(":%d", randomPort()))) + is.NoErr(os.Setenv("SOFT_SERVE_SSH_LISTEN_ADDR", fmt.Sprintf(":%d", test.RandomPort()))) tb.Cleanup(func() { is.NoErr(os.Unsetenv("SOFT_SERVE_DATA_PATH")) is.NoErr(os.Unsetenv("SOFT_SERVE_GIT_LISTEN_ADDR")) diff --git a/server/ssh.go b/server/ssh/ssh.go similarity index 90% rename from server/ssh.go rename to server/ssh/ssh.go index c99e41bf11b923a218d0ad43daf2a21ebc0e40a1..e3156e6e4974746925760be34892e255db37e715 100644 --- a/server/ssh.go +++ b/server/ssh/ssh.go @@ -1,4 +1,4 @@ -package server +package ssh import ( "context" @@ -13,6 +13,7 @@ import ( "github.com/charmbracelet/soft-serve/server/backend" cm "github.com/charmbracelet/soft-serve/server/cmd" "github.com/charmbracelet/soft-serve/server/config" + "github.com/charmbracelet/soft-serve/server/git" "github.com/charmbracelet/soft-serve/server/hooks" "github.com/charmbracelet/soft-serve/server/utils" "github.com/charmbracelet/ssh" @@ -26,6 +27,10 @@ import ( gossh "golang.org/x/crypto/ssh" ) +var ( + logger = log.WithPrefix("server.ssh") +) + var ( publicKeyCounter = promauto.NewCounterVec(prometheus.CounterOpts{ Namespace: "soft_serve", @@ -187,7 +192,7 @@ func (s *SSHServer) Middleware(cfg *config.Config) wish.Middleware { // https://git-scm.com/docs/gitrepository-layout repo := name + ".git" reposDir := filepath.Join(cfg.DataPath, "repos") - if err := ensureWithin(reposDir, repo); err != nil { + if err := git.EnsureWithin(reposDir, repo); err != nil { sshFatal(s, err) return } @@ -195,9 +200,9 @@ func (s *SSHServer) Middleware(cfg *config.Config) wish.Middleware { logger.Debug("git middleware", "cmd", gc, "access", access.String()) repoDir := filepath.Join(reposDir, repo) switch gc { - case receivePackBin: + case git.ReceivePackBin: if access < backend.ReadWriteAccess { - sshFatal(s, ErrNotAuthed) + sshFatal(s, git.ErrNotAuthed) return } if _, err := cfg.Backend.Repository(name); err != nil { @@ -208,29 +213,29 @@ func (s *SSHServer) Middleware(cfg *config.Config) wish.Middleware { } createRepoCounter.WithLabelValues(ak, s.User(), name).Inc() } - if err := receivePack(s, s, s.Stderr(), repoDir); err != nil { - sshFatal(s, ErrSystemMalfunction) + if err := git.ReceivePack(s, s, s.Stderr(), repoDir); err != nil { + sshFatal(s, git.ErrSystemMalfunction) } receivePackCounter.WithLabelValues(ak, s.User(), name).Inc() return - case uploadPackBin, uploadArchiveBin: + case git.UploadPackBin, git.UploadArchiveBin: if access < backend.ReadOnlyAccess { - sshFatal(s, ErrNotAuthed) + sshFatal(s, git.ErrNotAuthed) return } - gitPack := uploadPack + gitPack := git.UploadPack counter := uploadPackCounter - if gc == uploadArchiveBin { - gitPack = uploadArchive + if gc == git.UploadArchiveBin { + gitPack = git.UploadArchive counter = uploadArchiveCounter } err := gitPack(s, s, s.Stderr(), repoDir) - if errors.Is(err, ErrInvalidRepo) { - sshFatal(s, ErrInvalidRepo) + if errors.Is(err, git.ErrInvalidRepo) { + sshFatal(s, git.ErrInvalidRepo) } else if err != nil { - sshFatal(s, ErrSystemMalfunction) + sshFatal(s, git.ErrSystemMalfunction) } counter.WithLabelValues(ak, s.User(), name).Inc() @@ -244,6 +249,6 @@ func (s *SSHServer) Middleware(cfg *config.Config) wish.Middleware { // sshFatal prints to the session's STDOUT as a git response and exit 1. func sshFatal(s ssh.Session, v ...interface{}) { - writePktline(s, v...) + git.WritePktline(s, v...) s.Exit(1) // nolint: errcheck } diff --git a/server/stats.go b/server/stats/stats.go similarity index 98% rename from server/stats.go rename to server/stats/stats.go index 30b8cd0bef9ba125c293b7bd7a6d5b72e882bee0..d0029d65aed0b1d0d68aa7e4d625a10838b4d512 100644 --- a/server/stats.go +++ b/server/stats/stats.go @@ -1,4 +1,4 @@ -package server +package stats import ( "context" diff --git a/server/test/test.go b/server/test/test.go new file mode 100644 index 0000000000000000000000000000000000000000..2dfe85c97dea227186565a04ef3ad86b31e120f8 --- /dev/null +++ b/server/test/test.go @@ -0,0 +1,9 @@ +package test + +import "net" + +func RandomPort() int { + addr, _ := net.Listen("tcp", ":0") //nolint:gosec + _ = addr.Close() + return addr.Addr().(*net.TCPAddr).Port +} diff --git a/server/http.go b/server/web/http.go similarity index 98% rename from server/http.go rename to server/web/http.go index 60a8650c76adc523fd0963eb74f675f308ee8c24..694f58807f0750ea75ff84ed2637c0f977ef00ae 100644 --- a/server/http.go +++ b/server/web/http.go @@ -1,4 +1,4 @@ -package server +package web import ( "context" @@ -12,6 +12,7 @@ import ( "text/template" "time" + "github.com/charmbracelet/log" "github.com/charmbracelet/soft-serve/server/backend" "github.com/charmbracelet/soft-serve/server/config" "github.com/charmbracelet/soft-serve/server/utils" @@ -23,6 +24,10 @@ import ( "goji.io/pattern" ) +var ( + logger = log.WithPrefix("server.web") +) + var ( gitHttpCounter = promauto.NewCounterVec(prometheus.CounterOpts{ Namespace: "soft_serve",