From d0e496b041755ef61c8fb6cdf178b33aad64d13e Mon Sep 17 00:00:00 2001 From: Ayman Bagabas Date: Wed, 17 May 2023 13:12:08 -0700 Subject: [PATCH] feat: abstract config to accommodate for external config * refactor: tidy up server git use git services to implement handling git server commands pass config to git as environment variables * feat(git): enable partial clones * feat(server): use smart http git backend This implements the smart http git protocol which also supports git-receive-pack service. * fix(config): write config file when instructed to * feat: add a cache and implement a default lru policy * feat: abstract config to accommodate for external config --- cmd/soft/hook.go | 2 +- cmd/soft/migrate_config.go | 2 +- cmd/soft/serve.go | 8 +- internal/log/log.go | 14 +-- server/config/config.go | 171 ++++++++++++++++++++--------------- server/config/config_test.go | 7 +- server/config/file.go | 4 + 7 files changed, 116 insertions(+), 92 deletions(-) diff --git a/cmd/soft/hook.go b/cmd/soft/hook.go index c1f9d4233ab56846043584f052c1195bb235cffa..fc88e7f80714e72cb94e46689af703d960872826 100644 --- a/cmd/soft/hook.go +++ b/cmd/soft/hook.go @@ -31,7 +31,7 @@ var ( Hidden: true, PersistentPreRunE: func(cmd *cobra.Command, _ []string) error { ctx := cmd.Context() - cfg, err := config.ParseConfig(configPath) + cfg, err := config.NewConfig(configPath) if err != nil { return fmt.Errorf("could not parse config: %w", err) } diff --git a/cmd/soft/migrate_config.go b/cmd/soft/migrate_config.go index c0fd101fb03bb9b5d9ea639902996d63aaeb0d29..cd3b23ab39f69d05a12cac85a825053143370d10 100644 --- a/cmd/soft/migrate_config.go +++ b/cmd/soft/migrate_config.go @@ -316,7 +316,7 @@ var ( logger.Info("Writing config...") defer logger.Info("Done!") - return config.WriteConfig(filepath.Join(cfg.DataPath, "config.yaml"), cfg) + return cfg.WriteConfig() }, } ) diff --git a/cmd/soft/serve.go b/cmd/soft/serve.go index b2a4746f9a2af69cb68dc96d796aa6cf63929189..5af4ddd8e2c80687e01c094e877c2874c3c9a88e 100644 --- a/cmd/soft/serve.go +++ b/cmd/soft/serve.go @@ -24,11 +24,9 @@ var ( ctx := cmd.Context() // Set up config - cfg := config.DefaultConfig() - if !cfg.Exist() { - if err := cfg.WriteConfig(); err != nil { - return fmt.Errorf("failed to write default config: %w", err) - } + cfg, err := config.NewConfig("") + if err != nil { + return err } ctx = config.WithContext(ctx, cfg) diff --git a/internal/log/log.go b/internal/log/log.go index b6c4b1443d19a5bda654a22428efb1fb70cbe16d..6defed7b20b382884c97ec46ee900f17fcceb48e 100644 --- a/internal/log/log.go +++ b/internal/log/log.go @@ -2,7 +2,6 @@ package log import ( "os" - "path/filepath" "strconv" "strings" "time" @@ -15,14 +14,11 @@ var contextKey = &struct{ string }{"logger"} // NewDefaultLogger returns a new logger with default settings. func NewDefaultLogger() *log.Logger { - dp := os.Getenv("SOFT_SERVE_DATA_PATH") - if dp == "" { - dp = "data" - } - - cfg, err := config.ParseConfig(filepath.Join(dp, "config.yaml")) - if err != nil { - log.Errorf("failed to parse config: %v", err) + cfg := config.DefaultConfig() + if cfg.Exist() { + if err := config.ParseConfig(cfg, cfg.FilePath()); err != nil { + log.Errorf("failed to parse config: %v", err) + } } logger := log.NewWithOptions(os.Stderr, log.Options{ diff --git a/server/config/config.go b/server/config/config.go index c16b237de9d83ef81a6883fe5221932edc23b94e..dfac36a3f6d07c10866136b959b33140a6c880dc 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -10,7 +10,6 @@ import ( "time" "github.com/caarlos0/env/v8" - "github.com/charmbracelet/log" "github.com/charmbracelet/soft-serve/server/backend" "golang.org/x/crypto/ssh" "gopkg.in/yaml.v3" @@ -104,6 +103,9 @@ type Config struct { // Log is the logger configuration. Log LogConfig `envPrefix:"LOG_" yaml:"log"` + // Cache is the cache backend to use. + Cache string `env:"CACHE" yaml:"cache"` + // InitialAdminKeys is a list of public keys that will be added to the list of admins. InitialAdminKeys []string `env:"INITIAL_ADMIN_KEYS" envSeparator:"\n" yaml:"initial_admin_keys"` @@ -148,54 +150,65 @@ func (c *Config) Environ() []string { return envs } -func parseConfig(path string) (*Config, error) { - dataPath := filepath.Dir(path) - cfg := &Config{ - Name: "Soft Serve", - DataPath: dataPath, - SSH: SSHConfig{ - ListenAddr: ":23231", - PublicURL: "ssh://localhost:23231", - KeyPath: filepath.Join("ssh", "soft_serve_host_ed25519"), - ClientKeyPath: filepath.Join("ssh", "soft_serve_client_ed25519"), - MaxTimeout: 0, - IdleTimeout: 0, - }, - Git: GitConfig{ - ListenAddr: ":9418", - MaxTimeout: 0, - IdleTimeout: 3, - MaxConnections: 32, - }, - HTTP: HTTPConfig{ - ListenAddr: ":23232", - PublicURL: "http://localhost:23232", - }, - Stats: StatsConfig{ - ListenAddr: "localhost:23233", - }, - Log: LogConfig{ - Format: "text", - TimeFormat: time.DateTime, - }, +func parseFile(v interface{}, path string) error { + f, err := os.Open(path) + if err != nil { + return fmt.Errorf("open config file: %w", err) } - f, err := os.Open(path) - if err == nil { - defer f.Close() // nolint: errcheck - if err := yaml.NewDecoder(f).Decode(cfg); err != nil { - return cfg, fmt.Errorf("decode config: %w", err) - } + defer f.Close() // nolint: errcheck + if err := yaml.NewDecoder(f).Decode(v); err != nil { + return fmt.Errorf("decode config: %w", err) } - // Merge initial admin keys from both config file and environment variables. - initialAdminKeys := append([]string{}, cfg.InitialAdminKeys...) + return nil +} +func parseEnv(v interface{}) error { // Override with environment variables - if err := env.ParseWithOptions(cfg, env.Options{ + if err := env.ParseWithOptions(v, env.Options{ Prefix: "SOFT_SERVE_", }); err != nil { - return cfg, fmt.Errorf("parse environment variables: %w", err) + return fmt.Errorf("parse environment variables: %w", err) + } + + return nil +} + +// ParseConfig parses the configuration from environment variables the given +// file. +func ParseConfig(v interface{}, path string) error { + return errors.Join(parseFile(v, path), parseEnv(v)) +} + +// NewConfig retruns a new Config with values populated from environment +// variables and config file. +// +// If the config file does not exist, it will be created with the default +// values. +// +// Environment variables will override values in the config file except for the +// initial_admin_keys. +// +// If path is empty, the default config file path will be used. +func NewConfig(path string) (*Config, error) { + cfg := DefaultConfig() + if path != "" { + cfg.DataPath = filepath.Dir(path) + } + + // Parse file + if cfg.Exist() { + if err := parseFile(cfg, cfg.FilePath()); err != nil { + return cfg, err + } + } + + // Merge initial admin keys from both config file and environment variables. + initialAdminKeys := append([]string{}, cfg.InitialAdminKeys...) + + if err := parseEnv(cfg); err != nil { + return cfg, err } // Merge initial admin keys from environment variables. @@ -215,66 +228,78 @@ func parseConfig(path string) (*Config, error) { // Reset datapath to config dir. // This is necessary because the environment variable may be set to // a different directory. - cfg.DataPath = dataPath - - return cfg, nil -} - -// ParseConfig parses the configuration from the given file. -func ParseConfig(path string) (*Config, error) { - cfg, err := parseConfig(path) - if err != nil { - return cfg, err - } + // cfg.DataPath = dataPath if err := cfg.validate(); err != nil { return cfg, err } - return cfg, nil + return cfg, cfg.WriteConfig() } -// WriteConfig writes the configuration to the given file. -func WriteConfig(path string, cfg *Config) error { - if err := os.MkdirAll(filepath.Dir(path), os.ModePerm); err != nil { - return err - } - return os.WriteFile(path, []byte(newConfigFile(cfg)), 0o644) // nolint: errcheck -} - -// DefaultConfig returns a Config with the values populated with the defaults -// or specified environment variables. +// DefaultConfig returns a Config with the default values. func DefaultConfig() *Config { dataPath := os.Getenv("SOFT_SERVE_DATA_PATH") if dataPath == "" { dataPath = "data" } - cp := filepath.Join(dataPath, "config.yaml") - cfg, err := parseConfig(cp) - if err != nil && !errors.Is(err, os.ErrNotExist) { - log.Errorf("failed to parse config: %v", err) - } - - if err := cfg.validate(); err != nil { - log.Fatal(err) + cfg := &Config{ + Name: "Soft Serve", + DataPath: dataPath, + Cache: "lru", + SSH: SSHConfig{ + ListenAddr: ":23231", + PublicURL: "ssh://localhost:23231", + KeyPath: filepath.Join("ssh", "soft_serve_host_ed25519"), + ClientKeyPath: filepath.Join("ssh", "soft_serve_client_ed25519"), + MaxTimeout: 0, + IdleTimeout: 0, + }, + Git: GitConfig{ + ListenAddr: ":9418", + MaxTimeout: 0, + IdleTimeout: 3, + MaxConnections: 32, + }, + HTTP: HTTPConfig{ + ListenAddr: ":23232", + PublicURL: "http://localhost:23232", + }, + Stats: StatsConfig{ + ListenAddr: "localhost:23233", + }, + Log: LogConfig{ + Format: "text", + TimeFormat: time.DateTime, + }, } return cfg } +// FilePath returns the expected config file path. +func (c *Config) FilePath() string { + return filepath.Join(c.DataPath, "config.yaml") +} + // Exist returns true if the configuration file exists. func (c *Config) Exist() bool { - _, err := os.Stat(filepath.Join(c.DataPath, "config.yaml")) + _, err := os.Stat(c.FilePath()) return err == nil } // WriteConfig writes the configuration in the default path. func (c *Config) WriteConfig() error { - return WriteConfig(filepath.Join(c.DataPath, "config.yaml"), c) + fp := c.FilePath() + if err := os.MkdirAll(filepath.Dir(fp), os.ModePerm); err != nil { + return err + } + return os.WriteFile(fp, []byte(newConfigFile(c)), 0o644) // nolint: errcheck } // WithBackend sets the backend for the configuration. +// TODO: remove in favor of backend.FromContext. func (c *Config) WithBackend(backend backend.Backend) *Config { c.Backend = backend return c diff --git a/server/config/config_test.go b/server/config/config_test.go index 3812e4f9e72b871f3611b53ee2a1db1a12c803cd..fa933de244b02477c3a25faa7811b7c2f24b5ea5 100644 --- a/server/config/config_test.go +++ b/server/config/config_test.go @@ -18,7 +18,8 @@ func TestParseMultipleKeys(t *testing.T) { is.NoErr(os.Unsetenv("SOFT_SERVE_INITIAL_ADMIN_KEYS")) is.NoErr(os.Unsetenv("SOFT_SERVE_DATA_PATH")) }) - cfg := DefaultConfig() + cfg, err := NewConfig("") + is.NoErr(err) is.Equal(cfg.InitialAdminKeys, []string{ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINMwLvyV3ouVrTysUYGoJdl5Vgn5BACKov+n9PlzfPwH", "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFxIobhwtfdwN7m1TFt9wx3PsfvcAkISGPxmbmbauST8", @@ -36,7 +37,7 @@ func TestMergeInitAdminKeys(t *testing.T) { fp := filepath.Join(t.TempDir(), "config.yaml") err = os.WriteFile(fp, bts, 0o644) is.NoErr(err) - cfg, err := ParseConfig(fp) + cfg, err := NewConfig(fp) is.NoErr(err) is.Equal(cfg.InitialAdminKeys, []string{ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINMwLvyV3ouVrTysUYGoJdl5Vgn5BACKov+n9PlzfPwH", @@ -57,7 +58,7 @@ func TestValidateInitAdminKeys(t *testing.T) { fp := filepath.Join(t.TempDir(), "config.yaml") err = os.WriteFile(fp, bts, 0o644) is.NoErr(err) - cfg, err := ParseConfig(fp) + cfg, err := NewConfig(fp) is.NoErr(err) is.Equal(cfg.InitialAdminKeys, []string{ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINMwLvyV3ouVrTysUYGoJdl5Vgn5BACKov+n9PlzfPwH", diff --git a/server/config/file.go b/server/config/file.go index 09e5ce2e00dec68891f27e63bd5cf14fa1faca2c..a285cc6e4078b31090570f18e048b2743133a09d 100644 --- a/server/config/file.go +++ b/server/config/file.go @@ -11,6 +11,10 @@ var configFileTmpl = template.Must(template.New("config").Parse(`# Soft Serve Se # This is the name that will be displayed in the UI. name: "{{ .Name }}" +# Cache configuration. +# The cache backend to use. The default backend is "lru" memory cache. +cache: "{{ .Cache }}" + # Logging configuration. log: # Log format to use. Valid values are "json", "logfmt", and "text".