diff --git a/server/backend/sqlite/sqlite.go b/server/backend/sqlite/sqlite.go
index 1d07652d26b218a656a531115251abb962528d92..7ee17dfc404247f8f47ad856fa6e01cfb69484fb 100644
--- a/server/backend/sqlite/sqlite.go
+++ b/server/backend/sqlite/sqlite.go
@@ -26,10 +26,9 @@ var (
 // SqliteBackend is a backend that uses a SQLite database as a Soft Serve
 // backend.
 type SqliteBackend struct {
-	cfg              *config.Config
-	dp               string
-	db               *sqlx.DB
-	AdditionalAdmins []string
+	cfg *config.Config
+	dp  string
+	db  *sqlx.DB
 }
 
 var _ backend.Backend = (*SqliteBackend)(nil)
diff --git a/server/backend/sqlite/user.go b/server/backend/sqlite/user.go
index 068c85a3a2b39251b069696c3505eaca4b2659a8..0e3c609caf9b9f4ed131dc181124d49ed9e2d50a 100644
--- a/server/backend/sqlite/user.go
+++ b/server/backend/sqlite/user.go
@@ -120,7 +120,10 @@ func (d *SqliteBackend) AccessLevel(repo string, username string) backend.Access
 // It implements backend.Backend.
 func (d *SqliteBackend) AccessLevelByPublicKey(repo string, pk ssh.PublicKey) backend.AccessLevel {
 	ak := backend.MarshalAuthorizedKey(pk)
-	for _, k := range d.AdditionalAdmins {
+	if strings.HasPrefix(d.cfg.InternalPublicKey, ak) {
+		return backend.AdminAccess
+	}
+	for _, k := range d.cfg.InitialAdminKeys {
 		if k == ak {
 			return backend.AdminAccess
 		}
diff --git a/server/config/config.go b/server/config/config.go
index 79ab416df3d5ccc962d3d1f4e5d9d5d1bc0f52df..b0112d93b8b53d1bfa90a59a1d57de1bab559a0f 100644
--- a/server/config/config.go
+++ b/server/config/config.go
@@ -95,6 +95,12 @@ type Config struct {
 
 	// Backend is the Git backend to use.
 	Backend backend.Backend `yaml:"-"`
+
+	// InternalPublicKey is the public key of the internal SSH key.
+	InternalPublicKey string `yaml:"-"`
+
+	// ClientPublicKey is the public key of the client SSH key.
+	ClientPublicKey string `yaml:"-"`
 }
 
 // ParseConfig parses the configuration from the given file.
diff --git a/server/server.go b/server/server.go
index 992e0c7c13c0a656afd08a3c764481fe393d9b5a..beee1fa744ecc660e0274d5858750ef1e26aa945 100644
--- a/server/server.go
+++ b/server/server.go
@@ -45,12 +45,10 @@ func NewServer(cfg *config.Config) (*Server, error) {
 			logger.Fatal(err)
 		}
 
-		// Add the initial admin keys to the list of admins.
-		sb.AdditionalAdmins = cfg.InitialAdminKeys
 		cfg = cfg.WithBackend(sb)
 
 		// Create internal key.
-		_, err = keygen.NewWithWrite(
+		ikp, err := keygen.NewWithWrite(
 			filepath.Join(cfg.DataPath, cfg.SSH.InternalKeyPath),
 			nil,
 			keygen.Ed25519,
@@ -58,9 +56,10 @@ func NewServer(cfg *config.Config) (*Server, error) {
 		if err != nil {
 			return nil, err
 		}
+		cfg.InternalPublicKey = string(ikp.PublicKey())
 
 		// Create client key.
-		_, err = keygen.NewWithWrite(
+		ckp, err := keygen.NewWithWrite(
 			filepath.Join(cfg.DataPath, cfg.SSH.ClientKeyPath),
 			nil,
 			keygen.Ed25519,
@@ -68,6 +67,7 @@ func NewServer(cfg *config.Config) (*Server, error) {
 		if err != nil {
 			return nil, err
 		}
+		cfg.ClientPublicKey = string(ckp.PublicKey())
 	}
 
 	srv := &Server{
diff --git a/server/ssh.go b/server/ssh.go
index 9f6da44cfe8005f27db0ab4da3b97dda2a7e80f8..30ea8e498c24245a116733f7d6ca21280a30c6dd 100644
--- a/server/ssh.go
+++ b/server/ssh.go
@@ -148,13 +148,9 @@ func (s *SSHServer) PublicKeyHandler(ctx ssh.Context, pk ssh.PublicKey) (allowed
 		}
 	}
 
-	user, _ := s.cfg.Backend.UserByPublicKey(pk)
-	if user == nil {
-		logger.Debug("public key auth user not found")
-		return s.cfg.Backend.AnonAccess() >= backend.ReadOnlyAccess
-	}
-
-	allowed = s.cfg.Backend.AccessLevel("", user.Username()) >= backend.ReadOnlyAccess
+	ac := s.cfg.Backend.AccessLevelByPublicKey("", pk)
+	logger.Debugf("access level for %s: %d", ak, ac)
+	allowed = ac >= backend.ReadOnlyAccess
 	return
 }
 
@@ -191,6 +187,7 @@ func (s *SSHServer) Middleware(cfg *config.Config) wish.Middleware {
 						return
 					}
 
+					logger.Debug("git middleware", "cmd", gc, "access", access.String())
 					repoDir := filepath.Join(reposDir, repo)
 					switch gc {
 					case receivePackBin: