diff --git a/server/backend/sqlite/user.go b/server/backend/sqlite/user.go
index 0e3c609caf9b9f4ed131dc181124d49ed9e2d50a..86d36ac1e84d443910e7a5dfe0714f57f416777d 100644
--- a/server/backend/sqlite/user.go
+++ b/server/backend/sqlite/user.go
@@ -101,7 +101,6 @@ func (d *SqliteBackend) AccessLevel(repo string, username string) backend.Access
 		return backend.ReadOnlyAccess
 	}
 
-	// If the repository doesn't exist, the user has read/write access.
 	if user != nil {
 		// If the repository doesn't exist, the user has read/write access.
 		if anon > backend.ReadWriteAccess {
diff --git a/server/daemon.go b/server/daemon.go
index 007e3c68e3f19930471ca9afe06964e93cf90642..79b94d88418307fedd16858afa6c13842067b4f9 100644
--- a/server/daemon.go
+++ b/server/daemon.go
@@ -221,6 +221,11 @@ func (d *GitDaemon) handleClient(conn net.Conn) {
 			return
 		}
 
+		if !d.cfg.Backend.AllowKeyless() {
+			fatal(c, ErrNotAuthed)
+			return
+		}
+
 		name := utils.SanitizeRepo(string(opts[0]))
 		logger.Debugf("git: connect %s %s %s", c.RemoteAddr(), cmd, name)
 		defer logger.Debugf("git: disconnect %s %s %s", c.RemoteAddr(), cmd, name)
diff --git a/server/ssh.go b/server/ssh.go
index 30ea8e498c24245a116733f7d6ca21280a30c6dd..c99e41bf11b923a218d0ad43daf2a21ebc0e40a1 100644
--- a/server/ssh.go
+++ b/server/ssh.go
@@ -137,10 +137,15 @@ func (s *SSHServer) Shutdown(ctx context.Context) error {
 
 // PublicKeyAuthHandler handles public key authentication.
 func (s *SSHServer) PublicKeyHandler(ctx ssh.Context, pk ssh.PublicKey) (allowed bool) {
+	if pk == nil {
+		return s.cfg.Backend.AllowKeyless()
+	}
+
 	ak := backend.MarshalAuthorizedKey(pk)
 	defer func() {
 		publicKeyCounter.WithLabelValues(ak, ctx.User(), strconv.FormatBool(allowed)).Inc()
 	}()
+
 	for _, k := range s.cfg.InitialAdminKeys {
 		if k == ak {
 			allowed = true
@@ -156,7 +161,7 @@ func (s *SSHServer) PublicKeyHandler(ctx ssh.Context, pk ssh.PublicKey) (allowed
 
 // KeyboardInteractiveHandler handles keyboard interactive authentication.
 func (s *SSHServer) KeyboardInteractiveHandler(ctx ssh.Context, _ gossh.KeyboardInteractiveChallenge) bool {
-	ac := s.cfg.Backend.AllowKeyless() && s.PublicKeyHandler(ctx, nil)
+	ac := s.cfg.Backend.AllowKeyless()
 	keyboardInteractiveCounter.WithLabelValues(ctx.User(), strconv.FormatBool(ac)).Inc()
 	return ac
 }