diff --git a/internal/ui/lazylist/list.go b/internal/ui/lazylist/list.go
index 0cb6755681e7e56e7236bfca6ebf7dd2cfe41928..e5478b85ac2b46f01ec79b1433cabad2d5062163 100644
--- a/internal/ui/lazylist/list.go
+++ b/internal/ui/lazylist/list.go
@@ -259,25 +259,14 @@ func (l *List) ScrollBy(lines int) {
 		}
 	} else if lines < 0 {
 		// Scroll up
-		// Calculate from offset how many items needed to fill the viewport
-		// This is needed to know when to stop scrolling up
-		var totalLines int
-		var firstItemIdx int
-		for i := l.offsetIdx; i >= 0; i-- {
-			item := l.getItem(i)
-			totalLines += item.height
-			if l.gap > 0 && i < l.offsetIdx {
-				totalLines += l.gap
-			}
-			if totalLines >= l.height {
-				firstItemIdx = i
+		l.offsetLine += lines // lines is negative
+		for l.offsetLine < 0 {
+			if l.offsetIdx <= 0 {
+				// Reached top
+				l.ScrollToTop()
 				break
 			}
-		}
 
-		// Now scroll up by lines
-		l.offsetLine += lines // lines is negative
-		for l.offsetIdx > firstItemIdx && l.offsetLine < 0 {
 			// Move to previous item
 			l.offsetIdx--
 			prevItem := l.getItem(l.offsetIdx)
@@ -287,10 +276,6 @@ func (l *List) ScrollBy(lines int) {
 			}
 			l.offsetLine += totalHeight
 		}
-
-		if l.offsetLine < 0 {
-			l.offsetLine = 0
-		}
 	}
 }