e2e_test.go

  1package server
  2
  3import (
  4	"bufio"
  5	"bytes"
  6	"context"
  7	"encoding/json"
  8	"io"
  9	"net/http"
 10	"net/http/httptest"
 11	"net/url"
 12	"sync"
 13	"sync/atomic"
 14	"testing"
 15	"time"
 16
 17	"github.com/charmbracelet/crush/internal/app"
 18	"github.com/charmbracelet/crush/internal/backend"
 19	"github.com/charmbracelet/crush/internal/db"
 20	"github.com/charmbracelet/crush/internal/message"
 21	"github.com/charmbracelet/crush/internal/permission"
 22	"github.com/charmbracelet/crush/internal/proto"
 23	"github.com/charmbracelet/crush/internal/pubsub"
 24	"github.com/google/uuid"
 25	"github.com/stretchr/testify/require"
 26)
 27
 28// e2eHarness wires a Server, its Backend (with a custom shutdownFn we
 29// can observe), an httptest.NewServer, and a synthetic Workspace whose
 30// embedded App has a live event broker. It is the minimum scaffolding
 31// the multi-client end-to-end scenarios in PLAN item 6 need.
 32type e2eHarness struct {
 33	httpSrv     *httptest.Server
 34	srv         *Server
 35	backend     *backend.Backend
 36	workspace   *backend.Workspace
 37	app         *app.App
 38	shutdownHit atomic.Bool
 39
 40	// sseWG tracks every SSE reader goroutine spawned by
 41	// [e2eHarness.subscribeSSE]. The harness's cleanup hook waits on
 42	// it after the httptest server has been closed so that the test
 43	// cannot leave behind background readers (and therefore unclosed
 44	// response bodies) after returning.
 45	sseWG sync.WaitGroup
 46}
 47
 48// installServer attaches a fresh Server (with a custom shutdown
 49// callback that flips [e2eHarness.shutdownHit]) wrapped in an
 50// [httptest.Server] onto h. It registers the cleanup hooks for the
 51// httptest server and the SSE reader WaitGroup in the order required
 52// by the LIFO contract documented on [newE2EHarness].
 53//
 54// Callers that want a fully synthetic workspace use [newE2EHarness];
 55// callers that want to drive the real CreateWorkspace HTTP path use
 56// [newRealCreateHarness] and then [e2eHarness.postWorkspace].
 57func (h *e2eHarness) installServer(t *testing.T) {
 58	t.Helper()
 59	srv := &Server{}
 60	srv.backend = backend.New(context.Background(), nil, func() {
 61		h.shutdownHit.Store(true)
 62	})
 63	srv.installHandler()
 64
 65	hs := httptest.NewServer(srv.Handler())
 66	// Order matters: t.Cleanup is LIFO and the test's own per-
 67	// stream cancels (cancelA/cancelB) run first. After those, we
 68	// want hs.Close to fire first (so any handler still parked in
 69	// its `select` returns), THEN sseWG.Wait so every reader
 70	// goroutine exits and closes its response body. Any caller-
 71	// owned cleanups registered *before* installServer (e.g. App
 72	// teardown for the synthetic harness) therefore run LAST,
 73	// after the readers have drained.
 74	t.Cleanup(h.sseWG.Wait)
 75	t.Cleanup(hs.Close)
 76
 77	h.httpSrv = hs
 78	h.srv = srv
 79	h.backend = srv.backend
 80}
 81
 82// newE2EHarness builds an in-process server + a synthetic Workspace
 83// whose embedded App is a real [app.App] constructed via
 84// [app.NewForTest], so its event broker delivers everything the SSE
 85// pipeline expects. Used by the scenarios that do not need to
 86// exercise the path-dedupe behavior of [backend.CreateWorkspace].
 87//
 88// Cleanup tears down the App's broker only after sseWG.Wait and
 89// hs.Close have run, so SSE readers cannot observe a dead broker.
 90func newE2EHarness(t *testing.T) *e2eHarness {
 91	t.Helper()
 92
 93	h := &e2eHarness{}
 94
 95	// Register the App teardown FIRST so LIFO order puts it AFTER
 96	// the cleanups that installServer registers below (hs.Close +
 97	// sseWG.Wait).
 98	appCtx, cancel := context.WithCancel(context.Background())
 99	a := app.NewForTest(appCtx)
100	t.Cleanup(func() {
101		cancel()
102		a.ShutdownForTest()
103	})
104
105	h.installServer(t)
106
107	ws := &backend.Workspace{
108		ID:   uuid.New().String(),
109		Path: t.TempDir(),
110		App:  a,
111	}
112	// Synthetic workspaces have an incomplete App; bypass the
113	// default teardown so the "last workspace removed" path can run
114	// without panicking inside [app.App.Shutdown].
115	backend.SetWorkspaceShutdownFnForTest(ws, func() {})
116	backend.InsertWorkspaceForTest(h.backend, ws)
117
118	h.workspace = ws
119	h.app = a
120	return h
121}
122
123// newRealCreateHarness builds an in-process server WITHOUT any
124// pre-inserted workspace, intended for tests that drive the real
125// [backend.CreateWorkspace] HTTP path (path-dedupe scenario). It
126// isolates HOME/XDG_* via [t.Setenv] so [config.Init] doesn't read
127// the host machine's config, which means callers MUST NOT mark the
128// test as parallel.
129func newRealCreateHarness(t *testing.T) *e2eHarness {
130	t.Helper()
131	t.Setenv("HOME", t.TempDir())
132	t.Setenv("XDG_CACHE_HOME", t.TempDir())
133	t.Setenv("XDG_CONFIG_HOME", t.TempDir())
134	t.Setenv("XDG_DATA_HOME", t.TempDir())
135
136	h := &e2eHarness{}
137	h.installServer(t)
138	return h
139}
140
141// postWorkspace drives the real POST /v1/workspaces handler and
142// returns the resolved workspace proto. This is how scenario 1
143// exercises the path-dedupe behavior from PLAN item 1: two calls
144// with the same Path and distinct ClientIDs must return the same
145// workspace ID.
146func (h *e2eHarness) postWorkspace(t *testing.T, args proto.Workspace) proto.Workspace {
147	t.Helper()
148	body, err := json.Marshal(args)
149	require.NoError(t, err)
150	req, err := http.NewRequestWithContext(t.Context(), http.MethodPost,
151		h.httpSrv.URL+"/v1/workspaces", bytes.NewReader(body))
152	require.NoError(t, err)
153	req.Header.Set("Content-Type", "application/json")
154	resp, err := h.httpSrv.Client().Do(req)
155	require.NoError(t, err)
156	defer resp.Body.Close()
157	require.Equal(t, http.StatusOK, resp.StatusCode, "POST /v1/workspaces must succeed")
158	var out proto.Workspace
159	require.NoError(t, json.NewDecoder(resp.Body).Decode(&out))
160	require.NotEmpty(t, out.ID, "server must return a workspace id")
161	return out
162}
163
164// subscribeSSE opens an SSE stream against the test server for the
165// given workspace and client ID. It returns a channel of decoded
166// envelopes plus a cancel function that closes the stream. The
167// returned channel is closed when the stream ends.
168func (h *e2eHarness) subscribeSSE(t *testing.T, ctx context.Context, workspaceID, clientID string) (<-chan any, context.CancelFunc) {
169	t.Helper()
170	streamCtx, cancel := context.WithCancel(ctx)
171
172	q := url.Values{"client_id": []string{clientID}}
173	reqURL := h.httpSrv.URL + "/v1/workspaces/" + workspaceID + "/events?" + q.Encode()
174	req, err := http.NewRequestWithContext(streamCtx, http.MethodGet, reqURL, nil)
175	require.NoError(t, err)
176	req.Header.Set("Accept", "text/event-stream")
177
178	resp, err := h.httpSrv.Client().Do(req)
179	require.NoError(t, err)
180	require.Equal(t, http.StatusOK, resp.StatusCode, "SSE subscribe should return 200")
181
182	out := make(chan any, 64)
183	h.sseWG.Go(func() {
184		defer resp.Body.Close()
185		defer close(out)
186		reader := bufio.NewReader(resp.Body)
187		for {
188			line, err := reader.ReadBytes('\n')
189			if err != nil {
190				return
191			}
192			line = bytes.TrimSpace(line)
193			if len(line) == 0 {
194				continue
195			}
196			data, ok := bytes.CutPrefix(line, []byte("data:"))
197			if !ok {
198				continue
199			}
200			data = bytes.TrimSpace(data)
201			var p pubsub.Payload
202			if err := json.Unmarshal(data, &p); err != nil {
203				continue
204			}
205			ev, decoded := decodeSSEEnvelope(p)
206			if !decoded {
207				continue
208			}
209			select {
210			case out <- ev:
211			case <-streamCtx.Done():
212				return
213			}
214		}
215	})
216	return out, cancel
217}
218
219// decodeSSEEnvelope decodes the discriminated SSE envelope into the
220// concrete pubsub.Event[proto.X] payload the e2e tests care about.
221// Unknown payload types are skipped so tests can match on type
222// assertions without worrying about envelope noise.
223func decodeSSEEnvelope(p pubsub.Payload) (any, bool) {
224	switch p.Type {
225	case pubsub.PayloadTypePermissionRequest:
226		var e pubsub.Event[proto.PermissionRequest]
227		if err := json.Unmarshal(p.Payload, &e); err != nil {
228			return nil, false
229		}
230		return e, true
231	case pubsub.PayloadTypePermissionNotification:
232		var e pubsub.Event[proto.PermissionNotification]
233		if err := json.Unmarshal(p.Payload, &e); err != nil {
234			return nil, false
235		}
236		return e, true
237	case pubsub.PayloadTypeMessage:
238		var e pubsub.Event[proto.Message]
239		if err := json.Unmarshal(p.Payload, &e); err != nil {
240			return nil, false
241		}
242		return e, true
243	case pubsub.PayloadTypeAgentEvent:
244		var e pubsub.Event[proto.AgentEvent]
245		if err := json.Unmarshal(p.Payload, &e); err != nil {
246			return nil, false
247		}
248		return e, true
249	case pubsub.PayloadTypeRunComplete:
250		var e pubsub.Event[proto.RunComplete]
251		if err := json.Unmarshal(p.Payload, &e); err != nil {
252			return nil, false
253		}
254		return e, true
255	}
256	return nil, false
257}
258
259// grantPermission posts a permission grant via the HTTP surface and
260// returns the server's "resolved" verdict. Mirrors the client-side
261// GrantPermission flow without importing internal/client (which
262// would create an import cycle from this in-package test).
263func (h *e2eHarness) grantPermission(t *testing.T, ctx context.Context, workspaceID string, req proto.PermissionGrant) bool {
264	t.Helper()
265	body, err := json.Marshal(req)
266	require.NoError(t, err)
267	httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost,
268		h.httpSrv.URL+"/v1/workspaces/"+workspaceID+"/permissions/grant",
269		bytes.NewReader(body))
270	require.NoError(t, err)
271	httpReq.Header.Set("Content-Type", "application/json")
272	resp, err := h.httpSrv.Client().Do(httpReq)
273	require.NoError(t, err)
274	defer resp.Body.Close()
275	require.Equal(t, http.StatusOK, resp.StatusCode)
276	var out proto.PermissionGrantResponse
277	require.NoError(t, json.NewDecoder(resp.Body).Decode(&out))
278	return out.Resolved
279}
280
281// waitForAttached spins until the workspace's clients map reports at
282// least n entries with streams > 0. Catches the race where a test
283// publishes events before the server-side AttachClient has completed.
284func (h *e2eHarness) waitForAttached(t *testing.T, n int) {
285	t.Helper()
286	h.waitForAttachedOn(t, h.workspace, n)
287}
288
289// waitForAttachedOn is the workspace-explicit form of waitForAttached.
290// Tests that drive a workspace whose pointer is not stored on the
291// harness (e.g. the real CreateWorkspace path) pass the workspace in.
292func (h *e2eHarness) waitForAttachedOn(t *testing.T, ws *backend.Workspace, n int) {
293	t.Helper()
294	deadline := time.Now().Add(2 * time.Second)
295	for time.Now().Before(deadline) {
296		if backend.WorkspaceLiveStreamCountForTest(ws) >= n {
297			return
298		}
299		time.Sleep(5 * time.Millisecond)
300	}
301	t.Fatalf("expected %d attached streams, have %d", n,
302		backend.WorkspaceLiveStreamCountForTest(ws))
303}
304
305// drainUntil reads from evc until it sees an event of type T that
306// satisfies match, or ctx expires. Returns the matching event and
307// ok=true, or the zero value and ok=false on timeout.
308func drainUntil[T any](ctx context.Context, evc <-chan any, match func(T) bool) (T, bool) {
309	var zero T
310	for {
311		select {
312		case <-ctx.Done():
313			return zero, false
314		case ev, ok := <-evc:
315			if !ok {
316				return zero, false
317			}
318			typed, isT := ev.(T)
319			if !isT {
320				continue
321			}
322			if match == nil || match(typed) {
323				return typed, true
324			}
325		}
326	}
327}
328
329// TestE2E_TwoClientsReceiveSameMessage covers PLAN item 6 scenario 1:
330// two clients POST /v1/workspaces with the same Path and observe
331// that the server returns a single workspace (path-dedupe from PLAN
332// item 1) and that an event published on that workspace fans out to
333// both SSE streams.
334//
335// Cannot run in parallel: it isolates HOME/XDG_* via t.Setenv so
336// config.Init does not read the host machine's real config.
337func TestE2E_TwoClientsReceiveSameMessage(t *testing.T) {
338	h := newRealCreateHarness(t)
339	// Shorten the create-grace window so the workspace's pending
340	// creation holds release quickly during test cleanup once both
341	// SSE streams have been detached.
342	h.backend.SetCreateGrace(200 * time.Millisecond)
343
344	ctx, cancel := context.WithCancel(t.Context())
345	t.Cleanup(cancel)
346
347	cidA := uuid.New().String()
348	cidB := uuid.New().String()
349
350	// Shared workspace path. Two POSTs with this path must
351	// deduplicate at the backend's pathIndex and return the same
352	// workspace id.
353	wsPath := t.TempDir()
354	dataDir := t.TempDir()
355	args := proto.Workspace{Path: wsPath, DataDir: dataDir}
356
357	argsA := args
358	argsA.ClientID = cidA
359	wsRespA := h.postWorkspace(t, argsA)
360
361	argsB := args
362	argsB.ClientID = cidB
363	wsRespB := h.postWorkspace(t, argsB)
364
365	require.Equal(t, wsRespA.ID, wsRespB.ID,
366		"POST /v1/workspaces with the same Path must return the same workspace id")
367
368	// Look up the resulting workspace on the backend so the test
369	// can publish events through its real [app.App] event broker.
370	ws, err := h.backend.GetWorkspace(wsRespA.ID)
371	require.NoError(t, err)
372	// Override the shutdown callback so test cleanup doesn't run
373	// the full app.Shutdown path (which would tear down LSP/MCP
374	// resources the test doesn't need to exercise), but still
375	// release the pooled DB connection so Windows can clean up
376	// the temp data directory.
377	wsDataDir := ws.Cfg.Config().Options.DataDirectory
378	backend.SetWorkspaceShutdownFnForTest(ws, func() {
379		_ = db.Release(wsDataDir)
380	})
381
382	evcA, cancelA := h.subscribeSSE(t, ctx, ws.ID, cidA)
383	t.Cleanup(cancelA)
384	evcB, cancelB := h.subscribeSSE(t, ctx, ws.ID, cidB)
385	t.Cleanup(cancelB)
386
387	h.waitForAttachedOn(t, ws, 2)
388
389	const sessionID = "s-e2e-1"
390	msg := message.Message{
391		ID:        "m-1",
392		SessionID: sessionID,
393		Role:      message.Assistant,
394		Parts:     []message.ContentPart{message.TextContent{Text: "hello multi-client"}},
395	}
396	ws.SendEvent(pubsub.Event[message.Message]{
397		Type:    pubsub.CreatedEvent,
398		Payload: msg,
399	})
400
401	pickCtx, pickCancel := context.WithTimeout(ctx, 3*time.Second)
402	defer pickCancel()
403	gotA, okA := drainUntil(pickCtx, evcA, func(e pubsub.Event[proto.Message]) bool {
404		return e.Payload.ID == "m-1"
405	})
406	require.True(t, okA, "client A must receive the MessageEvent")
407	require.Equal(t, sessionID, gotA.Payload.SessionID)
408
409	gotB, okB := drainUntil(pickCtx, evcB, func(e pubsub.Event[proto.Message]) bool {
410		return e.Payload.ID == "m-1"
411	})
412	require.True(t, okB, "client B must receive the same MessageEvent")
413	require.Equal(t, sessionID, gotB.Payload.SessionID)
414}
415
416// TestE2E_PermissionFlowCrossClient covers PLAN item 6 scenario 2:
417// a tool-driven permission request is granted by client A; client B
418// observes a PermissionNotification; a redundant grant from B
419// returns the "already resolved" indicator (resolved=false from the
420// bool plumbing landed in item 3).
421func TestE2E_PermissionFlowCrossClient(t *testing.T) {
422	t.Parallel()
423	h := newE2EHarness(t)
424	ctx, cancel := context.WithCancel(t.Context())
425	t.Cleanup(cancel)
426
427	cidA := uuid.New().String()
428	cidB := uuid.New().String()
429
430	evcA, cancelA := h.subscribeSSE(t, ctx, h.workspace.ID, cidA)
431	t.Cleanup(cancelA)
432	evcB, cancelB := h.subscribeSSE(t, ctx, h.workspace.ID, cidB)
433	t.Cleanup(cancelB)
434
435	h.waitForAttached(t, 2)
436
437	// Drive the permission request from a goroutine simulating the
438	// tool path. Request blocks until resolved; capture the outcome.
439	const sessionID = "s-perm"
440	const toolCallID = "tc-1"
441	type result struct {
442		granted bool
443		err     error
444	}
445	done := make(chan result, 1)
446	go func() {
447		granted, err := h.app.Permissions.Request(ctx, permission.CreatePermissionRequest{
448			SessionID:   sessionID,
449			ToolCallID:  toolCallID,
450			ToolName:    "view",
451			Description: "read a file",
452			Action:      "read",
453			Path:        h.workspace.Path,
454		})
455		done <- result{granted: granted, err: err}
456	}()
457
458	// Wait for the PermissionRequest to arrive on client A's SSE
459	// stream. We need its ID to drive the grant.
460	pickCtx, pickCancel := context.WithTimeout(ctx, 3*time.Second)
461	defer pickCancel()
462	reqEv, ok := drainUntil(pickCtx, evcA, func(e pubsub.Event[proto.PermissionRequest]) bool {
463		return e.Payload.ToolCallID == toolCallID
464	})
465	require.True(t, ok, "client A must receive the PermissionRequest")
466
467	// Client A grants — first grant must report resolved=true.
468	resolvedA := h.grantPermission(t, ctx, h.workspace.ID, proto.PermissionGrant{
469		Permission: reqEv.Payload,
470		Action:     proto.PermissionAllow,
471	})
472	require.True(t, resolvedA, "client A's grant must resolve the pending request")
473
474	// The blocked Request call must now return granted=true.
475	select {
476	case r := <-done:
477		require.NoError(t, r.err)
478		require.True(t, r.granted)
479	case <-pickCtx.Done():
480		t.Fatal("permission Request did not return after grant")
481	}
482
483	// Client B must receive a PermissionNotification with
484	// Granted=true for the same ToolCallID. The initial neither-
485	// granted-nor-denied notification published at the start of
486	// Request also lands on B's stream — match on the granted one.
487	notif, ok := drainUntil(pickCtx, evcB, func(e pubsub.Event[proto.PermissionNotification]) bool {
488		return e.Payload.ToolCallID == toolCallID && e.Payload.Granted
489	})
490	require.True(t, ok, "client B must receive a granting PermissionNotification")
491	require.True(t, notif.Payload.Granted)
492	require.False(t, notif.Payload.Denied)
493
494	// A follow-up grant from client B must report resolved=false
495	// (the request was already resolved by A).
496	resolvedB := h.grantPermission(t, ctx, h.workspace.ID, proto.PermissionGrant{
497		Permission: reqEv.Payload,
498		Action:     proto.PermissionAllow,
499	})
500	require.False(t, resolvedB, "client B's follow-up grant must report already resolved")
501}
502
503// TestE2E_KillingClientASSEDoesNotBreakClientB covers PLAN item 6
504// scenario 3: terminating client A's SSE stream does not affect
505// client B's stream; client B continues to receive events.
506func TestE2E_KillingClientASSEDoesNotBreakClientB(t *testing.T) {
507	t.Parallel()
508	h := newE2EHarness(t)
509	ctxB, cancelB := context.WithCancel(t.Context())
510	t.Cleanup(cancelB)
511	ctxA, cancelA := context.WithCancel(t.Context())
512
513	cidA := uuid.New().String()
514	cidB := uuid.New().String()
515
516	_, killA := h.subscribeSSE(t, ctxA, h.workspace.ID, cidA)
517	t.Cleanup(killA)
518	evcB, killB := h.subscribeSSE(t, ctxB, h.workspace.ID, cidB)
519	t.Cleanup(killB)
520
521	h.waitForAttached(t, 2)
522
523	// Kill A's stream. The server's deferred DetachClient should
524	// drop A's claim, leaving B as the sole attached client.
525	cancelA()
526	killA()
527
528	require.Eventually(t, func() bool {
529		return backend.WorkspaceLiveStreamCountForTest(h.workspace) == 1
530	}, 3*time.Second, 10*time.Millisecond,
531		"expected client A's stream to drop the attached count to 1")
532
533	// Workspace must still exist (B is holding it open) and
534	// shutdown callback must not have fired yet.
535	_, err := h.backend.GetWorkspace(h.workspace.ID)
536	require.NoError(t, err, "workspace must still exist while B is attached")
537	require.False(t, h.shutdownHit.Load(),
538		"shutdown callback must not fire while B is still attached")
539
540	// Publish a fresh event; B must still receive it.
541	const sessionID = "s-after-a-died"
542	msg := message.Message{
543		ID:        "m-after",
544		SessionID: sessionID,
545		Role:      message.Assistant,
546		Parts:     []message.ContentPart{message.TextContent{Text: "still alive"}},
547	}
548	h.app.SendEvent(pubsub.Event[message.Message]{
549		Type:    pubsub.CreatedEvent,
550		Payload: msg,
551	})
552
553	pickCtx, pickCancel := context.WithTimeout(ctxB, 3*time.Second)
554	defer pickCancel()
555	got, ok := drainUntil(pickCtx, evcB, func(e pubsub.Event[proto.Message]) bool {
556		return e.Payload.ID == "m-after"
557	})
558	require.True(t, ok, "client B must still receive events after A's stream is killed")
559	require.Equal(t, sessionID, got.Payload.SessionID)
560}
561
562// TestE2E_ShutdownCallbackFiresWhenLastClientLeaves covers PLAN
563// item 6 scenario 4: once both clients disconnect, the backend
564// runs its "last workspace removed -> server shutdown" path.
565func TestE2E_ShutdownCallbackFiresWhenLastClientLeaves(t *testing.T) {
566	t.Parallel()
567	h := newE2EHarness(t)
568
569	ctxA, cancelA := context.WithCancel(t.Context())
570	ctxB, cancelB := context.WithCancel(t.Context())
571	t.Cleanup(cancelA)
572	t.Cleanup(cancelB)
573
574	cidA := uuid.New().String()
575	cidB := uuid.New().String()
576	_, killA := h.subscribeSSE(t, ctxA, h.workspace.ID, cidA)
577	t.Cleanup(killA)
578	_, killB := h.subscribeSSE(t, ctxB, h.workspace.ID, cidB)
579	t.Cleanup(killB)
580
581	h.waitForAttached(t, 2)
582	require.False(t, h.shutdownHit.Load(), "shutdown must not fire while clients are attached")
583
584	cancelA()
585	killA()
586	require.Eventually(t, func() bool {
587		return backend.WorkspaceLiveStreamCountForTest(h.workspace) == 1
588	}, 3*time.Second, 10*time.Millisecond)
589	require.False(t, h.shutdownHit.Load(),
590		"shutdown must not fire after only one client disconnects")
591
592	cancelB()
593	killB()
594	require.Eventually(t, h.shutdownHit.Load,
595		3*time.Second, 10*time.Millisecond,
596		"shutdown callback must fire once the last client disconnects")
597
598	// Workspace must be gone from the index.
599	_, err := h.backend.GetWorkspace(h.workspace.ID)
600	require.ErrorIs(t, err, backend.ErrWorkspaceNotFound)
601
602	// Subsequent GETs against the now-defunct workspace return
603	// 404, confirming the http surface still reflects the teardown.
604	req, err := http.NewRequestWithContext(t.Context(), http.MethodGet,
605		h.httpSrv.URL+"/v1/workspaces/"+h.workspace.ID, nil)
606	require.NoError(t, err)
607	r, err := h.httpSrv.Client().Do(req)
608	require.NoError(t, err)
609	_, _ = io.Copy(io.Discard, r.Body)
610	r.Body.Close()
611	require.Equal(t, http.StatusNotFound, r.StatusCode)
612}