1package wazevo
2
3import (
4 "context"
5 "fmt"
6 "reflect"
7 "runtime"
8 "sync/atomic"
9 "unsafe"
10
11 "github.com/tetratelabs/wazero/api"
12 "github.com/tetratelabs/wazero/experimental"
13 "github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi"
14 "github.com/tetratelabs/wazero/internal/expctxkeys"
15 "github.com/tetratelabs/wazero/internal/internalapi"
16 "github.com/tetratelabs/wazero/internal/wasm"
17 "github.com/tetratelabs/wazero/internal/wasmdebug"
18 "github.com/tetratelabs/wazero/internal/wasmruntime"
19)
20
21type (
22 // callEngine implements api.Function.
23 callEngine struct {
24 internalapi.WazeroOnly
25 stack []byte
26 // stackTop is the pointer to the *aligned* top of the stack. This must be updated
27 // whenever the stack is changed. This is passed to the assembly function
28 // at the very beginning of api.Function Call/CallWithStack.
29 stackTop uintptr
30 // executable is the pointer to the executable code for this function.
31 executable *byte
32 preambleExecutable *byte
33 // parent is the *moduleEngine from which this callEngine is created.
34 parent *moduleEngine
35 // indexInModule is the index of the function in the module.
36 indexInModule wasm.Index
37 // sizeOfParamResultSlice is the size of the parameter/result slice.
38 sizeOfParamResultSlice int
39 requiredParams int
40 // execCtx holds various information to be read/written by assembly functions.
41 execCtx executionContext
42 // execCtxPtr holds the pointer to the executionContext which doesn't change after callEngine is created.
43 execCtxPtr uintptr
44 numberOfResults int
45 stackIteratorImpl stackIterator
46 }
47
48 // executionContext is the struct to be read/written by assembly functions.
49 executionContext struct {
50 // exitCode holds the wazevoapi.ExitCode describing the state of the function execution.
51 exitCode wazevoapi.ExitCode
52 // callerModuleContextPtr holds the moduleContextOpaque for Go function calls.
53 callerModuleContextPtr *byte
54 // originalFramePointer holds the original frame pointer of the caller of the assembly function.
55 originalFramePointer uintptr
56 // originalStackPointer holds the original stack pointer of the caller of the assembly function.
57 originalStackPointer uintptr
58 // goReturnAddress holds the return address to go back to the caller of the assembly function.
59 goReturnAddress uintptr
60 // stackBottomPtr holds the pointer to the bottom of the stack.
61 stackBottomPtr *byte
62 // goCallReturnAddress holds the return address to go back to the caller of the Go function.
63 goCallReturnAddress *byte
64 // stackPointerBeforeGoCall holds the stack pointer before calling a Go function.
65 stackPointerBeforeGoCall *uint64
66 // stackGrowRequiredSize holds the required size of stack grow.
67 stackGrowRequiredSize uintptr
68 // memoryGrowTrampolineAddress holds the address of memory grow trampoline function.
69 memoryGrowTrampolineAddress *byte
70 // stackGrowCallTrampolineAddress holds the address of stack grow trampoline function.
71 stackGrowCallTrampolineAddress *byte
72 // checkModuleExitCodeTrampolineAddress holds the address of check-module-exit-code function.
73 checkModuleExitCodeTrampolineAddress *byte
74 // savedRegisters is the opaque spaces for save/restore registers.
75 // We want to align 16 bytes for each register, so we use [64][2]uint64.
76 savedRegisters [64][2]uint64
77 // goFunctionCallCalleeModuleContextOpaque is the pointer to the target Go function's moduleContextOpaque.
78 goFunctionCallCalleeModuleContextOpaque uintptr
79 // tableGrowTrampolineAddress holds the address of table grow trampoline function.
80 tableGrowTrampolineAddress *byte
81 // refFuncTrampolineAddress holds the address of ref-func trampoline function.
82 refFuncTrampolineAddress *byte
83 // memmoveAddress holds the address of memmove function implemented by Go runtime. See memmove.go.
84 memmoveAddress uintptr
85 // framePointerBeforeGoCall holds the frame pointer before calling a Go function. Note: only used in amd64.
86 framePointerBeforeGoCall uintptr
87 // memoryWait32TrampolineAddress holds the address of memory_wait32 trampoline function.
88 memoryWait32TrampolineAddress *byte
89 // memoryWait32TrampolineAddress holds the address of memory_wait64 trampoline function.
90 memoryWait64TrampolineAddress *byte
91 // memoryNotifyTrampolineAddress holds the address of the memory_notify trampoline function.
92 memoryNotifyTrampolineAddress *byte
93 }
94)
95
96func (c *callEngine) requiredInitialStackSize() int {
97 const initialStackSizeDefault = 10240
98 stackSize := initialStackSizeDefault
99 paramResultInBytes := c.sizeOfParamResultSlice * 8 * 2 // * 8 because uint64 is 8 bytes, and *2 because we need both separated param/result slots.
100 required := paramResultInBytes + 32 + 16 // 32 is enough to accommodate the call frame info, and 16 exists just in case when []byte is not aligned to 16 bytes.
101 if required > stackSize {
102 stackSize = required
103 }
104 return stackSize
105}
106
107func (c *callEngine) init() {
108 stackSize := c.requiredInitialStackSize()
109 if wazevoapi.StackGuardCheckEnabled {
110 stackSize += wazevoapi.StackGuardCheckGuardPageSize
111 }
112 c.stack = make([]byte, stackSize)
113 c.stackTop = alignedStackTop(c.stack)
114 if wazevoapi.StackGuardCheckEnabled {
115 c.execCtx.stackBottomPtr = &c.stack[wazevoapi.StackGuardCheckGuardPageSize]
116 } else {
117 c.execCtx.stackBottomPtr = &c.stack[0]
118 }
119 c.execCtxPtr = uintptr(unsafe.Pointer(&c.execCtx))
120}
121
122// alignedStackTop returns 16-bytes aligned stack top of given stack.
123// 16 bytes should be good for all platform (arm64/amd64).
124func alignedStackTop(s []byte) uintptr {
125 stackAddr := uintptr(unsafe.Pointer(&s[len(s)-1]))
126 return stackAddr - (stackAddr & (16 - 1))
127}
128
129// Definition implements api.Function.
130func (c *callEngine) Definition() api.FunctionDefinition {
131 return c.parent.module.Source.FunctionDefinition(c.indexInModule)
132}
133
134// Call implements api.Function.
135func (c *callEngine) Call(ctx context.Context, params ...uint64) ([]uint64, error) {
136 if c.requiredParams != len(params) {
137 return nil, fmt.Errorf("expected %d params, but passed %d", c.requiredParams, len(params))
138 }
139 paramResultSlice := make([]uint64, c.sizeOfParamResultSlice)
140 copy(paramResultSlice, params)
141 if err := c.callWithStack(ctx, paramResultSlice); err != nil {
142 return nil, err
143 }
144 return paramResultSlice[:c.numberOfResults], nil
145}
146
147func (c *callEngine) addFrame(builder wasmdebug.ErrorBuilder, addr uintptr) (def api.FunctionDefinition, listener experimental.FunctionListener) {
148 eng := c.parent.parent.parent
149 cm := eng.compiledModuleOfAddr(addr)
150 if cm == nil {
151 // This case, the module might have been closed and deleted from the engine.
152 // We fall back to searching the imported modules that can be referenced from this callEngine.
153
154 // First, we check itself.
155 if checkAddrInBytes(addr, c.parent.parent.executable) {
156 cm = c.parent.parent
157 } else {
158 // Otherwise, search all imported modules. TODO: maybe recursive, but not sure it's useful in practice.
159 p := c.parent
160 for i := range p.importedFunctions {
161 candidate := p.importedFunctions[i].me.parent
162 if checkAddrInBytes(addr, candidate.executable) {
163 cm = candidate
164 break
165 }
166 }
167 }
168 }
169
170 if cm != nil {
171 index := cm.functionIndexOf(addr)
172 def = cm.module.FunctionDefinition(cm.module.ImportFunctionCount + index)
173 var sources []string
174 if dw := cm.module.DWARFLines; dw != nil {
175 sourceOffset := cm.getSourceOffset(addr)
176 sources = dw.Line(sourceOffset)
177 }
178 builder.AddFrame(def.DebugName(), def.ParamTypes(), def.ResultTypes(), sources)
179 if len(cm.listeners) > 0 {
180 listener = cm.listeners[index]
181 }
182 }
183 return
184}
185
186// CallWithStack implements api.Function.
187func (c *callEngine) CallWithStack(ctx context.Context, paramResultStack []uint64) (err error) {
188 if c.sizeOfParamResultSlice > len(paramResultStack) {
189 return fmt.Errorf("need %d params, but stack size is %d", c.sizeOfParamResultSlice, len(paramResultStack))
190 }
191 return c.callWithStack(ctx, paramResultStack)
192}
193
194// CallWithStack implements api.Function.
195func (c *callEngine) callWithStack(ctx context.Context, paramResultStack []uint64) (err error) {
196 snapshotEnabled := ctx.Value(expctxkeys.EnableSnapshotterKey{}) != nil
197 if snapshotEnabled {
198 ctx = context.WithValue(ctx, expctxkeys.SnapshotterKey{}, c)
199 }
200
201 if wazevoapi.StackGuardCheckEnabled {
202 defer func() {
203 wazevoapi.CheckStackGuardPage(c.stack)
204 }()
205 }
206
207 p := c.parent
208 ensureTermination := p.parent.ensureTermination
209 m := p.module
210 if ensureTermination {
211 select {
212 case <-ctx.Done():
213 // If the provided context is already done, close the module and return the error.
214 m.CloseWithCtxErr(ctx)
215 return m.FailIfClosed()
216 default:
217 }
218 }
219
220 var paramResultPtr *uint64
221 if len(paramResultStack) > 0 {
222 paramResultPtr = ¶mResultStack[0]
223 }
224 defer func() {
225 r := recover()
226 if s, ok := r.(*snapshot); ok {
227 // A snapshot that wasn't handled was created by a different call engine possibly from a nested wasm invocation,
228 // let it propagate up to be handled by the caller.
229 panic(s)
230 }
231 if r != nil {
232 type listenerForAbort struct {
233 def api.FunctionDefinition
234 lsn experimental.FunctionListener
235 }
236
237 var listeners []listenerForAbort
238 builder := wasmdebug.NewErrorBuilder()
239 def, lsn := c.addFrame(builder, uintptr(unsafe.Pointer(c.execCtx.goCallReturnAddress)))
240 if lsn != nil {
241 listeners = append(listeners, listenerForAbort{def, lsn})
242 }
243 returnAddrs := unwindStack(
244 uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall)),
245 c.execCtx.framePointerBeforeGoCall,
246 c.stackTop,
247 nil,
248 )
249 for _, retAddr := range returnAddrs[:len(returnAddrs)-1] { // the last return addr is the trampoline, so we skip it.
250 def, lsn = c.addFrame(builder, retAddr)
251 if lsn != nil {
252 listeners = append(listeners, listenerForAbort{def, lsn})
253 }
254 }
255 err = builder.FromRecovered(r)
256
257 for _, lsn := range listeners {
258 lsn.lsn.Abort(ctx, m, lsn.def, err)
259 }
260 } else {
261 if err != wasmruntime.ErrRuntimeStackOverflow { // Stackoverflow case shouldn't be panic (to avoid extreme stack unwinding).
262 err = c.parent.module.FailIfClosed()
263 }
264 }
265
266 if err != nil {
267 // Ensures that we can reuse this callEngine even after an error.
268 c.execCtx.exitCode = wazevoapi.ExitCodeOK
269 }
270 }()
271
272 if ensureTermination {
273 done := m.CloseModuleOnCanceledOrTimeout(ctx)
274 defer done()
275 }
276
277 if c.stackTop&(16-1) != 0 {
278 panic("BUG: stack must be aligned to 16 bytes")
279 }
280 entrypoint(c.preambleExecutable, c.executable, c.execCtxPtr, c.parent.opaquePtr, paramResultPtr, c.stackTop)
281 for {
282 switch ec := c.execCtx.exitCode; ec & wazevoapi.ExitCodeMask {
283 case wazevoapi.ExitCodeOK:
284 return nil
285 case wazevoapi.ExitCodeGrowStack:
286 oldsp := uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall))
287 oldTop := c.stackTop
288 oldStack := c.stack
289 var newsp, newfp uintptr
290 if wazevoapi.StackGuardCheckEnabled {
291 newsp, newfp, err = c.growStackWithGuarded()
292 } else {
293 newsp, newfp, err = c.growStack()
294 }
295 if err != nil {
296 return err
297 }
298 adjustClonedStack(oldsp, oldTop, newsp, newfp, c.stackTop)
299 // Old stack must be alive until the new stack is adjusted.
300 runtime.KeepAlive(oldStack)
301 c.execCtx.exitCode = wazevoapi.ExitCodeOK
302 afterGoFunctionCallEntrypoint(c.execCtx.goCallReturnAddress, c.execCtxPtr, newsp, newfp)
303 case wazevoapi.ExitCodeGrowMemory:
304 mod := c.callerModuleInstance()
305 mem := mod.MemoryInstance
306 s := goCallStackView(c.execCtx.stackPointerBeforeGoCall)
307 argRes := &s[0]
308 if res, ok := mem.Grow(uint32(*argRes)); !ok {
309 *argRes = uint64(0xffffffff) // = -1 in signed 32-bit integer.
310 } else {
311 *argRes = uint64(res)
312 }
313 c.execCtx.exitCode = wazevoapi.ExitCodeOK
314 afterGoFunctionCallEntrypoint(c.execCtx.goCallReturnAddress, c.execCtxPtr, uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall)), c.execCtx.framePointerBeforeGoCall)
315 case wazevoapi.ExitCodeTableGrow:
316 mod := c.callerModuleInstance()
317 s := goCallStackView(c.execCtx.stackPointerBeforeGoCall)
318 tableIndex, num, ref := uint32(s[0]), uint32(s[1]), uintptr(s[2])
319 table := mod.Tables[tableIndex]
320 s[0] = uint64(uint32(int32(table.Grow(num, ref))))
321 c.execCtx.exitCode = wazevoapi.ExitCodeOK
322 afterGoFunctionCallEntrypoint(c.execCtx.goCallReturnAddress, c.execCtxPtr,
323 uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall)), c.execCtx.framePointerBeforeGoCall)
324 case wazevoapi.ExitCodeCallGoFunction:
325 index := wazevoapi.GoFunctionIndexFromExitCode(ec)
326 f := hostModuleGoFuncFromOpaque[api.GoFunction](index, c.execCtx.goFunctionCallCalleeModuleContextOpaque)
327 func() {
328 if snapshotEnabled {
329 defer snapshotRecoverFn(c)
330 }
331 f.Call(ctx, goCallStackView(c.execCtx.stackPointerBeforeGoCall))
332 }()
333 // Back to the native code.
334 c.execCtx.exitCode = wazevoapi.ExitCodeOK
335 afterGoFunctionCallEntrypoint(c.execCtx.goCallReturnAddress, c.execCtxPtr,
336 uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall)), c.execCtx.framePointerBeforeGoCall)
337 case wazevoapi.ExitCodeCallGoFunctionWithListener:
338 index := wazevoapi.GoFunctionIndexFromExitCode(ec)
339 f := hostModuleGoFuncFromOpaque[api.GoFunction](index, c.execCtx.goFunctionCallCalleeModuleContextOpaque)
340 listeners := hostModuleListenersSliceFromOpaque(c.execCtx.goFunctionCallCalleeModuleContextOpaque)
341 s := goCallStackView(c.execCtx.stackPointerBeforeGoCall)
342 // Call Listener.Before.
343 callerModule := c.callerModuleInstance()
344 listener := listeners[index]
345 hostModule := hostModuleFromOpaque(c.execCtx.goFunctionCallCalleeModuleContextOpaque)
346 def := hostModule.FunctionDefinition(wasm.Index(index))
347 listener.Before(ctx, callerModule, def, s, c.stackIterator(true))
348 // Call into the Go function.
349 func() {
350 if snapshotEnabled {
351 defer snapshotRecoverFn(c)
352 }
353 f.Call(ctx, s)
354 }()
355 // Call Listener.After.
356 listener.After(ctx, callerModule, def, s)
357 // Back to the native code.
358 c.execCtx.exitCode = wazevoapi.ExitCodeOK
359 afterGoFunctionCallEntrypoint(c.execCtx.goCallReturnAddress, c.execCtxPtr,
360 uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall)), c.execCtx.framePointerBeforeGoCall)
361 case wazevoapi.ExitCodeCallGoModuleFunction:
362 index := wazevoapi.GoFunctionIndexFromExitCode(ec)
363 f := hostModuleGoFuncFromOpaque[api.GoModuleFunction](index, c.execCtx.goFunctionCallCalleeModuleContextOpaque)
364 mod := c.callerModuleInstance()
365 func() {
366 if snapshotEnabled {
367 defer snapshotRecoverFn(c)
368 }
369 f.Call(ctx, mod, goCallStackView(c.execCtx.stackPointerBeforeGoCall))
370 }()
371 // Back to the native code.
372 c.execCtx.exitCode = wazevoapi.ExitCodeOK
373 afterGoFunctionCallEntrypoint(c.execCtx.goCallReturnAddress, c.execCtxPtr,
374 uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall)), c.execCtx.framePointerBeforeGoCall)
375 case wazevoapi.ExitCodeCallGoModuleFunctionWithListener:
376 index := wazevoapi.GoFunctionIndexFromExitCode(ec)
377 f := hostModuleGoFuncFromOpaque[api.GoModuleFunction](index, c.execCtx.goFunctionCallCalleeModuleContextOpaque)
378 listeners := hostModuleListenersSliceFromOpaque(c.execCtx.goFunctionCallCalleeModuleContextOpaque)
379 s := goCallStackView(c.execCtx.stackPointerBeforeGoCall)
380 // Call Listener.Before.
381 callerModule := c.callerModuleInstance()
382 listener := listeners[index]
383 hostModule := hostModuleFromOpaque(c.execCtx.goFunctionCallCalleeModuleContextOpaque)
384 def := hostModule.FunctionDefinition(wasm.Index(index))
385 listener.Before(ctx, callerModule, def, s, c.stackIterator(true))
386 // Call into the Go function.
387 func() {
388 if snapshotEnabled {
389 defer snapshotRecoverFn(c)
390 }
391 f.Call(ctx, callerModule, s)
392 }()
393 // Call Listener.After.
394 listener.After(ctx, callerModule, def, s)
395 // Back to the native code.
396 c.execCtx.exitCode = wazevoapi.ExitCodeOK
397 afterGoFunctionCallEntrypoint(c.execCtx.goCallReturnAddress, c.execCtxPtr,
398 uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall)), c.execCtx.framePointerBeforeGoCall)
399 case wazevoapi.ExitCodeCallListenerBefore:
400 stack := goCallStackView(c.execCtx.stackPointerBeforeGoCall)
401 index := wasm.Index(stack[0])
402 mod := c.callerModuleInstance()
403 listener := mod.Engine.(*moduleEngine).listeners[index]
404 def := mod.Source.FunctionDefinition(index + mod.Source.ImportFunctionCount)
405 listener.Before(ctx, mod, def, stack[1:], c.stackIterator(false))
406 c.execCtx.exitCode = wazevoapi.ExitCodeOK
407 afterGoFunctionCallEntrypoint(c.execCtx.goCallReturnAddress, c.execCtxPtr,
408 uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall)), c.execCtx.framePointerBeforeGoCall)
409 case wazevoapi.ExitCodeCallListenerAfter:
410 stack := goCallStackView(c.execCtx.stackPointerBeforeGoCall)
411 index := wasm.Index(stack[0])
412 mod := c.callerModuleInstance()
413 listener := mod.Engine.(*moduleEngine).listeners[index]
414 def := mod.Source.FunctionDefinition(index + mod.Source.ImportFunctionCount)
415 listener.After(ctx, mod, def, stack[1:])
416 c.execCtx.exitCode = wazevoapi.ExitCodeOK
417 afterGoFunctionCallEntrypoint(c.execCtx.goCallReturnAddress, c.execCtxPtr,
418 uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall)), c.execCtx.framePointerBeforeGoCall)
419 case wazevoapi.ExitCodeCheckModuleExitCode:
420 // Note: this operation must be done in Go, not native code. The reason is that
421 // native code cannot be preempted and that means it can block forever if there are not
422 // enough OS threads (which we don't have control over).
423 if err := m.FailIfClosed(); err != nil {
424 panic(err)
425 }
426 c.execCtx.exitCode = wazevoapi.ExitCodeOK
427 afterGoFunctionCallEntrypoint(c.execCtx.goCallReturnAddress, c.execCtxPtr,
428 uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall)), c.execCtx.framePointerBeforeGoCall)
429 case wazevoapi.ExitCodeRefFunc:
430 mod := c.callerModuleInstance()
431 s := goCallStackView(c.execCtx.stackPointerBeforeGoCall)
432 funcIndex := wasm.Index(s[0])
433 ref := mod.Engine.FunctionInstanceReference(funcIndex)
434 s[0] = uint64(ref)
435 c.execCtx.exitCode = wazevoapi.ExitCodeOK
436 afterGoFunctionCallEntrypoint(c.execCtx.goCallReturnAddress, c.execCtxPtr,
437 uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall)), c.execCtx.framePointerBeforeGoCall)
438 case wazevoapi.ExitCodeMemoryWait32:
439 mod := c.callerModuleInstance()
440 mem := mod.MemoryInstance
441 if !mem.Shared {
442 panic(wasmruntime.ErrRuntimeExpectedSharedMemory)
443 }
444
445 s := goCallStackView(c.execCtx.stackPointerBeforeGoCall)
446 timeout, exp, addr := int64(s[0]), uint32(s[1]), uintptr(s[2])
447 base := uintptr(unsafe.Pointer(&mem.Buffer[0]))
448
449 offset := uint32(addr - base)
450 res := mem.Wait32(offset, exp, timeout, func(mem *wasm.MemoryInstance, offset uint32) uint32 {
451 addr := unsafe.Add(unsafe.Pointer(&mem.Buffer[0]), offset)
452 return atomic.LoadUint32((*uint32)(addr))
453 })
454 s[0] = res
455 c.execCtx.exitCode = wazevoapi.ExitCodeOK
456 afterGoFunctionCallEntrypoint(c.execCtx.goCallReturnAddress, c.execCtxPtr,
457 uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall)), c.execCtx.framePointerBeforeGoCall)
458 case wazevoapi.ExitCodeMemoryWait64:
459 mod := c.callerModuleInstance()
460 mem := mod.MemoryInstance
461 if !mem.Shared {
462 panic(wasmruntime.ErrRuntimeExpectedSharedMemory)
463 }
464
465 s := goCallStackView(c.execCtx.stackPointerBeforeGoCall)
466 timeout, exp, addr := int64(s[0]), uint64(s[1]), uintptr(s[2])
467 base := uintptr(unsafe.Pointer(&mem.Buffer[0]))
468
469 offset := uint32(addr - base)
470 res := mem.Wait64(offset, exp, timeout, func(mem *wasm.MemoryInstance, offset uint32) uint64 {
471 addr := unsafe.Add(unsafe.Pointer(&mem.Buffer[0]), offset)
472 return atomic.LoadUint64((*uint64)(addr))
473 })
474 s[0] = uint64(res)
475 c.execCtx.exitCode = wazevoapi.ExitCodeOK
476 afterGoFunctionCallEntrypoint(c.execCtx.goCallReturnAddress, c.execCtxPtr,
477 uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall)), c.execCtx.framePointerBeforeGoCall)
478 case wazevoapi.ExitCodeMemoryNotify:
479 mod := c.callerModuleInstance()
480 mem := mod.MemoryInstance
481
482 s := goCallStackView(c.execCtx.stackPointerBeforeGoCall)
483 count, addr := uint32(s[0]), s[1]
484 offset := uint32(uintptr(addr) - uintptr(unsafe.Pointer(&mem.Buffer[0])))
485 res := mem.Notify(offset, count)
486 s[0] = uint64(res)
487 c.execCtx.exitCode = wazevoapi.ExitCodeOK
488 afterGoFunctionCallEntrypoint(c.execCtx.goCallReturnAddress, c.execCtxPtr,
489 uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall)), c.execCtx.framePointerBeforeGoCall)
490 case wazevoapi.ExitCodeUnreachable:
491 panic(wasmruntime.ErrRuntimeUnreachable)
492 case wazevoapi.ExitCodeMemoryOutOfBounds:
493 panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess)
494 case wazevoapi.ExitCodeTableOutOfBounds:
495 panic(wasmruntime.ErrRuntimeInvalidTableAccess)
496 case wazevoapi.ExitCodeIndirectCallNullPointer:
497 panic(wasmruntime.ErrRuntimeInvalidTableAccess)
498 case wazevoapi.ExitCodeIndirectCallTypeMismatch:
499 panic(wasmruntime.ErrRuntimeIndirectCallTypeMismatch)
500 case wazevoapi.ExitCodeIntegerOverflow:
501 panic(wasmruntime.ErrRuntimeIntegerOverflow)
502 case wazevoapi.ExitCodeIntegerDivisionByZero:
503 panic(wasmruntime.ErrRuntimeIntegerDivideByZero)
504 case wazevoapi.ExitCodeInvalidConversionToInteger:
505 panic(wasmruntime.ErrRuntimeInvalidConversionToInteger)
506 case wazevoapi.ExitCodeUnalignedAtomic:
507 panic(wasmruntime.ErrRuntimeUnalignedAtomic)
508 default:
509 panic("BUG")
510 }
511 }
512}
513
514func (c *callEngine) callerModuleInstance() *wasm.ModuleInstance {
515 return moduleInstanceFromOpaquePtr(c.execCtx.callerModuleContextPtr)
516}
517
518const callStackCeiling = uintptr(50000000) // in uint64 (8 bytes) == 400000000 bytes in total == 400mb.
519
520func (c *callEngine) growStackWithGuarded() (newSP uintptr, newFP uintptr, err error) {
521 if wazevoapi.StackGuardCheckEnabled {
522 wazevoapi.CheckStackGuardPage(c.stack)
523 }
524 newSP, newFP, err = c.growStack()
525 if err != nil {
526 return
527 }
528 if wazevoapi.StackGuardCheckEnabled {
529 c.execCtx.stackBottomPtr = &c.stack[wazevoapi.StackGuardCheckGuardPageSize]
530 }
531 return
532}
533
534// growStack grows the stack, and returns the new stack pointer.
535func (c *callEngine) growStack() (newSP, newFP uintptr, err error) {
536 currentLen := uintptr(len(c.stack))
537 if callStackCeiling < currentLen {
538 err = wasmruntime.ErrRuntimeStackOverflow
539 return
540 }
541
542 newLen := 2*currentLen + c.execCtx.stackGrowRequiredSize + 16 // Stack might be aligned to 16 bytes, so add 16 bytes just in case.
543 newSP, newFP, c.stackTop, c.stack = c.cloneStack(newLen)
544 c.execCtx.stackBottomPtr = &c.stack[0]
545 return
546}
547
548func (c *callEngine) cloneStack(l uintptr) (newSP, newFP, newTop uintptr, newStack []byte) {
549 newStack = make([]byte, l)
550
551 relSp := c.stackTop - uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall))
552 relFp := c.stackTop - c.execCtx.framePointerBeforeGoCall
553
554 // Copy the existing contents in the previous Go-allocated stack into the new one.
555 var prevStackAligned, newStackAligned []byte
556 {
557 //nolint:staticcheck
558 sh := (*reflect.SliceHeader)(unsafe.Pointer(&prevStackAligned))
559 sh.Data = c.stackTop - relSp
560 sh.Len = int(relSp)
561 sh.Cap = int(relSp)
562 }
563 newTop = alignedStackTop(newStack)
564 {
565 newSP = newTop - relSp
566 newFP = newTop - relFp
567 //nolint:staticcheck
568 sh := (*reflect.SliceHeader)(unsafe.Pointer(&newStackAligned))
569 sh.Data = newSP
570 sh.Len = int(relSp)
571 sh.Cap = int(relSp)
572 }
573 copy(newStackAligned, prevStackAligned)
574 return
575}
576
577func (c *callEngine) stackIterator(onHostCall bool) experimental.StackIterator {
578 c.stackIteratorImpl.reset(c, onHostCall)
579 return &c.stackIteratorImpl
580}
581
582// stackIterator implements experimental.StackIterator.
583type stackIterator struct {
584 retAddrs []uintptr
585 retAddrCursor int
586 eng *engine
587 pc uint64
588
589 currentDef *wasm.FunctionDefinition
590}
591
592func (si *stackIterator) reset(c *callEngine, onHostCall bool) {
593 if onHostCall {
594 si.retAddrs = append(si.retAddrs[:0], uintptr(unsafe.Pointer(c.execCtx.goCallReturnAddress)))
595 } else {
596 si.retAddrs = si.retAddrs[:0]
597 }
598 si.retAddrs = unwindStack(uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall)), c.execCtx.framePointerBeforeGoCall, c.stackTop, si.retAddrs)
599 si.retAddrs = si.retAddrs[:len(si.retAddrs)-1] // the last return addr is the trampoline, so we skip it.
600 si.retAddrCursor = 0
601 si.eng = c.parent.parent.parent
602}
603
604// Next implements the same method as documented on experimental.StackIterator.
605func (si *stackIterator) Next() bool {
606 if si.retAddrCursor >= len(si.retAddrs) {
607 return false
608 }
609
610 addr := si.retAddrs[si.retAddrCursor]
611 cm := si.eng.compiledModuleOfAddr(addr)
612 if cm != nil {
613 index := cm.functionIndexOf(addr)
614 def := cm.module.FunctionDefinition(cm.module.ImportFunctionCount + index)
615 si.currentDef = def
616 si.retAddrCursor++
617 si.pc = uint64(addr)
618 return true
619 }
620 return false
621}
622
623// ProgramCounter implements the same method as documented on experimental.StackIterator.
624func (si *stackIterator) ProgramCounter() experimental.ProgramCounter {
625 return experimental.ProgramCounter(si.pc)
626}
627
628// Function implements the same method as documented on experimental.StackIterator.
629func (si *stackIterator) Function() experimental.InternalFunction {
630 return si
631}
632
633// Definition implements the same method as documented on experimental.InternalFunction.
634func (si *stackIterator) Definition() api.FunctionDefinition {
635 return si.currentDef
636}
637
638// SourceOffsetForPC implements the same method as documented on experimental.InternalFunction.
639func (si *stackIterator) SourceOffsetForPC(pc experimental.ProgramCounter) uint64 {
640 upc := uintptr(pc)
641 cm := si.eng.compiledModuleOfAddr(upc)
642 return cm.getSourceOffset(upc)
643}
644
645// snapshot implements experimental.Snapshot
646type snapshot struct {
647 sp, fp, top uintptr
648 returnAddress *byte
649 stack []byte
650 savedRegisters [64][2]uint64
651 ret []uint64
652 c *callEngine
653}
654
655// Snapshot implements the same method as documented on experimental.Snapshotter.
656func (c *callEngine) Snapshot() experimental.Snapshot {
657 returnAddress := c.execCtx.goCallReturnAddress
658 oldTop, oldSp := c.stackTop, uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall))
659 newSP, newFP, newTop, newStack := c.cloneStack(uintptr(len(c.stack)) + 16)
660 adjustClonedStack(oldSp, oldTop, newSP, newFP, newTop)
661 return &snapshot{
662 sp: newSP,
663 fp: newFP,
664 top: newTop,
665 savedRegisters: c.execCtx.savedRegisters,
666 returnAddress: returnAddress,
667 stack: newStack,
668 c: c,
669 }
670}
671
672// Restore implements the same method as documented on experimental.Snapshot.
673func (s *snapshot) Restore(ret []uint64) {
674 s.ret = ret
675 panic(s)
676}
677
678func (s *snapshot) doRestore() {
679 spp := *(**uint64)(unsafe.Pointer(&s.sp))
680 view := goCallStackView(spp)
681 copy(view, s.ret)
682
683 c := s.c
684 c.stack = s.stack
685 c.stackTop = s.top
686 ec := &c.execCtx
687 ec.stackBottomPtr = &c.stack[0]
688 ec.stackPointerBeforeGoCall = spp
689 ec.framePointerBeforeGoCall = s.fp
690 ec.goCallReturnAddress = s.returnAddress
691 ec.savedRegisters = s.savedRegisters
692}
693
694// Error implements the same method on error.
695func (s *snapshot) Error() string {
696 return "unhandled snapshot restore, this generally indicates restore was called from a different " +
697 "exported function invocation than snapshot"
698}
699
700func snapshotRecoverFn(c *callEngine) {
701 if r := recover(); r != nil {
702 if s, ok := r.(*snapshot); ok && s.c == c {
703 s.doRestore()
704 } else {
705 panic(r)
706 }
707 }
708}