1package wazevo
2
3import (
4 "context"
5 "encoding/hex"
6 "errors"
7 "fmt"
8 "runtime"
9 "sort"
10 "sync"
11 "unsafe"
12
13 "github.com/tetratelabs/wazero/api"
14 "github.com/tetratelabs/wazero/experimental"
15 "github.com/tetratelabs/wazero/internal/engine/wazevo/backend"
16 "github.com/tetratelabs/wazero/internal/engine/wazevo/frontend"
17 "github.com/tetratelabs/wazero/internal/engine/wazevo/ssa"
18 "github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi"
19 "github.com/tetratelabs/wazero/internal/filecache"
20 "github.com/tetratelabs/wazero/internal/platform"
21 "github.com/tetratelabs/wazero/internal/version"
22 "github.com/tetratelabs/wazero/internal/wasm"
23)
24
25type (
26 // engine implements wasm.Engine.
27 engine struct {
28 wazeroVersion string
29 fileCache filecache.Cache
30 compiledModules map[wasm.ModuleID]*compiledModule
31 // sortedCompiledModules is a list of compiled modules sorted by the initial address of the executable.
32 sortedCompiledModules []*compiledModule
33 mux sync.RWMutex
34 // sharedFunctions is compiled functions shared by all modules.
35 sharedFunctions *sharedFunctions
36 // setFinalizer defaults to runtime.SetFinalizer, but overridable for tests.
37 setFinalizer func(obj interface{}, finalizer interface{})
38
39 // The followings are reused for compiling shared functions.
40 machine backend.Machine
41 be backend.Compiler
42 }
43
44 sharedFunctions struct {
45 // memoryGrowExecutable is a compiled trampoline executable for memory.grow builtin function.
46 memoryGrowExecutable []byte
47 // checkModuleExitCode is a compiled trampoline executable for checking module instance exit code. This
48 // is used when ensureTermination is true.
49 checkModuleExitCode []byte
50 // stackGrowExecutable is a compiled executable for growing stack builtin function.
51 stackGrowExecutable []byte
52 // tableGrowExecutable is a compiled trampoline executable for table.grow builtin function.
53 tableGrowExecutable []byte
54 // refFuncExecutable is a compiled trampoline executable for ref.func builtin function.
55 refFuncExecutable []byte
56 // memoryWait32Executable is a compiled trampoline executable for memory.wait32 builtin function
57 memoryWait32Executable []byte
58 // memoryWait64Executable is a compiled trampoline executable for memory.wait64 builtin function
59 memoryWait64Executable []byte
60 // memoryNotifyExecutable is a compiled trampoline executable for memory.notify builtin function
61 memoryNotifyExecutable []byte
62 listenerBeforeTrampolines map[*wasm.FunctionType][]byte
63 listenerAfterTrampolines map[*wasm.FunctionType][]byte
64 }
65
66 // compiledModule is a compiled variant of a wasm.Module and ready to be used for instantiation.
67 compiledModule struct {
68 *executables
69 // functionOffsets maps a local function index to the offset in the executable.
70 functionOffsets []int
71 parent *engine
72 module *wasm.Module
73 ensureTermination bool
74 listeners []experimental.FunctionListener
75 listenerBeforeTrampolines []*byte
76 listenerAfterTrampolines []*byte
77
78 // The followings are only available for non host modules.
79
80 offsets wazevoapi.ModuleContextOffsetData
81 sharedFunctions *sharedFunctions
82 sourceMap sourceMap
83 }
84
85 executables struct {
86 executable []byte
87 entryPreambles [][]byte
88 }
89)
90
91// sourceMap is a mapping from the offset of the executable to the offset of the original wasm binary.
92type sourceMap struct {
93 // executableOffsets is a sorted list of offsets of the executable. This is index-correlated with wasmBinaryOffsets,
94 // in other words executableOffsets[i] is the offset of the executable which corresponds to the offset of a Wasm
95 // binary pointed by wasmBinaryOffsets[i].
96 executableOffsets []uintptr
97 // wasmBinaryOffsets is the counterpart of executableOffsets.
98 wasmBinaryOffsets []uint64
99}
100
101var _ wasm.Engine = (*engine)(nil)
102
103// NewEngine returns the implementation of wasm.Engine.
104func NewEngine(ctx context.Context, _ api.CoreFeatures, fc filecache.Cache) wasm.Engine {
105 machine := newMachine()
106 be := backend.NewCompiler(ctx, machine, ssa.NewBuilder())
107 e := &engine{
108 compiledModules: make(map[wasm.ModuleID]*compiledModule),
109 setFinalizer: runtime.SetFinalizer,
110 machine: machine,
111 be: be,
112 fileCache: fc,
113 wazeroVersion: version.GetWazeroVersion(),
114 }
115 e.compileSharedFunctions()
116 return e
117}
118
119// CompileModule implements wasm.Engine.
120func (e *engine) CompileModule(ctx context.Context, module *wasm.Module, listeners []experimental.FunctionListener, ensureTermination bool) (err error) {
121 if wazevoapi.PerfMapEnabled {
122 wazevoapi.PerfMap.Lock()
123 defer wazevoapi.PerfMap.Unlock()
124 }
125
126 if _, ok, err := e.getCompiledModule(module, listeners, ensureTermination); ok { // cache hit!
127 return nil
128 } else if err != nil {
129 return err
130 }
131
132 if wazevoapi.DeterministicCompilationVerifierEnabled {
133 ctx = wazevoapi.NewDeterministicCompilationVerifierContext(ctx, len(module.CodeSection))
134 }
135 cm, err := e.compileModule(ctx, module, listeners, ensureTermination)
136 if err != nil {
137 return err
138 }
139 if err = e.addCompiledModule(module, cm); err != nil {
140 return err
141 }
142
143 if wazevoapi.DeterministicCompilationVerifierEnabled {
144 for i := 0; i < wazevoapi.DeterministicCompilationVerifyingIter; i++ {
145 _, err := e.compileModule(ctx, module, listeners, ensureTermination)
146 if err != nil {
147 return err
148 }
149 }
150 }
151
152 if len(listeners) > 0 {
153 cm.listeners = listeners
154 cm.listenerBeforeTrampolines = make([]*byte, len(module.TypeSection))
155 cm.listenerAfterTrampolines = make([]*byte, len(module.TypeSection))
156 for i := range module.TypeSection {
157 typ := &module.TypeSection[i]
158 before, after := e.getListenerTrampolineForType(typ)
159 cm.listenerBeforeTrampolines[i] = before
160 cm.listenerAfterTrampolines[i] = after
161 }
162 }
163 return nil
164}
165
166func (exec *executables) compileEntryPreambles(m *wasm.Module, machine backend.Machine, be backend.Compiler) {
167 exec.entryPreambles = make([][]byte, len(m.TypeSection))
168 for i := range m.TypeSection {
169 typ := &m.TypeSection[i]
170 sig := frontend.SignatureForWasmFunctionType(typ)
171 be.Init()
172 buf := machine.CompileEntryPreamble(&sig)
173 executable := mmapExecutable(buf)
174 exec.entryPreambles[i] = executable
175
176 if wazevoapi.PerfMapEnabled {
177 wazevoapi.PerfMap.AddEntry(uintptr(unsafe.Pointer(&executable[0])),
178 uint64(len(executable)), fmt.Sprintf("entry_preamble::type=%s", typ.String()))
179 }
180 }
181}
182
183func (e *engine) compileModule(ctx context.Context, module *wasm.Module, listeners []experimental.FunctionListener, ensureTermination bool) (*compiledModule, error) {
184 withListener := len(listeners) > 0
185 cm := &compiledModule{
186 offsets: wazevoapi.NewModuleContextOffsetData(module, withListener), parent: e, module: module,
187 ensureTermination: ensureTermination,
188 executables: &executables{},
189 }
190
191 if module.IsHostModule {
192 return e.compileHostModule(ctx, module, listeners)
193 }
194
195 importedFns, localFns := int(module.ImportFunctionCount), len(module.FunctionSection)
196 if localFns == 0 {
197 return cm, nil
198 }
199
200 rels := make([]backend.RelocationInfo, 0)
201 refToBinaryOffset := make([]int, importedFns+localFns)
202
203 if wazevoapi.DeterministicCompilationVerifierEnabled {
204 // The compilation must be deterministic regardless of the order of functions being compiled.
205 wazevoapi.DeterministicCompilationVerifierRandomizeIndexes(ctx)
206 }
207
208 needSourceInfo := module.DWARFLines != nil
209
210 // Creates new compiler instances which are reused for each function.
211 ssaBuilder := ssa.NewBuilder()
212 fe := frontend.NewFrontendCompiler(module, ssaBuilder, &cm.offsets, ensureTermination, withListener, needSourceInfo)
213 machine := newMachine()
214 be := backend.NewCompiler(ctx, machine, ssaBuilder)
215
216 cm.executables.compileEntryPreambles(module, machine, be)
217
218 totalSize := 0 // Total binary size of the executable.
219 cm.functionOffsets = make([]int, localFns)
220 bodies := make([][]byte, localFns)
221
222 // Trampoline relocation related variables.
223 trampolineInterval, callTrampolineIslandSize, err := machine.CallTrampolineIslandInfo(localFns)
224 if err != nil {
225 return nil, err
226 }
227 needCallTrampoline := callTrampolineIslandSize > 0
228 var callTrampolineIslandOffsets []int // Holds the offsets of trampoline islands.
229
230 for i := range module.CodeSection {
231 if wazevoapi.DeterministicCompilationVerifierEnabled {
232 i = wazevoapi.DeterministicCompilationVerifierGetRandomizedLocalFunctionIndex(ctx, i)
233 }
234
235 fidx := wasm.Index(i + importedFns)
236
237 if wazevoapi.NeedFunctionNameInContext {
238 def := module.FunctionDefinition(fidx)
239 name := def.DebugName()
240 if len(def.ExportNames()) > 0 {
241 name = def.ExportNames()[0]
242 }
243 ctx = wazevoapi.SetCurrentFunctionName(ctx, i, fmt.Sprintf("[%d/%d]%s", i, len(module.CodeSection)-1, name))
244 }
245
246 needListener := len(listeners) > 0 && listeners[i] != nil
247 body, relsPerFunc, err := e.compileLocalWasmFunction(ctx, module, wasm.Index(i), fe, ssaBuilder, be, needListener)
248 if err != nil {
249 return nil, fmt.Errorf("compile function %d/%d: %v", i, len(module.CodeSection)-1, err)
250 }
251
252 // Align 16-bytes boundary.
253 totalSize = (totalSize + 15) &^ 15
254 cm.functionOffsets[i] = totalSize
255
256 if needSourceInfo {
257 // At the beginning of the function, we add the offset of the function body so that
258 // we can resolve the source location of the call site of before listener call.
259 cm.sourceMap.executableOffsets = append(cm.sourceMap.executableOffsets, uintptr(totalSize))
260 cm.sourceMap.wasmBinaryOffsets = append(cm.sourceMap.wasmBinaryOffsets, module.CodeSection[i].BodyOffsetInCodeSection)
261
262 for _, info := range be.SourceOffsetInfo() {
263 cm.sourceMap.executableOffsets = append(cm.sourceMap.executableOffsets, uintptr(totalSize)+uintptr(info.ExecutableOffset))
264 cm.sourceMap.wasmBinaryOffsets = append(cm.sourceMap.wasmBinaryOffsets, uint64(info.SourceOffset))
265 }
266 }
267
268 fref := frontend.FunctionIndexToFuncRef(fidx)
269 refToBinaryOffset[fref] = totalSize
270
271 // At this point, relocation offsets are relative to the start of the function body,
272 // so we adjust it to the start of the executable.
273 for _, r := range relsPerFunc {
274 r.Offset += int64(totalSize)
275 rels = append(rels, r)
276 }
277
278 bodies[i] = body
279 totalSize += len(body)
280 if wazevoapi.PrintMachineCodeHexPerFunction {
281 fmt.Printf("[[[machine code for %s]]]\n%s\n\n", wazevoapi.GetCurrentFunctionName(ctx), hex.EncodeToString(body))
282 }
283
284 if needCallTrampoline {
285 // If the total size exceeds the trampoline interval, we need to add a trampoline island.
286 if totalSize/trampolineInterval > len(callTrampolineIslandOffsets) {
287 callTrampolineIslandOffsets = append(callTrampolineIslandOffsets, totalSize)
288 totalSize += callTrampolineIslandSize
289 }
290 }
291 }
292
293 // Allocate executable memory and then copy the generated machine code.
294 executable, err := platform.MmapCodeSegment(totalSize)
295 if err != nil {
296 panic(err)
297 }
298 cm.executable = executable
299
300 for i, b := range bodies {
301 offset := cm.functionOffsets[i]
302 copy(executable[offset:], b)
303 }
304
305 if wazevoapi.PerfMapEnabled {
306 wazevoapi.PerfMap.Flush(uintptr(unsafe.Pointer(&executable[0])), cm.functionOffsets)
307 }
308
309 if needSourceInfo {
310 for i := range cm.sourceMap.executableOffsets {
311 cm.sourceMap.executableOffsets[i] += uintptr(unsafe.Pointer(&cm.executable[0]))
312 }
313 }
314
315 // Resolve relocations for local function calls.
316 if len(rels) > 0 {
317 machine.ResolveRelocations(refToBinaryOffset, importedFns, executable, rels, callTrampolineIslandOffsets)
318 }
319
320 if runtime.GOARCH == "arm64" {
321 // On arm64, we cannot give all of rwx at the same time, so we change it to exec.
322 if err = platform.MprotectRX(executable); err != nil {
323 return nil, err
324 }
325 }
326 cm.sharedFunctions = e.sharedFunctions
327 e.setFinalizer(cm.executables, executablesFinalizer)
328 return cm, nil
329}
330
331func (e *engine) compileLocalWasmFunction(
332 ctx context.Context,
333 module *wasm.Module,
334 localFunctionIndex wasm.Index,
335 fe *frontend.Compiler,
336 ssaBuilder ssa.Builder,
337 be backend.Compiler,
338 needListener bool,
339) (body []byte, rels []backend.RelocationInfo, err error) {
340 typIndex := module.FunctionSection[localFunctionIndex]
341 typ := &module.TypeSection[typIndex]
342 codeSeg := &module.CodeSection[localFunctionIndex]
343
344 // Initializes both frontend and backend compilers.
345 fe.Init(localFunctionIndex, typIndex, typ, codeSeg.LocalTypes, codeSeg.Body, needListener, codeSeg.BodyOffsetInCodeSection)
346 be.Init()
347
348 // Lower Wasm to SSA.
349 fe.LowerToSSA()
350 if wazevoapi.PrintSSA && wazevoapi.PrintEnabledIndex(ctx) {
351 fmt.Printf("[[[SSA for %s]]]%s\n", wazevoapi.GetCurrentFunctionName(ctx), ssaBuilder.Format())
352 }
353
354 if wazevoapi.DeterministicCompilationVerifierEnabled {
355 wazevoapi.VerifyOrSetDeterministicCompilationContextValue(ctx, "SSA", ssaBuilder.Format())
356 }
357
358 // Run SSA-level optimization passes.
359 ssaBuilder.RunPasses()
360
361 if wazevoapi.PrintOptimizedSSA && wazevoapi.PrintEnabledIndex(ctx) {
362 fmt.Printf("[[[Optimized SSA for %s]]]%s\n", wazevoapi.GetCurrentFunctionName(ctx), ssaBuilder.Format())
363 }
364
365 if wazevoapi.DeterministicCompilationVerifierEnabled {
366 wazevoapi.VerifyOrSetDeterministicCompilationContextValue(ctx, "Optimized SSA", ssaBuilder.Format())
367 }
368
369 // Now our ssaBuilder contains the necessary information to further lower them to
370 // machine code.
371 original, rels, err := be.Compile(ctx)
372 if err != nil {
373 return nil, nil, fmt.Errorf("ssa->machine code: %v", err)
374 }
375
376 // TODO: optimize as zero copy.
377 copied := make([]byte, len(original))
378 copy(copied, original)
379 return copied, rels, nil
380}
381
382func (e *engine) compileHostModule(ctx context.Context, module *wasm.Module, listeners []experimental.FunctionListener) (*compiledModule, error) {
383 machine := newMachine()
384 be := backend.NewCompiler(ctx, machine, ssa.NewBuilder())
385
386 num := len(module.CodeSection)
387 cm := &compiledModule{module: module, listeners: listeners, executables: &executables{}}
388 cm.functionOffsets = make([]int, num)
389 totalSize := 0 // Total binary size of the executable.
390 bodies := make([][]byte, num)
391 var sig ssa.Signature
392 for i := range module.CodeSection {
393 totalSize = (totalSize + 15) &^ 15
394 cm.functionOffsets[i] = totalSize
395
396 typIndex := module.FunctionSection[i]
397 typ := &module.TypeSection[typIndex]
398
399 // We can relax until the index fits together in ExitCode as we do in wazevoapi.ExitCodeCallGoModuleFunctionWithIndex.
400 // However, 1 << 16 should be large enough for a real use case.
401 const hostFunctionNumMaximum = 1 << 16
402 if i >= hostFunctionNumMaximum {
403 return nil, fmt.Errorf("too many host functions (maximum %d)", hostFunctionNumMaximum)
404 }
405
406 sig.ID = ssa.SignatureID(typIndex) // This is important since we reuse the `machine` which caches the ABI based on the SignatureID.
407 sig.Params = append(sig.Params[:0],
408 ssa.TypeI64, // First argument must be exec context.
409 ssa.TypeI64, // The second argument is the moduleContextOpaque of this host module.
410 )
411 for _, t := range typ.Params {
412 sig.Params = append(sig.Params, frontend.WasmTypeToSSAType(t))
413 }
414
415 sig.Results = sig.Results[:0]
416 for _, t := range typ.Results {
417 sig.Results = append(sig.Results, frontend.WasmTypeToSSAType(t))
418 }
419
420 c := &module.CodeSection[i]
421 if c.GoFunc == nil {
422 panic("BUG: GoFunc must be set for host module")
423 }
424
425 withListener := len(listeners) > 0 && listeners[i] != nil
426 var exitCode wazevoapi.ExitCode
427 fn := c.GoFunc
428 switch fn.(type) {
429 case api.GoModuleFunction:
430 exitCode = wazevoapi.ExitCodeCallGoModuleFunctionWithIndex(i, withListener)
431 case api.GoFunction:
432 exitCode = wazevoapi.ExitCodeCallGoFunctionWithIndex(i, withListener)
433 }
434
435 be.Init()
436 machine.CompileGoFunctionTrampoline(exitCode, &sig, true)
437 if err := be.Finalize(ctx); err != nil {
438 return nil, err
439 }
440 body := be.Buf()
441
442 if wazevoapi.PerfMapEnabled {
443 name := module.FunctionDefinition(wasm.Index(i)).DebugName()
444 wazevoapi.PerfMap.AddModuleEntry(i,
445 int64(totalSize),
446 uint64(len(body)),
447 fmt.Sprintf("trampoline:%s", name))
448 }
449
450 // TODO: optimize as zero copy.
451 copied := make([]byte, len(body))
452 copy(copied, body)
453 bodies[i] = copied
454 totalSize += len(body)
455 }
456
457 if totalSize == 0 {
458 // Empty module.
459 return cm, nil
460 }
461
462 // Allocate executable memory and then copy the generated machine code.
463 executable, err := platform.MmapCodeSegment(totalSize)
464 if err != nil {
465 panic(err)
466 }
467 cm.executable = executable
468
469 for i, b := range bodies {
470 offset := cm.functionOffsets[i]
471 copy(executable[offset:], b)
472 }
473
474 if wazevoapi.PerfMapEnabled {
475 wazevoapi.PerfMap.Flush(uintptr(unsafe.Pointer(&executable[0])), cm.functionOffsets)
476 }
477
478 if runtime.GOARCH == "arm64" {
479 // On arm64, we cannot give all of rwx at the same time, so we change it to exec.
480 if err = platform.MprotectRX(executable); err != nil {
481 return nil, err
482 }
483 }
484 e.setFinalizer(cm.executables, executablesFinalizer)
485 return cm, nil
486}
487
488// Close implements wasm.Engine.
489func (e *engine) Close() (err error) {
490 e.mux.Lock()
491 defer e.mux.Unlock()
492 e.sortedCompiledModules = nil
493 e.compiledModules = nil
494 e.sharedFunctions = nil
495 return nil
496}
497
498// CompiledModuleCount implements wasm.Engine.
499func (e *engine) CompiledModuleCount() uint32 {
500 e.mux.RLock()
501 defer e.mux.RUnlock()
502 return uint32(len(e.compiledModules))
503}
504
505// DeleteCompiledModule implements wasm.Engine.
506func (e *engine) DeleteCompiledModule(m *wasm.Module) {
507 e.mux.Lock()
508 defer e.mux.Unlock()
509 cm, ok := e.compiledModules[m.ID]
510 if ok {
511 if len(cm.executable) > 0 {
512 e.deleteCompiledModuleFromSortedList(cm)
513 }
514 delete(e.compiledModules, m.ID)
515 }
516}
517
518func (e *engine) addCompiledModuleToSortedList(cm *compiledModule) {
519 ptr := uintptr(unsafe.Pointer(&cm.executable[0]))
520
521 index := sort.Search(len(e.sortedCompiledModules), func(i int) bool {
522 return uintptr(unsafe.Pointer(&e.sortedCompiledModules[i].executable[0])) >= ptr
523 })
524 e.sortedCompiledModules = append(e.sortedCompiledModules, nil)
525 copy(e.sortedCompiledModules[index+1:], e.sortedCompiledModules[index:])
526 e.sortedCompiledModules[index] = cm
527}
528
529func (e *engine) deleteCompiledModuleFromSortedList(cm *compiledModule) {
530 ptr := uintptr(unsafe.Pointer(&cm.executable[0]))
531
532 index := sort.Search(len(e.sortedCompiledModules), func(i int) bool {
533 return uintptr(unsafe.Pointer(&e.sortedCompiledModules[i].executable[0])) >= ptr
534 })
535 if index >= len(e.sortedCompiledModules) {
536 return
537 }
538 copy(e.sortedCompiledModules[index:], e.sortedCompiledModules[index+1:])
539 e.sortedCompiledModules = e.sortedCompiledModules[:len(e.sortedCompiledModules)-1]
540}
541
542func (e *engine) compiledModuleOfAddr(addr uintptr) *compiledModule {
543 e.mux.RLock()
544 defer e.mux.RUnlock()
545
546 index := sort.Search(len(e.sortedCompiledModules), func(i int) bool {
547 return uintptr(unsafe.Pointer(&e.sortedCompiledModules[i].executable[0])) > addr
548 })
549 index -= 1
550 if index < 0 {
551 return nil
552 }
553 candidate := e.sortedCompiledModules[index]
554 if checkAddrInBytes(addr, candidate.executable) {
555 // If a module is already deleted, the found module may have been wrong.
556 return candidate
557 }
558 return nil
559}
560
561func checkAddrInBytes(addr uintptr, b []byte) bool {
562 return uintptr(unsafe.Pointer(&b[0])) <= addr && addr <= uintptr(unsafe.Pointer(&b[len(b)-1]))
563}
564
565// NewModuleEngine implements wasm.Engine.
566func (e *engine) NewModuleEngine(m *wasm.Module, mi *wasm.ModuleInstance) (wasm.ModuleEngine, error) {
567 me := &moduleEngine{}
568
569 // Note: imported functions are resolved in moduleEngine.ResolveImportedFunction.
570 me.importedFunctions = make([]importedFunction, m.ImportFunctionCount)
571
572 compiled, ok := e.getCompiledModuleFromMemory(m)
573 if !ok {
574 return nil, errors.New("source module must be compiled before instantiation")
575 }
576 me.parent = compiled
577 me.module = mi
578 me.listeners = compiled.listeners
579
580 if m.IsHostModule {
581 me.opaque = buildHostModuleOpaque(m, compiled.listeners)
582 me.opaquePtr = &me.opaque[0]
583 } else {
584 if size := compiled.offsets.TotalSize; size != 0 {
585 opaque := newAlignedOpaque(size)
586 me.opaque = opaque
587 me.opaquePtr = &opaque[0]
588 }
589 }
590 return me, nil
591}
592
593func (e *engine) compileSharedFunctions() {
594 e.sharedFunctions = &sharedFunctions{
595 listenerBeforeTrampolines: make(map[*wasm.FunctionType][]byte),
596 listenerAfterTrampolines: make(map[*wasm.FunctionType][]byte),
597 }
598
599 e.be.Init()
600 {
601 src := e.machine.CompileGoFunctionTrampoline(wazevoapi.ExitCodeGrowMemory, &ssa.Signature{
602 Params: []ssa.Type{ssa.TypeI64 /* exec context */, ssa.TypeI32},
603 Results: []ssa.Type{ssa.TypeI32},
604 }, false)
605 e.sharedFunctions.memoryGrowExecutable = mmapExecutable(src)
606 if wazevoapi.PerfMapEnabled {
607 exe := e.sharedFunctions.memoryGrowExecutable
608 wazevoapi.PerfMap.AddEntry(uintptr(unsafe.Pointer(&exe[0])), uint64(len(exe)), "memory_grow_trampoline")
609 }
610 }
611
612 e.be.Init()
613 {
614 src := e.machine.CompileGoFunctionTrampoline(wazevoapi.ExitCodeTableGrow, &ssa.Signature{
615 Params: []ssa.Type{ssa.TypeI64 /* exec context */, ssa.TypeI32 /* table index */, ssa.TypeI32 /* num */, ssa.TypeI64 /* ref */},
616 Results: []ssa.Type{ssa.TypeI32},
617 }, false)
618 e.sharedFunctions.tableGrowExecutable = mmapExecutable(src)
619 if wazevoapi.PerfMapEnabled {
620 exe := e.sharedFunctions.tableGrowExecutable
621 wazevoapi.PerfMap.AddEntry(uintptr(unsafe.Pointer(&exe[0])), uint64(len(exe)), "table_grow_trampoline")
622 }
623 }
624
625 e.be.Init()
626 {
627 src := e.machine.CompileGoFunctionTrampoline(wazevoapi.ExitCodeCheckModuleExitCode, &ssa.Signature{
628 Params: []ssa.Type{ssa.TypeI32 /* exec context */},
629 Results: []ssa.Type{ssa.TypeI32},
630 }, false)
631 e.sharedFunctions.checkModuleExitCode = mmapExecutable(src)
632 if wazevoapi.PerfMapEnabled {
633 exe := e.sharedFunctions.checkModuleExitCode
634 wazevoapi.PerfMap.AddEntry(uintptr(unsafe.Pointer(&exe[0])), uint64(len(exe)), "check_module_exit_code_trampoline")
635 }
636 }
637
638 e.be.Init()
639 {
640 src := e.machine.CompileGoFunctionTrampoline(wazevoapi.ExitCodeRefFunc, &ssa.Signature{
641 Params: []ssa.Type{ssa.TypeI64 /* exec context */, ssa.TypeI32 /* function index */},
642 Results: []ssa.Type{ssa.TypeI64}, // returns the function reference.
643 }, false)
644 e.sharedFunctions.refFuncExecutable = mmapExecutable(src)
645 if wazevoapi.PerfMapEnabled {
646 exe := e.sharedFunctions.refFuncExecutable
647 wazevoapi.PerfMap.AddEntry(uintptr(unsafe.Pointer(&exe[0])), uint64(len(exe)), "ref_func_trampoline")
648 }
649 }
650
651 e.be.Init()
652 {
653 src := e.machine.CompileStackGrowCallSequence()
654 e.sharedFunctions.stackGrowExecutable = mmapExecutable(src)
655 if wazevoapi.PerfMapEnabled {
656 exe := e.sharedFunctions.stackGrowExecutable
657 wazevoapi.PerfMap.AddEntry(uintptr(unsafe.Pointer(&exe[0])), uint64(len(exe)), "stack_grow_trampoline")
658 }
659 }
660
661 e.be.Init()
662 {
663 src := e.machine.CompileGoFunctionTrampoline(wazevoapi.ExitCodeMemoryWait32, &ssa.Signature{
664 // exec context, timeout, expected, addr
665 Params: []ssa.Type{ssa.TypeI64, ssa.TypeI64, ssa.TypeI32, ssa.TypeI64},
666 // Returns the status.
667 Results: []ssa.Type{ssa.TypeI32},
668 }, false)
669 e.sharedFunctions.memoryWait32Executable = mmapExecutable(src)
670 if wazevoapi.PerfMapEnabled {
671 exe := e.sharedFunctions.memoryWait32Executable
672 wazevoapi.PerfMap.AddEntry(uintptr(unsafe.Pointer(&exe[0])), uint64(len(exe)), "memory_wait32_trampoline")
673 }
674 }
675
676 e.be.Init()
677 {
678 src := e.machine.CompileGoFunctionTrampoline(wazevoapi.ExitCodeMemoryWait64, &ssa.Signature{
679 // exec context, timeout, expected, addr
680 Params: []ssa.Type{ssa.TypeI64, ssa.TypeI64, ssa.TypeI64, ssa.TypeI64},
681 // Returns the status.
682 Results: []ssa.Type{ssa.TypeI32},
683 }, false)
684 e.sharedFunctions.memoryWait64Executable = mmapExecutable(src)
685 if wazevoapi.PerfMapEnabled {
686 exe := e.sharedFunctions.memoryWait64Executable
687 wazevoapi.PerfMap.AddEntry(uintptr(unsafe.Pointer(&exe[0])), uint64(len(exe)), "memory_wait64_trampoline")
688 }
689 }
690
691 e.be.Init()
692 {
693 src := e.machine.CompileGoFunctionTrampoline(wazevoapi.ExitCodeMemoryNotify, &ssa.Signature{
694 // exec context, count, addr
695 Params: []ssa.Type{ssa.TypeI64, ssa.TypeI32, ssa.TypeI64},
696 // Returns the number notified.
697 Results: []ssa.Type{ssa.TypeI32},
698 }, false)
699 e.sharedFunctions.memoryNotifyExecutable = mmapExecutable(src)
700 if wazevoapi.PerfMapEnabled {
701 exe := e.sharedFunctions.memoryNotifyExecutable
702 wazevoapi.PerfMap.AddEntry(uintptr(unsafe.Pointer(&exe[0])), uint64(len(exe)), "memory_notify_trampoline")
703 }
704 }
705
706 e.setFinalizer(e.sharedFunctions, sharedFunctionsFinalizer)
707}
708
709func sharedFunctionsFinalizer(sf *sharedFunctions) {
710 if err := platform.MunmapCodeSegment(sf.memoryGrowExecutable); err != nil {
711 panic(err)
712 }
713 if err := platform.MunmapCodeSegment(sf.checkModuleExitCode); err != nil {
714 panic(err)
715 }
716 if err := platform.MunmapCodeSegment(sf.stackGrowExecutable); err != nil {
717 panic(err)
718 }
719 if err := platform.MunmapCodeSegment(sf.tableGrowExecutable); err != nil {
720 panic(err)
721 }
722 if err := platform.MunmapCodeSegment(sf.refFuncExecutable); err != nil {
723 panic(err)
724 }
725 if err := platform.MunmapCodeSegment(sf.memoryWait32Executable); err != nil {
726 panic(err)
727 }
728 if err := platform.MunmapCodeSegment(sf.memoryWait64Executable); err != nil {
729 panic(err)
730 }
731 if err := platform.MunmapCodeSegment(sf.memoryNotifyExecutable); err != nil {
732 panic(err)
733 }
734 for _, f := range sf.listenerBeforeTrampolines {
735 if err := platform.MunmapCodeSegment(f); err != nil {
736 panic(err)
737 }
738 }
739 for _, f := range sf.listenerAfterTrampolines {
740 if err := platform.MunmapCodeSegment(f); err != nil {
741 panic(err)
742 }
743 }
744
745 sf.memoryGrowExecutable = nil
746 sf.checkModuleExitCode = nil
747 sf.stackGrowExecutable = nil
748 sf.tableGrowExecutable = nil
749 sf.refFuncExecutable = nil
750 sf.memoryWait32Executable = nil
751 sf.memoryWait64Executable = nil
752 sf.memoryNotifyExecutable = nil
753 sf.listenerBeforeTrampolines = nil
754 sf.listenerAfterTrampolines = nil
755}
756
757func executablesFinalizer(exec *executables) {
758 if len(exec.executable) > 0 {
759 if err := platform.MunmapCodeSegment(exec.executable); err != nil {
760 panic(err)
761 }
762 }
763 exec.executable = nil
764
765 for _, f := range exec.entryPreambles {
766 if err := platform.MunmapCodeSegment(f); err != nil {
767 panic(err)
768 }
769 }
770 exec.entryPreambles = nil
771}
772
773func mmapExecutable(src []byte) []byte {
774 executable, err := platform.MmapCodeSegment(len(src))
775 if err != nil {
776 panic(err)
777 }
778
779 copy(executable, src)
780
781 if runtime.GOARCH == "arm64" {
782 // On arm64, we cannot give all of rwx at the same time, so we change it to exec.
783 if err = platform.MprotectRX(executable); err != nil {
784 panic(err)
785 }
786 }
787 return executable
788}
789
790func (cm *compiledModule) functionIndexOf(addr uintptr) wasm.Index {
791 addr -= uintptr(unsafe.Pointer(&cm.executable[0]))
792 offset := cm.functionOffsets
793 index := sort.Search(len(offset), func(i int) bool {
794 return offset[i] > int(addr)
795 })
796 index--
797 if index < 0 {
798 panic("BUG")
799 }
800 return wasm.Index(index)
801}
802
803func (e *engine) getListenerTrampolineForType(functionType *wasm.FunctionType) (before, after *byte) {
804 e.mux.Lock()
805 defer e.mux.Unlock()
806
807 beforeBuf, ok := e.sharedFunctions.listenerBeforeTrampolines[functionType]
808 afterBuf := e.sharedFunctions.listenerAfterTrampolines[functionType]
809 if ok {
810 return &beforeBuf[0], &afterBuf[0]
811 }
812
813 beforeSig, afterSig := frontend.SignatureForListener(functionType)
814
815 e.be.Init()
816 buf := e.machine.CompileGoFunctionTrampoline(wazevoapi.ExitCodeCallListenerBefore, beforeSig, false)
817 beforeBuf = mmapExecutable(buf)
818
819 e.be.Init()
820 buf = e.machine.CompileGoFunctionTrampoline(wazevoapi.ExitCodeCallListenerAfter, afterSig, false)
821 afterBuf = mmapExecutable(buf)
822
823 e.sharedFunctions.listenerBeforeTrampolines[functionType] = beforeBuf
824 e.sharedFunctions.listenerAfterTrampolines[functionType] = afterBuf
825 return &beforeBuf[0], &afterBuf[0]
826}
827
828func (cm *compiledModule) getSourceOffset(pc uintptr) uint64 {
829 offsets := cm.sourceMap.executableOffsets
830 if len(offsets) == 0 {
831 return 0
832 }
833
834 index := sort.Search(len(offsets), func(i int) bool {
835 return offsets[i] >= pc
836 })
837
838 index--
839 if index < 0 {
840 return 0
841 }
842 return cm.sourceMap.wasmBinaryOffsets[index]
843}