1package wasm
2
3import (
4 "container/list"
5 "encoding/binary"
6 "fmt"
7 "math"
8 "reflect"
9 "sync"
10 "sync/atomic"
11 "time"
12 "unsafe"
13
14 "github.com/tetratelabs/wazero/api"
15 "github.com/tetratelabs/wazero/experimental"
16 "github.com/tetratelabs/wazero/internal/internalapi"
17 "github.com/tetratelabs/wazero/internal/wasmruntime"
18)
19
20const (
21 // MemoryPageSize is the unit of memory length in WebAssembly,
22 // and is defined as 2^16 = 65536.
23 // See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#memory-instances%E2%91%A0
24 MemoryPageSize = uint32(65536)
25 // MemoryLimitPages is maximum number of pages defined (2^16).
26 // See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#grow-mem
27 MemoryLimitPages = uint32(65536)
28 // MemoryPageSizeInBits satisfies the relation: "1 << MemoryPageSizeInBits == MemoryPageSize".
29 MemoryPageSizeInBits = 16
30)
31
32// compile-time check to ensure MemoryInstance implements api.Memory
33var _ api.Memory = &MemoryInstance{}
34
35type waiters struct {
36 mux sync.Mutex
37 l *list.List
38}
39
40// MemoryInstance represents a memory instance in a store, and implements api.Memory.
41//
42// Note: In WebAssembly 1.0 (20191205), there may be up to one Memory per store, which means the precise memory is always
43// wasm.Store Memories index zero: `store.Memories[0]`
44// See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#memory-instances%E2%91%A0.
45type MemoryInstance struct {
46 internalapi.WazeroOnlyType
47
48 Buffer []byte
49 Min, Cap, Max uint32
50 Shared bool
51 // definition is known at compile time.
52 definition api.MemoryDefinition
53
54 // Mux is used in interpreter mode to prevent overlapping calls to atomic instructions,
55 // introduced with WebAssembly threads proposal, and in compiler mode to make memory modifications
56 // within Grow non-racy for the Go race detector.
57 Mux sync.Mutex
58
59 // waiters implements atomic wait and notify. It is implemented similarly to golang.org/x/sync/semaphore,
60 // with a fixed weight of 1 and no spurious notifications.
61 waiters sync.Map
62
63 // ownerModuleEngine is the module engine that owns this memory instance.
64 ownerModuleEngine ModuleEngine
65
66 expBuffer experimental.LinearMemory
67}
68
69// NewMemoryInstance creates a new instance based on the parameters in the SectionIDMemory.
70func NewMemoryInstance(memSec *Memory, allocator experimental.MemoryAllocator, moduleEngine ModuleEngine) *MemoryInstance {
71 minBytes := MemoryPagesToBytesNum(memSec.Min)
72 capBytes := MemoryPagesToBytesNum(memSec.Cap)
73 maxBytes := MemoryPagesToBytesNum(memSec.Max)
74
75 var buffer []byte
76 var expBuffer experimental.LinearMemory
77 if allocator != nil {
78 expBuffer = allocator.Allocate(capBytes, maxBytes)
79 buffer = expBuffer.Reallocate(minBytes)
80 _ = buffer[:minBytes] // Bounds check that the minimum was allocated.
81 } else if memSec.IsShared {
82 // Shared memory needs a fixed buffer, so allocate with the maximum size.
83 //
84 // The rationale as to why we can simply use make([]byte) to a fixed buffer is that Go's GC is non-relocating.
85 // That is not a part of Go spec, but is well-known thing in Go community (wazero's compiler heavily relies on it!)
86 // * https://github.com/go4org/unsafe-assume-no-moving-gc
87 //
88 // Also, allocating Max here isn't harmful as the Go runtime uses mmap for large allocations, therefore,
89 // the memory buffer allocation here is virtual and doesn't consume physical memory until it's used.
90 // * https://github.com/golang/go/blob/8121604559035734c9677d5281bbdac8b1c17a1e/src/runtime/malloc.go#L1059
91 // * https://github.com/golang/go/blob/8121604559035734c9677d5281bbdac8b1c17a1e/src/runtime/malloc.go#L1165
92 buffer = make([]byte, minBytes, maxBytes)
93 } else {
94 buffer = make([]byte, minBytes, capBytes)
95 }
96 return &MemoryInstance{
97 Buffer: buffer,
98 Min: memSec.Min,
99 Cap: memoryBytesNumToPages(uint64(cap(buffer))),
100 Max: memSec.Max,
101 Shared: memSec.IsShared,
102 expBuffer: expBuffer,
103 ownerModuleEngine: moduleEngine,
104 }
105}
106
107// Definition implements the same method as documented on api.Memory.
108func (m *MemoryInstance) Definition() api.MemoryDefinition {
109 return m.definition
110}
111
112// Size implements the same method as documented on api.Memory.
113func (m *MemoryInstance) Size() uint32 {
114 return uint32(len(m.Buffer))
115}
116
117// ReadByte implements the same method as documented on api.Memory.
118func (m *MemoryInstance) ReadByte(offset uint32) (byte, bool) {
119 if !m.hasSize(offset, 1) {
120 return 0, false
121 }
122 return m.Buffer[offset], true
123}
124
125// ReadUint16Le implements the same method as documented on api.Memory.
126func (m *MemoryInstance) ReadUint16Le(offset uint32) (uint16, bool) {
127 if !m.hasSize(offset, 2) {
128 return 0, false
129 }
130 return binary.LittleEndian.Uint16(m.Buffer[offset : offset+2]), true
131}
132
133// ReadUint32Le implements the same method as documented on api.Memory.
134func (m *MemoryInstance) ReadUint32Le(offset uint32) (uint32, bool) {
135 return m.readUint32Le(offset)
136}
137
138// ReadFloat32Le implements the same method as documented on api.Memory.
139func (m *MemoryInstance) ReadFloat32Le(offset uint32) (float32, bool) {
140 v, ok := m.readUint32Le(offset)
141 if !ok {
142 return 0, false
143 }
144 return math.Float32frombits(v), true
145}
146
147// ReadUint64Le implements the same method as documented on api.Memory.
148func (m *MemoryInstance) ReadUint64Le(offset uint32) (uint64, bool) {
149 return m.readUint64Le(offset)
150}
151
152// ReadFloat64Le implements the same method as documented on api.Memory.
153func (m *MemoryInstance) ReadFloat64Le(offset uint32) (float64, bool) {
154 v, ok := m.readUint64Le(offset)
155 if !ok {
156 return 0, false
157 }
158 return math.Float64frombits(v), true
159}
160
161// Read implements the same method as documented on api.Memory.
162func (m *MemoryInstance) Read(offset, byteCount uint32) ([]byte, bool) {
163 if !m.hasSize(offset, uint64(byteCount)) {
164 return nil, false
165 }
166 return m.Buffer[offset : offset+byteCount : offset+byteCount], true
167}
168
169// WriteByte implements the same method as documented on api.Memory.
170func (m *MemoryInstance) WriteByte(offset uint32, v byte) bool {
171 if !m.hasSize(offset, 1) {
172 return false
173 }
174 m.Buffer[offset] = v
175 return true
176}
177
178// WriteUint16Le implements the same method as documented on api.Memory.
179func (m *MemoryInstance) WriteUint16Le(offset uint32, v uint16) bool {
180 if !m.hasSize(offset, 2) {
181 return false
182 }
183 binary.LittleEndian.PutUint16(m.Buffer[offset:], v)
184 return true
185}
186
187// WriteUint32Le implements the same method as documented on api.Memory.
188func (m *MemoryInstance) WriteUint32Le(offset, v uint32) bool {
189 return m.writeUint32Le(offset, v)
190}
191
192// WriteFloat32Le implements the same method as documented on api.Memory.
193func (m *MemoryInstance) WriteFloat32Le(offset uint32, v float32) bool {
194 return m.writeUint32Le(offset, math.Float32bits(v))
195}
196
197// WriteUint64Le implements the same method as documented on api.Memory.
198func (m *MemoryInstance) WriteUint64Le(offset uint32, v uint64) bool {
199 return m.writeUint64Le(offset, v)
200}
201
202// WriteFloat64Le implements the same method as documented on api.Memory.
203func (m *MemoryInstance) WriteFloat64Le(offset uint32, v float64) bool {
204 return m.writeUint64Le(offset, math.Float64bits(v))
205}
206
207// Write implements the same method as documented on api.Memory.
208func (m *MemoryInstance) Write(offset uint32, val []byte) bool {
209 if !m.hasSize(offset, uint64(len(val))) {
210 return false
211 }
212 copy(m.Buffer[offset:], val)
213 return true
214}
215
216// WriteString implements the same method as documented on api.Memory.
217func (m *MemoryInstance) WriteString(offset uint32, val string) bool {
218 if !m.hasSize(offset, uint64(len(val))) {
219 return false
220 }
221 copy(m.Buffer[offset:], val)
222 return true
223}
224
225// MemoryPagesToBytesNum converts the given pages into the number of bytes contained in these pages.
226func MemoryPagesToBytesNum(pages uint32) (bytesNum uint64) {
227 return uint64(pages) << MemoryPageSizeInBits
228}
229
230// Grow implements the same method as documented on api.Memory.
231func (m *MemoryInstance) Grow(delta uint32) (result uint32, ok bool) {
232 if m.Shared {
233 m.Mux.Lock()
234 defer m.Mux.Unlock()
235 }
236
237 currentPages := m.Pages()
238 if delta == 0 {
239 return currentPages, true
240 }
241
242 newPages := currentPages + delta
243 if newPages > m.Max || int32(delta) < 0 {
244 return 0, false
245 } else if m.expBuffer != nil {
246 buffer := m.expBuffer.Reallocate(MemoryPagesToBytesNum(newPages))
247 if buffer == nil {
248 // Allocator failed to grow.
249 return 0, false
250 }
251 if m.Shared {
252 if unsafe.SliceData(buffer) != unsafe.SliceData(m.Buffer) {
253 panic("shared memory cannot move, this is a bug in the memory allocator")
254 }
255 // We assume grow is called under a guest lock.
256 // But the memory length is accessed elsewhere,
257 // so use atomic to make the new length visible across threads.
258 atomicStoreLengthAndCap(&m.Buffer, uintptr(len(buffer)), uintptr(cap(buffer)))
259 m.Cap = memoryBytesNumToPages(uint64(cap(buffer)))
260 } else {
261 m.Buffer = buffer
262 m.Cap = newPages
263 }
264 } else if newPages > m.Cap { // grow the memory.
265 if m.Shared {
266 panic("shared memory cannot be grown, this is a bug in wazero")
267 }
268 m.Buffer = append(m.Buffer, make([]byte, MemoryPagesToBytesNum(delta))...)
269 m.Cap = newPages
270 } else { // We already have the capacity we need.
271 if m.Shared {
272 // We assume grow is called under a guest lock.
273 // But the memory length is accessed elsewhere,
274 // so use atomic to make the new length visible across threads.
275 atomicStoreLength(&m.Buffer, uintptr(MemoryPagesToBytesNum(newPages)))
276 } else {
277 m.Buffer = m.Buffer[:MemoryPagesToBytesNum(newPages)]
278 }
279 }
280 m.ownerModuleEngine.MemoryGrown()
281 return currentPages, true
282}
283
284// Pages implements the same method as documented on api.Memory.
285func (m *MemoryInstance) Pages() (result uint32) {
286 return memoryBytesNumToPages(uint64(len(m.Buffer)))
287}
288
289// PagesToUnitOfBytes converts the pages to a human-readable form similar to what's specified. e.g. 1 -> "64Ki"
290//
291// See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#memory-instances%E2%91%A0
292func PagesToUnitOfBytes(pages uint32) string {
293 k := pages * 64
294 if k < 1024 {
295 return fmt.Sprintf("%d Ki", k)
296 }
297 m := k / 1024
298 if m < 1024 {
299 return fmt.Sprintf("%d Mi", m)
300 }
301 g := m / 1024
302 if g < 1024 {
303 return fmt.Sprintf("%d Gi", g)
304 }
305 return fmt.Sprintf("%d Ti", g/1024)
306}
307
308// Below are raw functions used to implement the api.Memory API:
309
310// Uses atomic write to update the length of a slice.
311func atomicStoreLengthAndCap(slice *[]byte, length uintptr, cap uintptr) {
312 //nolint:staticcheck
313 slicePtr := (*reflect.SliceHeader)(unsafe.Pointer(slice))
314 capPtr := (*uintptr)(unsafe.Pointer(&slicePtr.Cap))
315 atomic.StoreUintptr(capPtr, cap)
316 lenPtr := (*uintptr)(unsafe.Pointer(&slicePtr.Len))
317 atomic.StoreUintptr(lenPtr, length)
318}
319
320// Uses atomic write to update the length of a slice.
321func atomicStoreLength(slice *[]byte, length uintptr) {
322 //nolint:staticcheck
323 slicePtr := (*reflect.SliceHeader)(unsafe.Pointer(slice))
324 lenPtr := (*uintptr)(unsafe.Pointer(&slicePtr.Len))
325 atomic.StoreUintptr(lenPtr, length)
326}
327
328// memoryBytesNumToPages converts the given number of bytes into the number of pages.
329func memoryBytesNumToPages(bytesNum uint64) (pages uint32) {
330 return uint32(bytesNum >> MemoryPageSizeInBits)
331}
332
333// hasSize returns true if Len is sufficient for byteCount at the given offset.
334//
335// Note: This is always fine, because memory can grow, but never shrink.
336func (m *MemoryInstance) hasSize(offset uint32, byteCount uint64) bool {
337 return uint64(offset)+byteCount <= uint64(len(m.Buffer)) // uint64 prevents overflow on add
338}
339
340// readUint32Le implements ReadUint32Le without using a context. This is extracted as both ints and floats are stored in
341// memory as uint32le.
342func (m *MemoryInstance) readUint32Le(offset uint32) (uint32, bool) {
343 if !m.hasSize(offset, 4) {
344 return 0, false
345 }
346 return binary.LittleEndian.Uint32(m.Buffer[offset : offset+4]), true
347}
348
349// readUint64Le implements ReadUint64Le without using a context. This is extracted as both ints and floats are stored in
350// memory as uint64le.
351func (m *MemoryInstance) readUint64Le(offset uint32) (uint64, bool) {
352 if !m.hasSize(offset, 8) {
353 return 0, false
354 }
355 return binary.LittleEndian.Uint64(m.Buffer[offset : offset+8]), true
356}
357
358// writeUint32Le implements WriteUint32Le without using a context. This is extracted as both ints and floats are stored
359// in memory as uint32le.
360func (m *MemoryInstance) writeUint32Le(offset uint32, v uint32) bool {
361 if !m.hasSize(offset, 4) {
362 return false
363 }
364 binary.LittleEndian.PutUint32(m.Buffer[offset:], v)
365 return true
366}
367
368// writeUint64Le implements WriteUint64Le without using a context. This is extracted as both ints and floats are stored
369// in memory as uint64le.
370func (m *MemoryInstance) writeUint64Le(offset uint32, v uint64) bool {
371 if !m.hasSize(offset, 8) {
372 return false
373 }
374 binary.LittleEndian.PutUint64(m.Buffer[offset:], v)
375 return true
376}
377
378// Wait32 suspends the caller until the offset is notified by a different agent.
379func (m *MemoryInstance) Wait32(offset uint32, exp uint32, timeout int64, reader func(mem *MemoryInstance, offset uint32) uint32) uint64 {
380 w := m.getWaiters(offset)
381 w.mux.Lock()
382
383 cur := reader(m, offset)
384 if cur != exp {
385 w.mux.Unlock()
386 return 1
387 }
388
389 return m.wait(w, timeout)
390}
391
392// Wait64 suspends the caller until the offset is notified by a different agent.
393func (m *MemoryInstance) Wait64(offset uint32, exp uint64, timeout int64, reader func(mem *MemoryInstance, offset uint32) uint64) uint64 {
394 w := m.getWaiters(offset)
395 w.mux.Lock()
396
397 cur := reader(m, offset)
398 if cur != exp {
399 w.mux.Unlock()
400 return 1
401 }
402
403 return m.wait(w, timeout)
404}
405
406func (m *MemoryInstance) wait(w *waiters, timeout int64) uint64 {
407 if w.l == nil {
408 w.l = list.New()
409 }
410
411 // The specification requires a trap if the number of existing waiters + 1 == 2^32, so we add a check here.
412 // In practice, it is unlikely the application would ever accumulate such a large number of waiters as it
413 // indicates several GB of RAM used just for the list of waiters.
414 // https://github.com/WebAssembly/threads/blob/main/proposals/threads/Overview.md#wait
415 if uint64(w.l.Len()+1) == 1<<32 {
416 w.mux.Unlock()
417 panic(wasmruntime.ErrRuntimeTooManyWaiters)
418 }
419
420 ready := make(chan struct{})
421 elem := w.l.PushBack(ready)
422 w.mux.Unlock()
423
424 if timeout < 0 {
425 <-ready
426 return 0
427 } else {
428 select {
429 case <-ready:
430 return 0
431 case <-time.After(time.Duration(timeout)):
432 // While we could see if the channel completed by now and ignore the timeout, similar to x/sync/semaphore,
433 // the Wasm spec doesn't specify this behavior, so we keep things simple by prioritizing the timeout.
434 w.mux.Lock()
435 w.l.Remove(elem)
436 w.mux.Unlock()
437 return 2
438 }
439 }
440}
441
442func (m *MemoryInstance) getWaiters(offset uint32) *waiters {
443 wAny, ok := m.waiters.Load(offset)
444 if !ok {
445 // The first time an address is waited on, simultaneous waits will cause extra allocations.
446 // Further operations will be loaded above, which is also the general pattern of usage with
447 // mutexes.
448 wAny, _ = m.waiters.LoadOrStore(offset, &waiters{})
449 }
450
451 return wAny.(*waiters)
452}
453
454// Notify wakes up at most count waiters at the given offset.
455func (m *MemoryInstance) Notify(offset uint32, count uint32) uint32 {
456 wAny, ok := m.waiters.Load(offset)
457 if !ok {
458 return 0
459 }
460 w := wAny.(*waiters)
461
462 w.mux.Lock()
463 defer w.mux.Unlock()
464 if w.l == nil {
465 return 0
466 }
467
468 res := uint32(0)
469 for num := w.l.Len(); num > 0 && res < count; num = w.l.Len() {
470 w := w.l.Remove(w.l.Front()).(chan struct{})
471 close(w)
472 res++
473 }
474
475 return res
476}