1// Copyright (c) 2012-2018 Ugorji Nwoke. All rights reserved.
2// Use of this source code is governed by a MIT license found in the LICENSE file.
3
4package codec
5
6import (
7 "bufio"
8 "encoding"
9 "errors"
10 "fmt"
11 "io"
12 "reflect"
13 "sort"
14 "strconv"
15 "sync"
16 "time"
17)
18
19const defEncByteBufSize = 1 << 6 // 4:16, 6:64, 8:256, 10:1024
20
21var errEncoderNotInitialized = errors.New("Encoder not initialized")
22
23// encWriter abstracts writing to a byte array or to an io.Writer.
24type encWriter interface {
25 writeb([]byte)
26 writestr(string)
27 writen1(byte)
28 writen2(byte, byte)
29 atEndOfEncode()
30}
31
32// encDriver abstracts the actual codec (binc vs msgpack, etc)
33type encDriver interface {
34 EncodeNil()
35 EncodeInt(i int64)
36 EncodeUint(i uint64)
37 EncodeBool(b bool)
38 EncodeFloat32(f float32)
39 EncodeFloat64(f float64)
40 // encodeExtPreamble(xtag byte, length int)
41 EncodeRawExt(re *RawExt, e *Encoder)
42 EncodeExt(v interface{}, xtag uint64, ext Ext, e *Encoder)
43 EncodeString(c charEncoding, v string)
44 // EncodeSymbol(v string)
45 EncodeStringBytes(c charEncoding, v []byte)
46 EncodeTime(time.Time)
47 //encBignum(f *big.Int)
48 //encStringRunes(c charEncoding, v []rune)
49 WriteArrayStart(length int)
50 WriteArrayElem()
51 WriteArrayEnd()
52 WriteMapStart(length int)
53 WriteMapElemKey()
54 WriteMapElemValue()
55 WriteMapEnd()
56
57 reset()
58 atEndOfEncode()
59}
60
61type ioEncStringWriter interface {
62 WriteString(s string) (n int, err error)
63}
64
65type encDriverAsis interface {
66 EncodeAsis(v []byte)
67}
68
69type encDriverNoopContainerWriter struct{}
70
71func (encDriverNoopContainerWriter) WriteArrayStart(length int) {}
72func (encDriverNoopContainerWriter) WriteArrayElem() {}
73func (encDriverNoopContainerWriter) WriteArrayEnd() {}
74func (encDriverNoopContainerWriter) WriteMapStart(length int) {}
75func (encDriverNoopContainerWriter) WriteMapElemKey() {}
76func (encDriverNoopContainerWriter) WriteMapElemValue() {}
77func (encDriverNoopContainerWriter) WriteMapEnd() {}
78func (encDriverNoopContainerWriter) atEndOfEncode() {}
79
80type encDriverTrackContainerWriter struct {
81 c containerState
82}
83
84func (e *encDriverTrackContainerWriter) WriteArrayStart(length int) { e.c = containerArrayStart }
85func (e *encDriverTrackContainerWriter) WriteArrayElem() { e.c = containerArrayElem }
86func (e *encDriverTrackContainerWriter) WriteArrayEnd() { e.c = containerArrayEnd }
87func (e *encDriverTrackContainerWriter) WriteMapStart(length int) { e.c = containerMapStart }
88func (e *encDriverTrackContainerWriter) WriteMapElemKey() { e.c = containerMapKey }
89func (e *encDriverTrackContainerWriter) WriteMapElemValue() { e.c = containerMapValue }
90func (e *encDriverTrackContainerWriter) WriteMapEnd() { e.c = containerMapEnd }
91func (e *encDriverTrackContainerWriter) atEndOfEncode() {}
92
93// type ioEncWriterWriter interface {
94// WriteByte(c byte) error
95// WriteString(s string) (n int, err error)
96// Write(p []byte) (n int, err error)
97// }
98
99// EncodeOptions captures configuration options during encode.
100type EncodeOptions struct {
101 // WriterBufferSize is the size of the buffer used when writing.
102 //
103 // if > 0, we use a smart buffer internally for performance purposes.
104 WriterBufferSize int
105
106 // ChanRecvTimeout is the timeout used when selecting from a chan.
107 //
108 // Configuring this controls how we receive from a chan during the encoding process.
109 // - If ==0, we only consume the elements currently available in the chan.
110 // - if <0, we consume until the chan is closed.
111 // - If >0, we consume until this timeout.
112 ChanRecvTimeout time.Duration
113
114 // StructToArray specifies to encode a struct as an array, and not as a map
115 StructToArray bool
116
117 // Canonical representation means that encoding a value will always result in the same
118 // sequence of bytes.
119 //
120 // This only affects maps, as the iteration order for maps is random.
121 //
122 // The implementation MAY use the natural sort order for the map keys if possible:
123 //
124 // - If there is a natural sort order (ie for number, bool, string or []byte keys),
125 // then the map keys are first sorted in natural order and then written
126 // with corresponding map values to the strema.
127 // - If there is no natural sort order, then the map keys will first be
128 // encoded into []byte, and then sorted,
129 // before writing the sorted keys and the corresponding map values to the stream.
130 //
131 Canonical bool
132
133 // CheckCircularRef controls whether we check for circular references
134 // and error fast during an encode.
135 //
136 // If enabled, an error is received if a pointer to a struct
137 // references itself either directly or through one of its fields (iteratively).
138 //
139 // This is opt-in, as there may be a performance hit to checking circular references.
140 CheckCircularRef bool
141
142 // RecursiveEmptyCheck controls whether we descend into interfaces, structs and pointers
143 // when checking if a value is empty.
144 //
145 // Note that this may make OmitEmpty more expensive, as it incurs a lot more reflect calls.
146 RecursiveEmptyCheck bool
147
148 // Raw controls whether we encode Raw values.
149 // This is a "dangerous" option and must be explicitly set.
150 // If set, we blindly encode Raw values as-is, without checking
151 // if they are a correct representation of a value in that format.
152 // If unset, we error out.
153 Raw bool
154
155 // // AsSymbols defines what should be encoded as symbols.
156 // //
157 // // Encoding as symbols can reduce the encoded size significantly.
158 // //
159 // // However, during decoding, each string to be encoded as a symbol must
160 // // be checked to see if it has been seen before. Consequently, encoding time
161 // // will increase if using symbols, because string comparisons has a clear cost.
162 // //
163 // // Sample values:
164 // // AsSymbolNone
165 // // AsSymbolAll
166 // // AsSymbolMapStringKeys
167 // // AsSymbolMapStringKeysFlag | AsSymbolStructFieldNameFlag
168 // AsSymbols AsSymbolFlag
169}
170
171// ---------------------------------------------
172
173// ioEncWriter implements encWriter and can write to an io.Writer implementation
174type ioEncWriter struct {
175 w io.Writer
176 ww io.Writer
177 bw io.ByteWriter
178 sw ioEncStringWriter
179 fw ioFlusher
180 b [8]byte
181}
182
183func (z *ioEncWriter) WriteByte(b byte) (err error) {
184 z.b[0] = b
185 _, err = z.w.Write(z.b[:1])
186 return
187}
188
189func (z *ioEncWriter) WriteString(s string) (n int, err error) {
190 return z.w.Write(bytesView(s))
191}
192
193func (z *ioEncWriter) writeb(bs []byte) {
194 if _, err := z.ww.Write(bs); err != nil {
195 panic(err)
196 }
197}
198
199func (z *ioEncWriter) writestr(s string) {
200 if _, err := z.sw.WriteString(s); err != nil {
201 panic(err)
202 }
203}
204
205func (z *ioEncWriter) writen1(b byte) {
206 if err := z.bw.WriteByte(b); err != nil {
207 panic(err)
208 }
209}
210
211func (z *ioEncWriter) writen2(b1, b2 byte) {
212 var err error
213 if err = z.bw.WriteByte(b1); err == nil {
214 if err = z.bw.WriteByte(b2); err == nil {
215 return
216 }
217 }
218 panic(err)
219}
220
221// func (z *ioEncWriter) writen5(b1, b2, b3, b4, b5 byte) {
222// z.b[0], z.b[1], z.b[2], z.b[3], z.b[4] = b1, b2, b3, b4, b5
223// if _, err := z.ww.Write(z.b[:5]); err != nil {
224// panic(err)
225// }
226// }
227
228func (z *ioEncWriter) atEndOfEncode() {
229 if z.fw != nil {
230 if err := z.fw.Flush(); err != nil {
231 panic(err)
232 }
233 }
234}
235
236// ---------------------------------------------
237
238// bytesEncAppender implements encWriter and can write to an byte slice.
239type bytesEncAppender struct {
240 b []byte
241 out *[]byte
242}
243
244func (z *bytesEncAppender) writeb(s []byte) {
245 z.b = append(z.b, s...)
246}
247func (z *bytesEncAppender) writestr(s string) {
248 z.b = append(z.b, s...)
249}
250func (z *bytesEncAppender) writen1(b1 byte) {
251 z.b = append(z.b, b1)
252}
253func (z *bytesEncAppender) writen2(b1, b2 byte) {
254 z.b = append(z.b, b1, b2)
255}
256func (z *bytesEncAppender) atEndOfEncode() {
257 *(z.out) = z.b
258}
259func (z *bytesEncAppender) reset(in []byte, out *[]byte) {
260 z.b = in[:0]
261 z.out = out
262}
263
264// ---------------------------------------------
265
266func (e *Encoder) rawExt(f *codecFnInfo, rv reflect.Value) {
267 e.e.EncodeRawExt(rv2i(rv).(*RawExt), e)
268}
269
270func (e *Encoder) ext(f *codecFnInfo, rv reflect.Value) {
271 e.e.EncodeExt(rv2i(rv), f.xfTag, f.xfFn, e)
272}
273
274func (e *Encoder) selferMarshal(f *codecFnInfo, rv reflect.Value) {
275 rv2i(rv).(Selfer).CodecEncodeSelf(e)
276}
277
278func (e *Encoder) binaryMarshal(f *codecFnInfo, rv reflect.Value) {
279 bs, fnerr := rv2i(rv).(encoding.BinaryMarshaler).MarshalBinary()
280 e.marshal(bs, fnerr, false, cRAW)
281}
282
283func (e *Encoder) textMarshal(f *codecFnInfo, rv reflect.Value) {
284 bs, fnerr := rv2i(rv).(encoding.TextMarshaler).MarshalText()
285 e.marshal(bs, fnerr, false, cUTF8)
286}
287
288func (e *Encoder) jsonMarshal(f *codecFnInfo, rv reflect.Value) {
289 bs, fnerr := rv2i(rv).(jsonMarshaler).MarshalJSON()
290 e.marshal(bs, fnerr, true, cUTF8)
291}
292
293func (e *Encoder) raw(f *codecFnInfo, rv reflect.Value) {
294 e.rawBytes(rv2i(rv).(Raw))
295}
296
297func (e *Encoder) kInvalid(f *codecFnInfo, rv reflect.Value) {
298 e.e.EncodeNil()
299}
300
301func (e *Encoder) kErr(f *codecFnInfo, rv reflect.Value) {
302 e.errorf("unsupported kind %s, for %#v", rv.Kind(), rv)
303}
304
305func (e *Encoder) kSlice(f *codecFnInfo, rv reflect.Value) {
306 ti := f.ti
307 ee := e.e
308 // array may be non-addressable, so we have to manage with care
309 // (don't call rv.Bytes, rv.Slice, etc).
310 // E.g. type struct S{B [2]byte};
311 // Encode(S{}) will bomb on "panic: slice of unaddressable array".
312 if f.seq != seqTypeArray {
313 if rv.IsNil() {
314 ee.EncodeNil()
315 return
316 }
317 // If in this method, then there was no extension function defined.
318 // So it's okay to treat as []byte.
319 if ti.rtid == uint8SliceTypId {
320 ee.EncodeStringBytes(cRAW, rv.Bytes())
321 return
322 }
323 }
324 if f.seq == seqTypeChan && ti.chandir&uint8(reflect.RecvDir) == 0 {
325 e.errorf("send-only channel cannot be encoded")
326 }
327 elemsep := e.esep
328 rtelem := ti.elem
329 rtelemIsByte := uint8TypId == rt2id(rtelem) // NOT rtelem.Kind() == reflect.Uint8
330 var l int
331 // if a slice, array or chan of bytes, treat specially
332 if rtelemIsByte {
333 switch f.seq {
334 case seqTypeSlice:
335 ee.EncodeStringBytes(cRAW, rv.Bytes())
336 case seqTypeArray:
337 l = rv.Len()
338 if rv.CanAddr() {
339 ee.EncodeStringBytes(cRAW, rv.Slice(0, l).Bytes())
340 } else {
341 var bs []byte
342 if l <= cap(e.b) {
343 bs = e.b[:l]
344 } else {
345 bs = make([]byte, l)
346 }
347 reflect.Copy(reflect.ValueOf(bs), rv)
348 ee.EncodeStringBytes(cRAW, bs)
349 }
350 case seqTypeChan:
351 // do not use range, so that the number of elements encoded
352 // does not change, and encoding does not hang waiting on someone to close chan.
353 // for b := range rv2i(rv).(<-chan byte) { bs = append(bs, b) }
354 // ch := rv2i(rv).(<-chan byte) // fix error - that this is a chan byte, not a <-chan byte.
355
356 if rv.IsNil() {
357 ee.EncodeNil()
358 break
359 }
360 bs := e.b[:0]
361 irv := rv2i(rv)
362 ch, ok := irv.(<-chan byte)
363 if !ok {
364 ch = irv.(chan byte)
365 }
366
367 L1:
368 switch timeout := e.h.ChanRecvTimeout; {
369 case timeout == 0: // only consume available
370 for {
371 select {
372 case b := <-ch:
373 bs = append(bs, b)
374 default:
375 break L1
376 }
377 }
378 case timeout > 0: // consume until timeout
379 tt := time.NewTimer(timeout)
380 for {
381 select {
382 case b := <-ch:
383 bs = append(bs, b)
384 case <-tt.C:
385 // close(tt.C)
386 break L1
387 }
388 }
389 default: // consume until close
390 for b := range ch {
391 bs = append(bs, b)
392 }
393 }
394
395 ee.EncodeStringBytes(cRAW, bs)
396 }
397 return
398 }
399
400 // if chan, consume chan into a slice, and work off that slice.
401 var rvcs reflect.Value
402 if f.seq == seqTypeChan {
403 rvcs = reflect.Zero(reflect.SliceOf(rtelem))
404 timeout := e.h.ChanRecvTimeout
405 if timeout < 0 { // consume until close
406 for {
407 recv, recvOk := rv.Recv()
408 if !recvOk {
409 break
410 }
411 rvcs = reflect.Append(rvcs, recv)
412 }
413 } else {
414 cases := make([]reflect.SelectCase, 2)
415 cases[0] = reflect.SelectCase{Dir: reflect.SelectRecv, Chan: rv}
416 if timeout == 0 {
417 cases[1] = reflect.SelectCase{Dir: reflect.SelectDefault}
418 } else {
419 tt := time.NewTimer(timeout)
420 cases[1] = reflect.SelectCase{Dir: reflect.SelectRecv, Chan: reflect.ValueOf(tt.C)}
421 }
422 for {
423 chosen, recv, recvOk := reflect.Select(cases)
424 if chosen == 1 || !recvOk {
425 break
426 }
427 rvcs = reflect.Append(rvcs, recv)
428 }
429 }
430 rv = rvcs // TODO: ensure this doesn't mess up anywhere that rv of kind chan is expected
431 }
432
433 l = rv.Len()
434 if ti.mbs {
435 if l%2 == 1 {
436 e.errorf("mapBySlice requires even slice length, but got %v", l)
437 return
438 }
439 ee.WriteMapStart(l / 2)
440 } else {
441 ee.WriteArrayStart(l)
442 }
443
444 if l > 0 {
445 var fn *codecFn
446 for rtelem.Kind() == reflect.Ptr {
447 rtelem = rtelem.Elem()
448 }
449 // if kind is reflect.Interface, do not pre-determine the
450 // encoding type, because preEncodeValue may break it down to
451 // a concrete type and kInterface will bomb.
452 if rtelem.Kind() != reflect.Interface {
453 fn = e.cfer().get(rtelem, true, true)
454 }
455 for j := 0; j < l; j++ {
456 if elemsep {
457 if ti.mbs {
458 if j%2 == 0 {
459 ee.WriteMapElemKey()
460 } else {
461 ee.WriteMapElemValue()
462 }
463 } else {
464 ee.WriteArrayElem()
465 }
466 }
467 e.encodeValue(rv.Index(j), fn, true)
468 }
469 }
470
471 if ti.mbs {
472 ee.WriteMapEnd()
473 } else {
474 ee.WriteArrayEnd()
475 }
476}
477
478func (e *Encoder) kStructNoOmitempty(f *codecFnInfo, rv reflect.Value) {
479 fti := f.ti
480 elemsep := e.esep
481 tisfi := fti.sfiSrc
482 toMap := !(fti.toArray || e.h.StructToArray)
483 if toMap {
484 tisfi = fti.sfiSort
485 }
486 ee := e.e
487
488 sfn := structFieldNode{v: rv, update: false}
489 if toMap {
490 ee.WriteMapStart(len(tisfi))
491 if elemsep {
492 for _, si := range tisfi {
493 ee.WriteMapElemKey()
494 // ee.EncodeString(cUTF8, si.encName)
495 encStructFieldKey(ee, fti.keyType, si.encName)
496 ee.WriteMapElemValue()
497 e.encodeValue(sfn.field(si), nil, true)
498 }
499 } else {
500 for _, si := range tisfi {
501 // ee.EncodeString(cUTF8, si.encName)
502 encStructFieldKey(ee, fti.keyType, si.encName)
503 e.encodeValue(sfn.field(si), nil, true)
504 }
505 }
506 ee.WriteMapEnd()
507 } else {
508 ee.WriteArrayStart(len(tisfi))
509 if elemsep {
510 for _, si := range tisfi {
511 ee.WriteArrayElem()
512 e.encodeValue(sfn.field(si), nil, true)
513 }
514 } else {
515 for _, si := range tisfi {
516 e.encodeValue(sfn.field(si), nil, true)
517 }
518 }
519 ee.WriteArrayEnd()
520 }
521}
522
523func encStructFieldKey(ee encDriver, keyType valueType, s string) {
524 var m must
525
526 // use if-else-if, not switch (which compiles to binary-search)
527 // since keyType is typically valueTypeString, branch prediction is pretty good.
528
529 if keyType == valueTypeString {
530 ee.EncodeString(cUTF8, s)
531 } else if keyType == valueTypeInt {
532 ee.EncodeInt(m.Int(strconv.ParseInt(s, 10, 64)))
533 } else if keyType == valueTypeUint {
534 ee.EncodeUint(m.Uint(strconv.ParseUint(s, 10, 64)))
535 } else if keyType == valueTypeFloat {
536 ee.EncodeFloat64(m.Float(strconv.ParseFloat(s, 64)))
537 } else {
538 ee.EncodeString(cUTF8, s)
539 }
540}
541
542func (e *Encoder) kStruct(f *codecFnInfo, rv reflect.Value) {
543 fti := f.ti
544 elemsep := e.esep
545 tisfi := fti.sfiSrc
546 toMap := !(fti.toArray || e.h.StructToArray)
547 // if toMap, use the sorted array. If toArray, use unsorted array (to match sequence in struct)
548 if toMap {
549 tisfi = fti.sfiSort
550 }
551 newlen := len(fti.sfiSort)
552 ee := e.e
553
554 // Use sync.Pool to reduce allocating slices unnecessarily.
555 // The cost of sync.Pool is less than the cost of new allocation.
556 //
557 // Each element of the array pools one of encStructPool(8|16|32|64).
558 // It allows the re-use of slices up to 64 in length.
559 // A performance cost of encoding structs was collecting
560 // which values were empty and should be omitted.
561 // We needed slices of reflect.Value and string to collect them.
562 // This shared pool reduces the amount of unnecessary creation we do.
563 // The cost is that of locking sometimes, but sync.Pool is efficient
564 // enough to reduce thread contention.
565
566 var spool *sync.Pool
567 var poolv interface{}
568 var fkvs []stringRv
569 // fmt.Printf(">>>>>>>>>>>>>> encode.kStruct: newlen: %d\n", newlen)
570 if newlen <= 8 {
571 spool, poolv = pool.stringRv8()
572 fkvs = poolv.(*[8]stringRv)[:newlen]
573 } else if newlen <= 16 {
574 spool, poolv = pool.stringRv16()
575 fkvs = poolv.(*[16]stringRv)[:newlen]
576 } else if newlen <= 32 {
577 spool, poolv = pool.stringRv32()
578 fkvs = poolv.(*[32]stringRv)[:newlen]
579 } else if newlen <= 64 {
580 spool, poolv = pool.stringRv64()
581 fkvs = poolv.(*[64]stringRv)[:newlen]
582 } else if newlen <= 128 {
583 spool, poolv = pool.stringRv128()
584 fkvs = poolv.(*[128]stringRv)[:newlen]
585 } else {
586 fkvs = make([]stringRv, newlen)
587 }
588
589 newlen = 0
590 var kv stringRv
591 recur := e.h.RecursiveEmptyCheck
592 sfn := structFieldNode{v: rv, update: false}
593 for _, si := range tisfi {
594 // kv.r = si.field(rv, false)
595 kv.r = sfn.field(si)
596 if toMap {
597 if si.omitEmpty() && isEmptyValue(kv.r, e.h.TypeInfos, recur, recur) {
598 continue
599 }
600 kv.v = si.encName
601 } else {
602 // use the zero value.
603 // if a reference or struct, set to nil (so you do not output too much)
604 if si.omitEmpty() && isEmptyValue(kv.r, e.h.TypeInfos, recur, recur) {
605 switch kv.r.Kind() {
606 case reflect.Struct, reflect.Interface, reflect.Ptr, reflect.Array, reflect.Map, reflect.Slice:
607 kv.r = reflect.Value{} //encode as nil
608 }
609 }
610 }
611 fkvs[newlen] = kv
612 newlen++
613 }
614
615 if toMap {
616 ee.WriteMapStart(newlen)
617 if elemsep {
618 for j := 0; j < newlen; j++ {
619 kv = fkvs[j]
620 ee.WriteMapElemKey()
621 // ee.EncodeString(cUTF8, kv.v)
622 encStructFieldKey(ee, fti.keyType, kv.v)
623 ee.WriteMapElemValue()
624 e.encodeValue(kv.r, nil, true)
625 }
626 } else {
627 for j := 0; j < newlen; j++ {
628 kv = fkvs[j]
629 // ee.EncodeString(cUTF8, kv.v)
630 encStructFieldKey(ee, fti.keyType, kv.v)
631 e.encodeValue(kv.r, nil, true)
632 }
633 }
634 ee.WriteMapEnd()
635 } else {
636 ee.WriteArrayStart(newlen)
637 if elemsep {
638 for j := 0; j < newlen; j++ {
639 ee.WriteArrayElem()
640 e.encodeValue(fkvs[j].r, nil, true)
641 }
642 } else {
643 for j := 0; j < newlen; j++ {
644 e.encodeValue(fkvs[j].r, nil, true)
645 }
646 }
647 ee.WriteArrayEnd()
648 }
649
650 // do not use defer. Instead, use explicit pool return at end of function.
651 // defer has a cost we are trying to avoid.
652 // If there is a panic and these slices are not returned, it is ok.
653 if spool != nil {
654 spool.Put(poolv)
655 }
656}
657
658func (e *Encoder) kMap(f *codecFnInfo, rv reflect.Value) {
659 ee := e.e
660 if rv.IsNil() {
661 ee.EncodeNil()
662 return
663 }
664
665 l := rv.Len()
666 ee.WriteMapStart(l)
667 elemsep := e.esep
668 if l == 0 {
669 ee.WriteMapEnd()
670 return
671 }
672 // var asSymbols bool
673 // determine the underlying key and val encFn's for the map.
674 // This eliminates some work which is done for each loop iteration i.e.
675 // rv.Type(), ref.ValueOf(rt).Pointer(), then check map/list for fn.
676 //
677 // However, if kind is reflect.Interface, do not pre-determine the
678 // encoding type, because preEncodeValue may break it down to
679 // a concrete type and kInterface will bomb.
680 var keyFn, valFn *codecFn
681 ti := f.ti
682 rtkey0 := ti.key
683 rtkey := rtkey0
684 rtval0 := ti.elem
685 rtval := rtval0
686 // rtkeyid := rt2id(rtkey0)
687 for rtval.Kind() == reflect.Ptr {
688 rtval = rtval.Elem()
689 }
690 if rtval.Kind() != reflect.Interface {
691 valFn = e.cfer().get(rtval, true, true)
692 }
693 mks := rv.MapKeys()
694
695 if e.h.Canonical {
696 e.kMapCanonical(rtkey, rv, mks, valFn)
697 ee.WriteMapEnd()
698 return
699 }
700
701 var keyTypeIsString = stringTypId == rt2id(rtkey0) // rtkeyid
702 if !keyTypeIsString {
703 for rtkey.Kind() == reflect.Ptr {
704 rtkey = rtkey.Elem()
705 }
706 if rtkey.Kind() != reflect.Interface {
707 // rtkeyid = rt2id(rtkey)
708 keyFn = e.cfer().get(rtkey, true, true)
709 }
710 }
711
712 // for j, lmks := 0, len(mks); j < lmks; j++ {
713 for j := range mks {
714 if elemsep {
715 ee.WriteMapElemKey()
716 }
717 if keyTypeIsString {
718 ee.EncodeString(cUTF8, mks[j].String())
719 } else {
720 e.encodeValue(mks[j], keyFn, true)
721 }
722 if elemsep {
723 ee.WriteMapElemValue()
724 }
725 e.encodeValue(rv.MapIndex(mks[j]), valFn, true)
726
727 }
728 ee.WriteMapEnd()
729}
730
731func (e *Encoder) kMapCanonical(rtkey reflect.Type, rv reflect.Value, mks []reflect.Value, valFn *codecFn) {
732 ee := e.e
733 elemsep := e.esep
734 // we previously did out-of-band if an extension was registered.
735 // This is not necessary, as the natural kind is sufficient for ordering.
736
737 switch rtkey.Kind() {
738 case reflect.Bool:
739 mksv := make([]boolRv, len(mks))
740 for i, k := range mks {
741 v := &mksv[i]
742 v.r = k
743 v.v = k.Bool()
744 }
745 sort.Sort(boolRvSlice(mksv))
746 for i := range mksv {
747 if elemsep {
748 ee.WriteMapElemKey()
749 }
750 ee.EncodeBool(mksv[i].v)
751 if elemsep {
752 ee.WriteMapElemValue()
753 }
754 e.encodeValue(rv.MapIndex(mksv[i].r), valFn, true)
755 }
756 case reflect.String:
757 mksv := make([]stringRv, len(mks))
758 for i, k := range mks {
759 v := &mksv[i]
760 v.r = k
761 v.v = k.String()
762 }
763 sort.Sort(stringRvSlice(mksv))
764 for i := range mksv {
765 if elemsep {
766 ee.WriteMapElemKey()
767 }
768 ee.EncodeString(cUTF8, mksv[i].v)
769 if elemsep {
770 ee.WriteMapElemValue()
771 }
772 e.encodeValue(rv.MapIndex(mksv[i].r), valFn, true)
773 }
774 case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uint, reflect.Uintptr:
775 mksv := make([]uintRv, len(mks))
776 for i, k := range mks {
777 v := &mksv[i]
778 v.r = k
779 v.v = k.Uint()
780 }
781 sort.Sort(uintRvSlice(mksv))
782 for i := range mksv {
783 if elemsep {
784 ee.WriteMapElemKey()
785 }
786 ee.EncodeUint(mksv[i].v)
787 if elemsep {
788 ee.WriteMapElemValue()
789 }
790 e.encodeValue(rv.MapIndex(mksv[i].r), valFn, true)
791 }
792 case reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Int:
793 mksv := make([]intRv, len(mks))
794 for i, k := range mks {
795 v := &mksv[i]
796 v.r = k
797 v.v = k.Int()
798 }
799 sort.Sort(intRvSlice(mksv))
800 for i := range mksv {
801 if elemsep {
802 ee.WriteMapElemKey()
803 }
804 ee.EncodeInt(mksv[i].v)
805 if elemsep {
806 ee.WriteMapElemValue()
807 }
808 e.encodeValue(rv.MapIndex(mksv[i].r), valFn, true)
809 }
810 case reflect.Float32:
811 mksv := make([]floatRv, len(mks))
812 for i, k := range mks {
813 v := &mksv[i]
814 v.r = k
815 v.v = k.Float()
816 }
817 sort.Sort(floatRvSlice(mksv))
818 for i := range mksv {
819 if elemsep {
820 ee.WriteMapElemKey()
821 }
822 ee.EncodeFloat32(float32(mksv[i].v))
823 if elemsep {
824 ee.WriteMapElemValue()
825 }
826 e.encodeValue(rv.MapIndex(mksv[i].r), valFn, true)
827 }
828 case reflect.Float64:
829 mksv := make([]floatRv, len(mks))
830 for i, k := range mks {
831 v := &mksv[i]
832 v.r = k
833 v.v = k.Float()
834 }
835 sort.Sort(floatRvSlice(mksv))
836 for i := range mksv {
837 if elemsep {
838 ee.WriteMapElemKey()
839 }
840 ee.EncodeFloat64(mksv[i].v)
841 if elemsep {
842 ee.WriteMapElemValue()
843 }
844 e.encodeValue(rv.MapIndex(mksv[i].r), valFn, true)
845 }
846 case reflect.Struct:
847 if rv.Type() == timeTyp {
848 mksv := make([]timeRv, len(mks))
849 for i, k := range mks {
850 v := &mksv[i]
851 v.r = k
852 v.v = rv2i(k).(time.Time)
853 }
854 sort.Sort(timeRvSlice(mksv))
855 for i := range mksv {
856 if elemsep {
857 ee.WriteMapElemKey()
858 }
859 ee.EncodeTime(mksv[i].v)
860 if elemsep {
861 ee.WriteMapElemValue()
862 }
863 e.encodeValue(rv.MapIndex(mksv[i].r), valFn, true)
864 }
865 break
866 }
867 fallthrough
868 default:
869 // out-of-band
870 // first encode each key to a []byte first, then sort them, then record
871 var mksv []byte = make([]byte, 0, len(mks)*16) // temporary byte slice for the encoding
872 e2 := NewEncoderBytes(&mksv, e.hh)
873 mksbv := make([]bytesRv, len(mks))
874 for i, k := range mks {
875 v := &mksbv[i]
876 l := len(mksv)
877 e2.MustEncode(k)
878 v.r = k
879 v.v = mksv[l:]
880 }
881 sort.Sort(bytesRvSlice(mksbv))
882 for j := range mksbv {
883 if elemsep {
884 ee.WriteMapElemKey()
885 }
886 e.asis(mksbv[j].v)
887 if elemsep {
888 ee.WriteMapElemValue()
889 }
890 e.encodeValue(rv.MapIndex(mksbv[j].r), valFn, true)
891 }
892 }
893}
894
895// // --------------------------------------------------
896
897type encWriterSwitch struct {
898 wi *ioEncWriter
899 // wb bytesEncWriter
900 wb bytesEncAppender
901 wx bool // if bytes, wx=true
902 esep bool // whether it has elem separators
903 isas bool // whether e.as != nil
904}
905
906// // TODO: Uncomment after mid-stack inlining enabled in go 1.11
907
908// func (z *encWriterSwitch) writeb(s []byte) {
909// if z.wx {
910// z.wb.writeb(s)
911// } else {
912// z.wi.writeb(s)
913// }
914// }
915// func (z *encWriterSwitch) writestr(s string) {
916// if z.wx {
917// z.wb.writestr(s)
918// } else {
919// z.wi.writestr(s)
920// }
921// }
922// func (z *encWriterSwitch) writen1(b1 byte) {
923// if z.wx {
924// z.wb.writen1(b1)
925// } else {
926// z.wi.writen1(b1)
927// }
928// }
929// func (z *encWriterSwitch) writen2(b1, b2 byte) {
930// if z.wx {
931// z.wb.writen2(b1, b2)
932// } else {
933// z.wi.writen2(b1, b2)
934// }
935// }
936
937// An Encoder writes an object to an output stream in the codec format.
938type Encoder struct {
939 panicHdl
940 // hopefully, reduce derefencing cost by laying the encWriter inside the Encoder
941 e encDriver
942 // NOTE: Encoder shouldn't call it's write methods,
943 // as the handler MAY need to do some coordination.
944 w encWriter
945
946 h *BasicHandle
947 bw *bufio.Writer
948 as encDriverAsis
949
950 // ---- cpu cache line boundary?
951
952 // ---- cpu cache line boundary?
953 encWriterSwitch
954 err error
955
956 // ---- cpu cache line boundary?
957 codecFnPooler
958 ci set
959 js bool // here, so that no need to piggy back on *codecFner for this
960 be bool // here, so that no need to piggy back on *codecFner for this
961 _ [6]byte // padding
962
963 // ---- writable fields during execution --- *try* to keep in sep cache line
964
965 // ---- cpu cache line boundary?
966 // b [scratchByteArrayLen]byte
967 // _ [cacheLineSize - scratchByteArrayLen]byte // padding
968 b [cacheLineSize - 0]byte // used for encoding a chan or (non-addressable) array of bytes
969}
970
971// NewEncoder returns an Encoder for encoding into an io.Writer.
972//
973// For efficiency, Users are encouraged to pass in a memory buffered writer
974// (eg bufio.Writer, bytes.Buffer).
975func NewEncoder(w io.Writer, h Handle) *Encoder {
976 e := newEncoder(h)
977 e.Reset(w)
978 return e
979}
980
981// NewEncoderBytes returns an encoder for encoding directly and efficiently
982// into a byte slice, using zero-copying to temporary slices.
983//
984// It will potentially replace the output byte slice pointed to.
985// After encoding, the out parameter contains the encoded contents.
986func NewEncoderBytes(out *[]byte, h Handle) *Encoder {
987 e := newEncoder(h)
988 e.ResetBytes(out)
989 return e
990}
991
992func newEncoder(h Handle) *Encoder {
993 e := &Encoder{h: h.getBasicHandle(), err: errEncoderNotInitialized}
994 e.hh = h
995 e.esep = h.hasElemSeparators()
996 return e
997}
998
999func (e *Encoder) resetCommon() {
1000 if e.e == nil || e.hh.recreateEncDriver(e.e) {
1001 e.e = e.hh.newEncDriver(e)
1002 e.as, e.isas = e.e.(encDriverAsis)
1003 // e.cr, _ = e.e.(containerStateRecv)
1004 }
1005 e.be = e.hh.isBinary()
1006 _, e.js = e.hh.(*JsonHandle)
1007 e.e.reset()
1008 e.err = nil
1009}
1010
1011// Reset resets the Encoder with a new output stream.
1012//
1013// This accommodates using the state of the Encoder,
1014// where it has "cached" information about sub-engines.
1015func (e *Encoder) Reset(w io.Writer) {
1016 if w == nil {
1017 return
1018 }
1019 if e.wi == nil {
1020 e.wi = new(ioEncWriter)
1021 }
1022 var ok bool
1023 e.wx = false
1024 e.wi.w = w
1025 if e.h.WriterBufferSize > 0 {
1026 e.bw = bufio.NewWriterSize(w, e.h.WriterBufferSize)
1027 e.wi.bw = e.bw
1028 e.wi.sw = e.bw
1029 e.wi.fw = e.bw
1030 e.wi.ww = e.bw
1031 } else {
1032 if e.wi.bw, ok = w.(io.ByteWriter); !ok {
1033 e.wi.bw = e.wi
1034 }
1035 if e.wi.sw, ok = w.(ioEncStringWriter); !ok {
1036 e.wi.sw = e.wi
1037 }
1038 e.wi.fw, _ = w.(ioFlusher)
1039 e.wi.ww = w
1040 }
1041 e.w = e.wi
1042 e.resetCommon()
1043}
1044
1045// ResetBytes resets the Encoder with a new destination output []byte.
1046func (e *Encoder) ResetBytes(out *[]byte) {
1047 if out == nil {
1048 return
1049 }
1050 var in []byte
1051 if out != nil {
1052 in = *out
1053 }
1054 if in == nil {
1055 in = make([]byte, defEncByteBufSize)
1056 }
1057 e.wx = true
1058 e.wb.reset(in, out)
1059 e.w = &e.wb
1060 e.resetCommon()
1061}
1062
1063// Encode writes an object into a stream.
1064//
1065// Encoding can be configured via the struct tag for the fields.
1066// The key (in the struct tags) that we look at is configurable.
1067//
1068// By default, we look up the "codec" key in the struct field's tags,
1069// and fall bak to the "json" key if "codec" is absent.
1070// That key in struct field's tag value is the key name,
1071// followed by an optional comma and options.
1072//
1073// To set an option on all fields (e.g. omitempty on all fields), you
1074// can create a field called _struct, and set flags on it. The options
1075// which can be set on _struct are:
1076// - omitempty: so all fields are omitted if empty
1077// - toarray: so struct is encoded as an array
1078// - int: so struct key names are encoded as signed integers (instead of strings)
1079// - uint: so struct key names are encoded as unsigned integers (instead of strings)
1080// - float: so struct key names are encoded as floats (instead of strings)
1081// More details on these below.
1082//
1083// Struct values "usually" encode as maps. Each exported struct field is encoded unless:
1084// - the field's tag is "-", OR
1085// - the field is empty (empty or the zero value) and its tag specifies the "omitempty" option.
1086//
1087// When encoding as a map, the first string in the tag (before the comma)
1088// is the map key string to use when encoding.
1089// ...
1090// This key is typically encoded as a string.
1091// However, there are instances where the encoded stream has mapping keys encoded as numbers.
1092// For example, some cbor streams have keys as integer codes in the stream, but they should map
1093// to fields in a structured object. Consequently, a struct is the natural representation in code.
1094// For these, configure the struct to encode/decode the keys as numbers (instead of string).
1095// This is done with the int,uint or float option on the _struct field (see above).
1096//
1097// However, struct values may encode as arrays. This happens when:
1098// - StructToArray Encode option is set, OR
1099// - the tag on the _struct field sets the "toarray" option
1100// Note that omitempty is ignored when encoding struct values as arrays,
1101// as an entry must be encoded for each field, to maintain its position.
1102//
1103// Values with types that implement MapBySlice are encoded as stream maps.
1104//
1105// The empty values (for omitempty option) are false, 0, any nil pointer
1106// or interface value, and any array, slice, map, or string of length zero.
1107//
1108// Anonymous fields are encoded inline except:
1109// - the struct tag specifies a replacement name (first value)
1110// - the field is of an interface type
1111//
1112// Examples:
1113//
1114// // NOTE: 'json:' can be used as struct tag key, in place 'codec:' below.
1115// type MyStruct struct {
1116// _struct bool `codec:",omitempty"` //set omitempty for every field
1117// Field1 string `codec:"-"` //skip this field
1118// Field2 int `codec:"myName"` //Use key "myName" in encode stream
1119// Field3 int32 `codec:",omitempty"` //use key "Field3". Omit if empty.
1120// Field4 bool `codec:"f4,omitempty"` //use key "f4". Omit if empty.
1121// io.Reader //use key "Reader".
1122// MyStruct `codec:"my1" //use key "my1".
1123// MyStruct //inline it
1124// ...
1125// }
1126//
1127// type MyStruct struct {
1128// _struct bool `codec:",toarray"` //encode struct as an array
1129// }
1130//
1131// type MyStruct struct {
1132// _struct bool `codec:",uint"` //encode struct with "unsigned integer" keys
1133// Field1 string `codec:"1"` //encode Field1 key using: EncodeInt(1)
1134// Field2 string `codec:"2"` //encode Field2 key using: EncodeInt(2)
1135// }
1136//
1137// The mode of encoding is based on the type of the value. When a value is seen:
1138// - If a Selfer, call its CodecEncodeSelf method
1139// - If an extension is registered for it, call that extension function
1140// - If implements encoding.(Binary|Text|JSON)Marshaler, call Marshal(Binary|Text|JSON) method
1141// - Else encode it based on its reflect.Kind
1142//
1143// Note that struct field names and keys in map[string]XXX will be treated as symbols.
1144// Some formats support symbols (e.g. binc) and will properly encode the string
1145// only once in the stream, and use a tag to refer to it thereafter.
1146func (e *Encoder) Encode(v interface{}) (err error) {
1147 defer e.deferred(&err)
1148 e.MustEncode(v)
1149 return
1150}
1151
1152// MustEncode is like Encode, but panics if unable to Encode.
1153// This provides insight to the code location that triggered the error.
1154func (e *Encoder) MustEncode(v interface{}) {
1155 if e.err != nil {
1156 panic(e.err)
1157 }
1158 e.encode(v)
1159 e.e.atEndOfEncode()
1160 e.w.atEndOfEncode()
1161 e.alwaysAtEnd()
1162}
1163
1164func (e *Encoder) deferred(err1 *error) {
1165 e.alwaysAtEnd()
1166 if recoverPanicToErr {
1167 if x := recover(); x != nil {
1168 panicValToErr(e, x, err1)
1169 panicValToErr(e, x, &e.err)
1170 }
1171 }
1172}
1173
1174// func (e *Encoder) alwaysAtEnd() {
1175// e.codecFnPooler.alwaysAtEnd()
1176// }
1177
1178func (e *Encoder) encode(iv interface{}) {
1179 if iv == nil || definitelyNil(iv) {
1180 e.e.EncodeNil()
1181 return
1182 }
1183 if v, ok := iv.(Selfer); ok {
1184 v.CodecEncodeSelf(e)
1185 return
1186 }
1187
1188 // a switch with only concrete types can be optimized.
1189 // consequently, we deal with nil and interfaces outside.
1190
1191 switch v := iv.(type) {
1192 case Raw:
1193 e.rawBytes(v)
1194 case reflect.Value:
1195 e.encodeValue(v, nil, true)
1196
1197 case string:
1198 e.e.EncodeString(cUTF8, v)
1199 case bool:
1200 e.e.EncodeBool(v)
1201 case int:
1202 e.e.EncodeInt(int64(v))
1203 case int8:
1204 e.e.EncodeInt(int64(v))
1205 case int16:
1206 e.e.EncodeInt(int64(v))
1207 case int32:
1208 e.e.EncodeInt(int64(v))
1209 case int64:
1210 e.e.EncodeInt(v)
1211 case uint:
1212 e.e.EncodeUint(uint64(v))
1213 case uint8:
1214 e.e.EncodeUint(uint64(v))
1215 case uint16:
1216 e.e.EncodeUint(uint64(v))
1217 case uint32:
1218 e.e.EncodeUint(uint64(v))
1219 case uint64:
1220 e.e.EncodeUint(v)
1221 case uintptr:
1222 e.e.EncodeUint(uint64(v))
1223 case float32:
1224 e.e.EncodeFloat32(v)
1225 case float64:
1226 e.e.EncodeFloat64(v)
1227 case time.Time:
1228 e.e.EncodeTime(v)
1229 case []uint8:
1230 e.e.EncodeStringBytes(cRAW, v)
1231
1232 case *Raw:
1233 e.rawBytes(*v)
1234
1235 case *string:
1236 e.e.EncodeString(cUTF8, *v)
1237 case *bool:
1238 e.e.EncodeBool(*v)
1239 case *int:
1240 e.e.EncodeInt(int64(*v))
1241 case *int8:
1242 e.e.EncodeInt(int64(*v))
1243 case *int16:
1244 e.e.EncodeInt(int64(*v))
1245 case *int32:
1246 e.e.EncodeInt(int64(*v))
1247 case *int64:
1248 e.e.EncodeInt(*v)
1249 case *uint:
1250 e.e.EncodeUint(uint64(*v))
1251 case *uint8:
1252 e.e.EncodeUint(uint64(*v))
1253 case *uint16:
1254 e.e.EncodeUint(uint64(*v))
1255 case *uint32:
1256 e.e.EncodeUint(uint64(*v))
1257 case *uint64:
1258 e.e.EncodeUint(*v)
1259 case *uintptr:
1260 e.e.EncodeUint(uint64(*v))
1261 case *float32:
1262 e.e.EncodeFloat32(*v)
1263 case *float64:
1264 e.e.EncodeFloat64(*v)
1265 case *time.Time:
1266 e.e.EncodeTime(*v)
1267
1268 case *[]uint8:
1269 e.e.EncodeStringBytes(cRAW, *v)
1270
1271 default:
1272 if !fastpathEncodeTypeSwitch(iv, e) {
1273 // checkfastpath=true (not false), as underlying slice/map type may be fast-path
1274 e.encodeValue(reflect.ValueOf(iv), nil, true)
1275 }
1276 }
1277}
1278
1279func (e *Encoder) encodeValue(rv reflect.Value, fn *codecFn, checkFastpath bool) {
1280 // if a valid fn is passed, it MUST BE for the dereferenced type of rv
1281 var sptr uintptr
1282 var rvp reflect.Value
1283 var rvpValid bool
1284TOP:
1285 switch rv.Kind() {
1286 case reflect.Ptr:
1287 if rv.IsNil() {
1288 e.e.EncodeNil()
1289 return
1290 }
1291 rvpValid = true
1292 rvp = rv
1293 rv = rv.Elem()
1294 if e.h.CheckCircularRef && rv.Kind() == reflect.Struct {
1295 // TODO: Movable pointers will be an issue here. Future problem.
1296 sptr = rv.UnsafeAddr()
1297 break TOP
1298 }
1299 goto TOP
1300 case reflect.Interface:
1301 if rv.IsNil() {
1302 e.e.EncodeNil()
1303 return
1304 }
1305 rv = rv.Elem()
1306 goto TOP
1307 case reflect.Slice, reflect.Map:
1308 if rv.IsNil() {
1309 e.e.EncodeNil()
1310 return
1311 }
1312 case reflect.Invalid, reflect.Func:
1313 e.e.EncodeNil()
1314 return
1315 }
1316
1317 if sptr != 0 && (&e.ci).add(sptr) {
1318 e.errorf("circular reference found: # %d", sptr)
1319 }
1320
1321 if fn == nil {
1322 rt := rv.Type()
1323 // always pass checkCodecSelfer=true, in case T or ****T is passed, where *T is a Selfer
1324 fn = e.cfer().get(rt, checkFastpath, true)
1325 }
1326 if fn.i.addrE {
1327 if rvpValid {
1328 fn.fe(e, &fn.i, rvp)
1329 } else if rv.CanAddr() {
1330 fn.fe(e, &fn.i, rv.Addr())
1331 } else {
1332 rv2 := reflect.New(rv.Type())
1333 rv2.Elem().Set(rv)
1334 fn.fe(e, &fn.i, rv2)
1335 }
1336 } else {
1337 fn.fe(e, &fn.i, rv)
1338 }
1339 if sptr != 0 {
1340 (&e.ci).remove(sptr)
1341 }
1342}
1343
1344func (e *Encoder) marshal(bs []byte, fnerr error, asis bool, c charEncoding) {
1345 if fnerr != nil {
1346 panic(fnerr)
1347 }
1348 if bs == nil {
1349 e.e.EncodeNil()
1350 } else if asis {
1351 e.asis(bs)
1352 } else {
1353 e.e.EncodeStringBytes(c, bs)
1354 }
1355}
1356
1357func (e *Encoder) asis(v []byte) {
1358 if e.isas {
1359 e.as.EncodeAsis(v)
1360 } else {
1361 e.w.writeb(v)
1362 }
1363}
1364
1365func (e *Encoder) rawBytes(vv Raw) {
1366 v := []byte(vv)
1367 if !e.h.Raw {
1368 e.errorf("Raw values cannot be encoded: %v", v)
1369 }
1370 e.asis(v)
1371}
1372
1373func (e *Encoder) wrapErrstr(v interface{}, err *error) {
1374 *err = fmt.Errorf("%s encode error: %v", e.hh.Name(), v)
1375}