1 // Go support for Protocol Buffers - Google's data interchange format
3 // Copyright 2010 Google Inc. All rights reserved.
4 // http://code.google.com/p/goprotobuf/
6 // Redistribution and use in source and binary forms, with or without
7 // modification, are permitted provided that the following conditions are
10 // * Redistributions of source code must retain the above copyright
11 // notice, this list of conditions and the following disclaimer.
12 // * Redistributions in binary form must reproduce the above
13 // copyright notice, this list of conditions and the following disclaimer
14 // in the documentation and/or other materials provided with the
16 // * Neither the name of Google Inc. nor the names of its
17 // contributors may be used to endorse or promote products derived from
18 // this software without specific prior written permission.
20 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
23 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
24 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
25 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
26 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
27 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
28 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
29 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
30 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35 * Routines for decoding protocol buffer data to construct in-memory representations.
47 // ErrWrongType occurs when the wire encoding for the field disagrees with
48 // that specified in the type being decoded. This is usually caused by attempting
49 // to convert an encoded protocol buffer into a struct of the wrong type.
50 var ErrWrongType = os.NewError("field/encoding mismatch: wrong type for field")
52 // The fundamental decoders that interpret bytes on the wire.
53 // Those that take integer types all return uint64 and are
54 // therefore of type valueDecoder.
56 // DecodeVarint reads a varint-encoded integer from the slice.
57 // It returns the integer and the number of bytes consumed, or
58 // zero if there is not enough.
59 // This is the format for the
60 // int32, int64, uint32, uint64, bool, and enum
61 // protocol buffer types.
62 func DecodeVarint(buf []byte) (x uint64, n int) {
64 for shift := uint(0); ; shift += 7 {
70 x |= (b & 0x7F) << shift
78 // DecodeVarint reads a varint-encoded integer from the Buffer.
79 // This is the format for the
80 // int32, int64, uint32, uint64, bool, and enum
81 // protocol buffer types.
82 func (p *Buffer) DecodeVarint() (x uint64, err os.Error) {
88 for shift := uint(0); ; shift += 7 {
90 err = io.ErrUnexpectedEOF
95 x |= (uint64(b) & 0x7F) << shift
104 // DecodeFixed64 reads a 64-bit integer from the Buffer.
105 // This is the format for the
106 // fixed64, sfixed64, and double protocol buffer types.
107 func (p *Buffer) DecodeFixed64() (x uint64, err os.Error) {
111 err = io.ErrUnexpectedEOF
116 x = uint64(p.buf[i-8])
117 x |= uint64(p.buf[i-7]) << 8
118 x |= uint64(p.buf[i-6]) << 16
119 x |= uint64(p.buf[i-5]) << 24
120 x |= uint64(p.buf[i-4]) << 32
121 x |= uint64(p.buf[i-3]) << 40
122 x |= uint64(p.buf[i-2]) << 48
123 x |= uint64(p.buf[i-1]) << 56
127 // DecodeFixed32 reads a 32-bit integer from the Buffer.
128 // This is the format for the
129 // fixed32, sfixed32, and float protocol buffer types.
130 func (p *Buffer) DecodeFixed32() (x uint64, err os.Error) {
134 err = io.ErrUnexpectedEOF
139 x = uint64(p.buf[i-4])
140 x |= uint64(p.buf[i-3]) << 8
141 x |= uint64(p.buf[i-2]) << 16
142 x |= uint64(p.buf[i-1]) << 24
146 // DecodeZigzag64 reads a zigzag-encoded 64-bit integer
148 // This is the format used for the sint64 protocol buffer type.
149 func (p *Buffer) DecodeZigzag64() (x uint64, err os.Error) {
150 x, err = p.DecodeVarint()
154 x = (x >> 1) ^ uint64((int64(x&1)<<63)>>63)
158 // DecodeZigzag32 reads a zigzag-encoded 32-bit integer
160 // This is the format used for the sint32 protocol buffer type.
161 func (p *Buffer) DecodeZigzag32() (x uint64, err os.Error) {
162 x, err = p.DecodeVarint()
166 x = uint64((uint32(x) >> 1) ^ uint32((int32(x&1)<<31)>>31))
170 // These are not ValueDecoders: they produce an array of bytes or a string.
171 // bytes, embedded messages
173 // DecodeRawBytes reads a count-delimited byte buffer from the Buffer.
174 // This is the format used for the bytes protocol buffer
175 // type and for embedded messages.
176 func (p *Buffer) DecodeRawBytes(alloc bool) (buf []byte, err os.Error) {
177 n, err := p.DecodeVarint()
183 if p.index+nb > len(p.buf) {
184 err = io.ErrUnexpectedEOF
189 // todo: check if can get more uses of alloc=false
190 buf = p.buf[p.index : p.index+nb]
195 buf = make([]byte, nb)
196 copy(buf, p.buf[p.index:])
201 // DecodeStringBytes reads an encoded string from the Buffer.
202 // This is the format used for the proto2 string type.
203 func (p *Buffer) DecodeStringBytes() (s string, err os.Error) {
204 buf, err := p.DecodeRawBytes(false)
208 return string(buf), nil
211 // Skip the next item in the buffer. Its wire type is decoded and presented as an argument.
212 // If the protocol buffer has extensions, and the field matches, add it as an extension.
213 // Otherwise, if the XXX_unrecognized field exists, append the skipped data there.
214 func (o *Buffer) skipAndSave(t reflect.Type, tag, wire int, base uintptr) os.Error {
218 err := o.skip(t, tag, wire)
223 x := fieldIndex(t, "XXX_unrecognized")
228 p := propByIndex(t, x)
229 ptr := (*[]byte)(unsafe.Pointer(base + p.offset))
232 // This is the first skipped element,
233 // allocate a new buffer.
237 // Add the skipped field to struct field
241 o.EncodeVarint(uint64(tag<<3 | wire))
242 *ptr = append(o.buf, obuf[oi:o.index]...)
249 // Skip the next item in the buffer. Its wire type is decoded and presented as an argument.
250 func (o *Buffer) skip(t reflect.Type, tag, wire int) os.Error {
257 _, err = o.DecodeVarint()
259 _, err = o.DecodeFixed64()
261 _, err = o.DecodeRawBytes(false)
263 _, err = o.DecodeFixed32()
266 u, err = o.DecodeVarint()
270 fwire := int(u & 0x7)
271 if fwire == WireEndGroup {
275 err = o.skip(t, ftag, fwire)
281 fmt.Fprintf(os.Stderr, "proto: can't skip wire type %d for %s\n", wire, t)
286 // Unmarshaler is the interface representing objects that can unmarshal themselves.
287 type Unmarshaler interface {
288 Unmarshal([]byte) os.Error
291 // Unmarshal parses the protocol buffer representation in buf and places the
292 // decoded result in pb. If the struct underlying pb does not match
293 // the data in buf, the results can be unpredictable.
294 func Unmarshal(buf []byte, pb interface{}) os.Error {
295 // If the object can unmarshal itself, let it.
296 if u, ok := pb.(Unmarshaler); ok {
297 return u.Unmarshal(buf)
300 return NewBuffer(buf).Unmarshal(pb)
303 // Unmarshal parses the protocol buffer representation in the
304 // Buffer and places the decoded result in pb. If the struct
305 // underlying pb does not match the data in the buffer, the results can be
307 func (p *Buffer) Unmarshal(pb interface{}) os.Error {
308 // If the object can unmarshal itself, let it.
309 if u, ok := pb.(Unmarshaler); ok {
310 err := u.Unmarshal(p.buf[p.index:])
315 mstat := runtime.MemStats.Mallocs
317 typ, base, err := getbase(pb)
322 err = p.unmarshalType(typ, false, base)
324 mstat = runtime.MemStats.Mallocs - mstat
325 stats.Dmalloc += mstat
331 // unmarshalType does the work of unmarshaling a structure.
332 func (o *Buffer) unmarshalType(t reflect.Type, is_group bool, base uintptr) os.Error {
334 prop := GetProperties(st)
335 required, reqFields := prop.reqCount, uint64(0)
336 sbase := getsbase(prop) // scratch area for data items
339 for err == nil && o.index < len(o.buf) {
342 u, err = o.DecodeVarint()
347 if wire == WireEndGroup {
349 return nil // input is satisfied
354 fieldnum, ok := prop.tags[tag]
356 // Maybe it's an extension?
358 iv := unsafe.Unreflect(t, unsafe.Pointer(&o.ptr))
359 if e, ok := iv.(extendableProto); ok && isExtensionField(e, int32(tag)) {
360 if err = o.skip(st, tag, wire); err == nil {
361 e.ExtensionMap()[int32(tag)] = Extension{enc: append([]byte(nil), o.buf[oi:o.index]...)}
365 err = o.skipAndSave(st, tag, wire, base)
368 p := prop.Prop[fieldnum]
371 fmt.Fprintf(os.Stderr, "no protobuf decoder for %s.%s\n", t, st.Field(fieldnum).Name)
375 if wire != WireStartGroup && wire != p.WireType {
376 if wire == WireBytes && p.packedDec != nil {
384 err = dec(o, p, base, sbase)
385 if err == nil && p.Required {
386 // Successfully decoded a required field.
388 // use bitmap for fields 1-64 to catch field reuse.
389 var mask uint64 = 1 << uint64(tag-1)
390 if reqFields&mask == 0 {
391 // new required field
396 // This is imprecise. It can be fooled by a required field
397 // with a tag > 64 that is encoded twice; that's very rare.
398 // A fully correct implementation would require allocating
399 // a data structure, which we would like to avoid.
406 return io.ErrUnexpectedEOF
409 return &ErrRequiredNotSet{st}
415 // Make *pslice have base address base, length 0, and capacity startSize.
416 func initSlice(pslice unsafe.Pointer, base uintptr) {
417 sp := (*reflect.SliceHeader)(pslice)
423 // Individual type decoders
425 // u is the decoded value,
426 // v is a pointer to the field (pointer) in the struct
427 // x is a pointer to the preallocated scratch space to hold the decoded value.
430 func (o *Buffer) dec_bool(p *Properties, base uintptr, sbase uintptr) os.Error {
431 u, err := p.valDec(o)
435 v := (**uint8)(unsafe.Pointer(base + p.offset))
436 x := (*uint8)(unsafe.Pointer(sbase + p.scratch))
443 func (o *Buffer) dec_int32(p *Properties, base uintptr, sbase uintptr) os.Error {
444 u, err := p.valDec(o)
448 v := (**int32)(unsafe.Pointer(base + p.offset))
449 x := (*int32)(unsafe.Pointer(sbase + p.scratch))
456 func (o *Buffer) dec_int64(p *Properties, base uintptr, sbase uintptr) os.Error {
457 u, err := p.valDec(o)
461 v := (**int64)(unsafe.Pointer(base + p.offset))
462 x := (*int64)(unsafe.Pointer(sbase + p.scratch))
469 func (o *Buffer) dec_string(p *Properties, base uintptr, sbase uintptr) os.Error {
470 s, err := o.DecodeStringBytes()
474 v := (**string)(unsafe.Pointer(base + p.offset))
475 x := (*string)(unsafe.Pointer(sbase + p.scratch))
481 // Decode a slice of bytes ([]byte).
482 func (o *Buffer) dec_slice_byte(p *Properties, base uintptr, sbase uintptr) os.Error {
483 b, err := o.DecodeRawBytes(false)
488 x := (*[]uint8)(unsafe.Pointer(base + p.offset))
492 initSlice(unsafe.Pointer(x), sbase+p.scratch)
500 // Decode a slice of bools ([]bool).
501 func (o *Buffer) dec_slice_bool(p *Properties, base uintptr, sbase uintptr) os.Error {
502 u, err := p.valDec(o)
506 x := (*[]bool)(unsafe.Pointer(base + p.offset))
510 initSlice(unsafe.Pointer(x), sbase+p.scratch)
514 *x = append(y, u != 0)
518 // Decode a slice of bools ([]bool) in packed format.
519 func (o *Buffer) dec_slice_packed_bool(p *Properties, base uintptr, sbase uintptr) os.Error {
520 x := (*[]bool)(unsafe.Pointer(base + p.offset))
522 nn, err := o.DecodeVarint()
526 nb := int(nn) // number of bytes of encoded bools
530 initSlice(unsafe.Pointer(x), sbase+p.scratch)
534 for i := 0; i < nb; i++ {
535 u, err := p.valDec(o)
539 y = append(y, u != 0)
546 // Decode a slice of int32s ([]int32).
547 func (o *Buffer) dec_slice_int32(p *Properties, base uintptr, sbase uintptr) os.Error {
548 u, err := p.valDec(o)
552 x := (*[]int32)(unsafe.Pointer(base + p.offset))
556 initSlice(unsafe.Pointer(x), sbase+p.scratch)
560 *x = append(y, int32(u))
564 // Decode a slice of int32s ([]int32) in packed format.
565 func (o *Buffer) dec_slice_packed_int32(p *Properties, base uintptr, sbase uintptr) os.Error {
566 x := (*[]int32)(unsafe.Pointer(base + p.offset))
568 nn, err := o.DecodeVarint()
572 nb := int(nn) // number of bytes of encoded int32s
576 initSlice(unsafe.Pointer(x), sbase+p.scratch)
582 u, err := p.valDec(o)
586 y = append(y, int32(u))
593 // Decode a slice of int64s ([]int64).
594 func (o *Buffer) dec_slice_int64(p *Properties, base uintptr, sbase uintptr) os.Error {
595 u, err := p.valDec(o)
599 x := (*[]int64)(unsafe.Pointer(base + p.offset))
603 initSlice(unsafe.Pointer(x), sbase+p.scratch)
607 *x = append(y, int64(u))
611 // Decode a slice of int64s ([]int64) in packed format.
612 func (o *Buffer) dec_slice_packed_int64(p *Properties, base uintptr, sbase uintptr) os.Error {
613 x := (*[]int64)(unsafe.Pointer(base + p.offset))
615 nn, err := o.DecodeVarint()
619 nb := int(nn) // number of bytes of encoded int64s
623 initSlice(unsafe.Pointer(x), sbase+p.scratch)
629 u, err := p.valDec(o)
633 y = append(y, int64(u))
640 // Decode a slice of strings ([]string).
641 func (o *Buffer) dec_slice_string(p *Properties, base uintptr, sbase uintptr) os.Error {
642 s, err := o.DecodeStringBytes()
646 x := (*[]string)(unsafe.Pointer(base + p.offset))
650 initSlice(unsafe.Pointer(x), sbase+p.scratch)
658 // Decode a slice of slice of bytes ([][]byte).
659 func (o *Buffer) dec_slice_slice_byte(p *Properties, base uintptr, sbase uintptr) os.Error {
660 b, err := o.DecodeRawBytes(true)
664 x := (*[][]byte)(unsafe.Pointer(base + p.offset))
668 initSlice(unsafe.Pointer(x), sbase+p.scratch)
677 func (o *Buffer) dec_struct_group(p *Properties, base uintptr, sbase uintptr) os.Error {
678 ptr := (**struct{})(unsafe.Pointer(base + p.offset))
679 typ := p.stype.Elem()
680 structv := unsafe.New(typ)
681 bas := uintptr(structv)
682 *ptr = (*struct{})(structv)
684 err := o.unmarshalType(p.stype, true, bas)
689 // Decode an embedded message.
690 func (o *Buffer) dec_struct_message(p *Properties, base uintptr, sbase uintptr) (err os.Error) {
691 raw, e := o.DecodeRawBytes(false)
696 ptr := (**struct{})(unsafe.Pointer(base + p.offset))
697 typ := p.stype.Elem()
698 structv := unsafe.New(typ)
699 bas := uintptr(structv)
700 *ptr = (*struct{})(structv)
702 // If the object can unmarshal itself, let it.
703 iv := unsafe.Unreflect(p.stype, unsafe.Pointer(ptr))
704 if u, ok := iv.(Unmarshaler); ok {
705 return u.Unmarshal(raw)
713 err = o.unmarshalType(p.stype, false, bas)
720 // Decode a slice of embedded messages.
721 func (o *Buffer) dec_slice_struct_message(p *Properties, base uintptr, sbase uintptr) os.Error {
722 return o.dec_slice_struct(p, false, base, sbase)
725 // Decode a slice of embedded groups.
726 func (o *Buffer) dec_slice_struct_group(p *Properties, base uintptr, sbase uintptr) os.Error {
727 return o.dec_slice_struct(p, true, base, sbase)
730 // Decode a slice of structs ([]*struct).
731 func (o *Buffer) dec_slice_struct(p *Properties, is_group bool, base uintptr, sbase uintptr) os.Error {
733 x := (*[]*struct{})(unsafe.Pointer(base + p.offset))
736 initSlice(unsafe.Pointer(x), sbase+p.scratch)
740 typ := p.stype.Elem()
741 structv := unsafe.New(typ)
742 bas := uintptr(structv)
743 y = append(y, (*struct{})(structv))
747 err := o.unmarshalType(p.stype, is_group, bas)
751 raw, err := o.DecodeRawBytes(true)
756 // If the object can unmarshal itself, let it.
757 iv := unsafe.Unreflect(p.stype, unsafe.Pointer(&y[len(y)-1]))
758 if u, ok := iv.(Unmarshaler); ok {
759 return u.Unmarshal(raw)
767 err = o.unmarshalType(p.stype, is_group, bas)