proto/properties.go

   1 // Go support for Protocol Buffers - Google's data interchange format
   2 //
   3 // Copyright 2010 Google Inc.  All rights reserved.
   4 // http://code.google.com/p/goprotobuf/
   5 //
   6 // Redistribution and use in source and binary forms, with or without
   7 // modification, are permitted provided that the following conditions are
   8 // met:
   9 //
  10 //     * Redistributions of source code must retain the above copyright
  11 // notice, this list of conditions and the following disclaimer.
  12 //     * Redistributions in binary form must reproduce the above
  13 // copyright notice, this list of conditions and the following disclaimer
  14 // in the documentation and/or other materials provided with the
  15 // distribution.
  16 //     * Neither the name of Google Inc. nor the names of its
  17 // contributors may be used to endorse or promote products derived from
  18 // this software without specific prior written permission.
  19 //
  20 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  21 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  22 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  23 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  24 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  25 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  26 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  27 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  28 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  29 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  30 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  31
  32 package proto
  33
  34 /*
  35  * Routines for encoding data into the wire format for protocol buffers.
  36  */
  37
  38 import (
  39         "fmt"
  40         "os"
  41         "reflect"
  42         "strconv"
  43         "strings"
  44         "sync"
  45         "unsafe"
  46 )
  47
  48 const debug bool = false
  49
  50 // Constants that identify the encoding of a value on the wire.
  51 const (
  52         WireVarint     = 0
  53         WireFixed64    = 1
  54         WireBytes      = 2
  55         WireStartGroup = 3
  56         WireEndGroup   = 4
  57         WireFixed32    = 5
  58 )
  59
  60 const startSize = 10 // initial slice/string sizes
  61
  62 // Encoders are defined in encoder.go
  63 // An encoder outputs the full representation of a field, including its
  64 // tag and encoder type.
  65 type encoder func(p *Buffer, prop *Properties, base uintptr) os.Error
  66
  67 // A valueEncoder encodes a single integer in a particular encoding.
  68 type valueEncoder func(o *Buffer, x uint64) os.Error
  69
  70 // Decoders are defined in decode.go
  71 // A decoder creates a value from its wire representation.
  72 // Unrecognized subelements are saved in unrec.
  73 type decoder func(p *Buffer, prop *Properties, base uintptr, sbase uintptr) os.Error
  74
  75 // A valueDecoder decodes a single integer in a particular encoding.
  76 type valueDecoder func(o *Buffer) (x uint64, err os.Error)
  77
  78 // StructProperties represents properties for all the fields of a struct.
  79 type StructProperties struct {
  80         Prop      []*Properties  // properties for each field
  81         reqCount  int            // required count
  82         tags      map[int]int    // map from proto tag to struct field number
  83         origNames map[string]int // map from original name to struct field number
  84         nscratch  uintptr        // size of scratch space
  85 }
  86
  87 // Properties represents the protocol-specific behavior of a single struct field.
  88 type Properties struct {
  89         Name       string // name of the field, for error messages
  90         OrigName   string // original name before protocol compiler (always set)
  91         Wire       string
  92         WireType   int
  93         Tag        int
  94         Required   bool
  95         Optional   bool
  96         Repeated   bool
  97         Packed     bool   // relevant for repeated primitives only
  98         Enum       string // set for enum types only
  99         Default    string // default value
 100         def_uint64 uint64
 101
 102         enc     encoder
 103         valEnc  valueEncoder // set for bool and numeric types only
 104         offset  uintptr
 105         tagcode []byte // encoding of EncodeVarint((Tag<<3)|WireType)
 106         tagbuf  [8]byte
 107         stype   reflect.Type
 108
 109         dec     decoder
 110         valDec  valueDecoder // set for bool and numeric types only
 111         scratch uintptr
 112         sizeof  uintptr // calculations of scratch space
 113         alignof uintptr
 114
 115         // If this is a packable field, this will be the decoder for the packed version of the field.
 116         packedDec decoder
 117 }
 118
 119 // String formats the properties in the protobuf struct field tag style.
 120 func (p *Properties) String() string {
 121         s := p.Wire
 122         s = ","
 123         s += strconv.Itoa(p.Tag)
 124         if p.Required {
 125                 s += ",req"
 126         }
 127         if p.Optional {
 128                 s += ",opt"
 129         }
 130         if p.Repeated {
 131                 s += ",rep"
 132         }
 133         if p.Packed {
 134                 s += ",packed"
 135         }
 136         if p.OrigName != p.Name {
 137                 s += ",name=" + p.OrigName
 138         }
 139         if len(p.Enum) > 0 {
 140                 s += ",enum=" + p.Enum
 141         }
 142         if len(p.Default) > 0 {
 143                 s += ",def=" + p.Default
 144         }
 145         return s
 146 }
 147
 148 // Parse populates p by parsing a string in the protobuf struct field tag style.
 149 func (p *Properties) Parse(s string) {
 150         // "bytes,49,opt,def=hello!,name=foo"
 151         fields := strings.Split(s, ",") // breaks def=, but handled below.
 152         if len(fields) < 2 {
 153                 fmt.Fprintf(os.Stderr, "proto: tag has too few fields: %q\n", s)
 154                 return
 155         }
 156
 157         p.Wire = fields[0]
 158         switch p.Wire {
 159         case "varint":
 160                 p.WireType = WireVarint
 161                 p.valEnc = (*Buffer).EncodeVarint
 162                 p.valDec = (*Buffer).DecodeVarint
 163         case "fixed32":
 164                 p.WireType = WireFixed32
 165                 p.valEnc = (*Buffer).EncodeFixed32
 166                 p.valDec = (*Buffer).DecodeFixed32
 167         case "fixed64":
 168                 p.WireType = WireFixed64
 169                 p.valEnc = (*Buffer).EncodeFixed64
 170                 p.valDec = (*Buffer).DecodeFixed64
 171         case "zigzag32":
 172                 p.WireType = WireVarint
 173                 p.valEnc = (*Buffer).EncodeZigzag32
 174                 p.valDec = (*Buffer).DecodeZigzag32
 175         case "zigzag64":
 176                 p.WireType = WireVarint
 177                 p.valEnc = (*Buffer).EncodeZigzag64
 178                 p.valDec = (*Buffer).DecodeZigzag64
 179         case "bytes", "group":
 180                 p.WireType = WireBytes
 181                 // no numeric converter for non-numeric types
 182         default:
 183                 fmt.Fprintf(os.Stderr, "proto: tag has unknown wire type: %q\n", s)
 184                 return
 185         }
 186
 187         var err os.Error
 188         p.Tag, err = strconv.Atoi(fields[1])
 189         if err != nil {
 190                 return
 191         }
 192
 193         for i := 2; i < len(fields); i++ {
 194                 f := fields[i]
 195                 switch {
 196                 case f == "req":
 197                         p.Required = true
 198                 case f == "opt":
 199                         p.Optional = true
 200                 case f == "rep":
 201                         p.Repeated = true
 202                 case f == "packed":
 203                         p.Packed = true
 204                 case len(f) >= 5 && f[0:5] == "name=":
 205                         p.OrigName = f[5:len(f)]
 206                 case len(f) >= 5 && f[0:5] == "enum=":
 207                         p.Enum = f[5:len(f)]
 208                 case len(f) >= 4 && f[0:4] == "def=":
 209                         p.Default = f[4:len(f)] // rest of string
 210                         if i+1 < len(fields) {
 211                                 // Commas aren't escaped, and def is always last.
 212                                 p.Default += "," + strings.Join(fields[i+1:len(fields)], ",")
 213                                 break
 214                         }
 215                 }
 216         }
 217 }
 218
 219 func logNoSliceEnc(t1, t2 reflect.Type) {
 220         fmt.Fprintf(os.Stderr, "proto: no slice oenc for %T = []%T\n", t1, t2)
 221 }
 222
 223 // Initialize the fields for encoding and decoding.
 224 func (p *Properties) setEncAndDec(typ reflect.Type) {
 225         var vbool bool
 226         var vbyte byte
 227         var vint32 int32
 228         var vint64 int64
 229         var vfloat32 float32
 230         var vfloat64 float64
 231         var vstring string
 232         var vslice []byte
 233
 234         p.enc = nil
 235         p.dec = nil
 236
 237         switch t1 := typ; t1.Kind() {
 238         default:
 239                 fmt.Fprintf(os.Stderr, "proto: no coders for %T\n", t1)
 240                 break
 241
 242         case reflect.Ptr:
 243                 switch t2 := t1.Elem(); t2.Kind() {
 244                 default:
 245                         fmt.Fprintf(os.Stderr, "proto: no encoder function for %T -> %T\n", t1, t2)
 246                         break
 247                 case reflect.Bool:
 248                         p.enc = (*Buffer).enc_bool
 249                         p.dec = (*Buffer).dec_bool
 250                         p.alignof = unsafe.Alignof(vbool)
 251                         p.sizeof = unsafe.Sizeof(vbool)
 252                 case reflect.Int32, reflect.Uint32:
 253                         p.enc = (*Buffer).enc_int32
 254                         p.dec = (*Buffer).dec_int32
 255                         p.alignof = unsafe.Alignof(vint32)
 256                         p.sizeof = unsafe.Sizeof(vint32)
 257                 case reflect.Int64, reflect.Uint64:
 258                         p.enc = (*Buffer).enc_int64
 259                         p.dec = (*Buffer).dec_int64
 260                         p.alignof = unsafe.Alignof(vint64)
 261                         p.sizeof = unsafe.Sizeof(vint64)
 262                 case reflect.Float32:
 263                         p.enc = (*Buffer).enc_int32 // can just treat them as bits
 264                         p.dec = (*Buffer).dec_int32
 265                         p.alignof = unsafe.Alignof(vfloat32)
 266                         p.sizeof = unsafe.Sizeof(vfloat32)
 267                 case reflect.Float64:
 268                         p.enc = (*Buffer).enc_int64 // can just treat them as bits
 269                         p.dec = (*Buffer).dec_int64
 270                         p.alignof = unsafe.Alignof(vfloat64)
 271                         p.sizeof = unsafe.Sizeof(vfloat64)
 272                 case reflect.String:
 273                         p.enc = (*Buffer).enc_string
 274                         p.dec = (*Buffer).dec_string
 275                         p.alignof = unsafe.Alignof(vstring)
 276                         p.sizeof = unsafe.Sizeof(vstring) + startSize*unsafe.Sizeof(vbyte)
 277                 case reflect.Struct:
 278                         p.stype = t1
 279                         if p.Wire == "bytes" {
 280                                 p.enc = (*Buffer).enc_struct_message
 281                                 p.dec = (*Buffer).dec_struct_message
 282                         } else {
 283                                 p.enc = (*Buffer).enc_struct_group
 284                                 p.dec = (*Buffer).dec_struct_group
 285                         }
 286                 }
 287
 288         case reflect.Slice:
 289                 switch t2 := t1.Elem(); t2.Kind() {
 290                 default:
 291                         logNoSliceEnc(t1, t2)
 292                         break
 293                 case reflect.Bool:
 294                         if p.Packed {
 295                                 p.enc = (*Buffer).enc_slice_packed_bool
 296                         } else {
 297                                 p.enc = (*Buffer).enc_slice_bool
 298                         }
 299                         p.dec = (*Buffer).dec_slice_bool
 300                         p.packedDec = (*Buffer).dec_slice_packed_bool
 301                         p.alignof = unsafe.Alignof(vbool)
 302                         p.sizeof = startSize * unsafe.Sizeof(vbool)
 303                 case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
 304                         switch t2.Bits() {
 305                         case 32:
 306                                 if p.Packed {
 307                                         p.enc = (*Buffer).enc_slice_packed_int32
 308                                 } else {
 309                                         p.enc = (*Buffer).enc_slice_int32
 310                                 }
 311                                 p.dec = (*Buffer).dec_slice_int32
 312                                 p.packedDec = (*Buffer).dec_slice_packed_int32
 313                                 p.alignof = unsafe.Alignof(vint32)
 314                                 p.sizeof = startSize * unsafe.Sizeof(vint32)
 315                         case 64:
 316                                 if p.Packed {
 317                                         p.enc = (*Buffer).enc_slice_packed_int64
 318                                 } else {
 319                                         p.enc = (*Buffer).enc_slice_int64
 320                                 }
 321                                 p.dec = (*Buffer).dec_slice_int64
 322                                 p.packedDec = (*Buffer).dec_slice_packed_int64
 323                                 p.alignof = unsafe.Alignof(vint64)
 324                                 p.sizeof = startSize * unsafe.Sizeof(vint64)
 325                         case 8:
 326                                 if t2.Kind() == reflect.Uint8 {
 327                                         p.enc = (*Buffer).enc_slice_byte
 328                                         p.dec = (*Buffer).dec_slice_byte
 329                                         p.alignof = unsafe.Alignof(vbyte)
 330                                         p.sizeof = startSize * unsafe.Sizeof(vbyte)
 331                                 }
 332                         default:
 333                                 logNoSliceEnc(t1, t2)
 334                                 break
 335                         }
 336                 case reflect.Float32, reflect.Float64:
 337                         switch t2.Bits() {
 338                         case 32:
 339                                 // can just treat them as bits
 340                                 if p.Packed {
 341                                         p.enc = (*Buffer).enc_slice_packed_int32
 342                                 } else {
 343                                         p.enc = (*Buffer).enc_slice_int32
 344                                 }
 345                                 p.dec = (*Buffer).dec_slice_int32
 346                                 p.packedDec = (*Buffer).dec_slice_packed_int32
 347                                 p.alignof = unsafe.Alignof(vfloat32)
 348                                 p.sizeof = startSize * unsafe.Sizeof(vfloat32)
 349                         case 64:
 350                                 // can just treat them as bits
 351                                 if p.Packed {
 352                                         p.enc = (*Buffer).enc_slice_packed_int64
 353                                 } else {
 354                                         p.enc = (*Buffer).enc_slice_int64
 355                                 }
 356                                 p.dec = (*Buffer).dec_slice_int64
 357                                 p.packedDec = (*Buffer).dec_slice_packed_int64
 358                                 p.alignof = unsafe.Alignof(vfloat64)
 359                                 p.sizeof = startSize * unsafe.Sizeof(vfloat64)
 360                         default:
 361                                 logNoSliceEnc(t1, t2)
 362                                 break
 363                         }
 364                 case reflect.String:
 365                         p.enc = (*Buffer).enc_slice_string
 366                         p.dec = (*Buffer).dec_slice_string
 367                         p.alignof = unsafe.Alignof(vstring)
 368                         p.sizeof = startSize * unsafe.Sizeof(vstring)
 369                 case reflect.Ptr:
 370                         switch t3 := t2.Elem(); t3.Kind() {
 371                         default:
 372                                 fmt.Fprintf(os.Stderr, "proto: no ptr oenc for %T -> %T -> %T\n", t1, t2, t3)
 373                                 break
 374                         case reflect.Struct:
 375                                 p.stype = t2
 376                                 p.enc = (*Buffer).enc_slice_struct_group
 377                                 p.dec = (*Buffer).dec_slice_struct_group
 378                                 if p.Wire == "bytes" {
 379                                         p.enc = (*Buffer).enc_slice_struct_message
 380                                         p.dec = (*Buffer).dec_slice_struct_message
 381                                 }
 382                                 p.alignof = unsafe.Alignof(vslice)
 383                                 p.sizeof = startSize * unsafe.Sizeof(vslice)
 384                         }
 385                 case reflect.Slice:
 386                         switch t2.Elem().Kind() {
 387                         default:
 388                                 fmt.Fprintf(os.Stderr, "proto: no slice elem oenc for %T -> %T -> %T\n", t1, t2, t2.Elem())
 389                                 break
 390                         case reflect.Uint8:
 391                                 p.enc = (*Buffer).enc_slice_slice_byte
 392                                 p.dec = (*Buffer).dec_slice_slice_byte
 393                                 p.alignof = unsafe.Alignof(vslice)
 394                                 p.sizeof = startSize * unsafe.Sizeof(vslice)
 395                         }
 396                 }
 397         }
 398
 399         // precalculate tag code
 400         wire := p.WireType
 401         if p.Packed {
 402                 wire = WireBytes
 403         }
 404         x := uint32(p.Tag)<<3 | uint32(wire)
 405         i := 0
 406         for i = 0; x > 127; i++ {
 407                 p.tagbuf[i] = 0x80 | uint8(x&0x7F)
 408                 x >>= 7
 409         }
 410         p.tagbuf[i] = uint8(x)
 411         p.tagcode = p.tagbuf[0 : i+1]
 412 }
 413
 414 // Init populates the properties from a protocol buffer struct tag.
 415 func (p *Properties) Init(typ reflect.Type, name, tag string, offset uintptr) {
 416         // "bytes,49,opt,def=hello!"
 417         p.Name = name
 418         p.OrigName = name
 419         p.offset = offset
 420
 421         if tag == "" {
 422                 return
 423         }
 424         p.Parse(tag)
 425         p.setEncAndDec(typ)
 426 }
 427
 428 var (
 429         mutex         sync.Mutex
 430         propertiesMap = make(map[reflect.Type]*StructProperties)
 431 )
 432
 433 // GetProperties returns the list of properties for the type represented by t.
 434 func GetProperties(t reflect.Type) *StructProperties {
 435         mutex.Lock()
 436         if prop, ok := propertiesMap[t]; ok {
 437                 mutex.Unlock()
 438                 stats.Chit++
 439                 return prop
 440         }
 441         stats.Cmiss++
 442
 443         prop := new(StructProperties)
 444
 445         // build properties
 446         prop.Prop = make([]*Properties, t.NumField())
 447         prop.origNames = make(map[string]int)
 448         for i := 0; i < t.NumField(); i++ {
 449                 f := t.Field(i)
 450                 p := new(Properties)
 451                 p.Init(f.Type, f.Name, f.Tag.Get("protobuf"), f.Offset)
 452                 if f.Name == "XXX_extensions" { // special case
 453                         var vmap map[int32][]byte
 454                         p.enc = (*Buffer).enc_map
 455                         p.dec = nil // not needed
 456                         p.alignof = unsafe.Alignof(vmap)
 457                         p.sizeof = unsafe.Sizeof(vmap)
 458                 }
 459                 prop.Prop[i] = p
 460                 prop.origNames[p.OrigName] = i
 461                 if debug {
 462                         print(i, " ", f.Name, " ", t.String(), " ")
 463                         if p.Tag > 0 {
 464                                 print(p.String())
 465                         }
 466                         print("\n")
 467                 }
 468                 if p.enc == nil && !strings.HasPrefix(f.Name, "XXX_") {
 469                         fmt.Fprintln(os.Stderr, "proto: no encoder for", f.Name, f.Type.String(), "[GetProperties]")
 470                 }
 471         }
 472
 473         // build required counts
 474         // build scratch offsets
 475         // build tags
 476         reqCount := 0
 477         scratch := uintptr(0)
 478         prop.tags = make(map[int]int)
 479         for i, p := range prop.Prop {
 480                 if p.Required {
 481                         reqCount++
 482                 }
 483                 scratch = align(scratch, p.alignof)
 484                 p.scratch = scratch
 485                 scratch += p.sizeof
 486                 prop.tags[p.Tag] = i
 487         }
 488         prop.reqCount = reqCount
 489         prop.nscratch = scratch
 490
 491         propertiesMap[t] = prop
 492         mutex.Unlock()
 493         return prop
 494 }
 495
 496 // Alignment of the data in the scratch area.  It doesn't have to be
 497 // exact, just conservative.  Returns the first number >= o that divides s.
 498 func align(o uintptr, s uintptr) uintptr {
 499         if s != 0 {
 500                 for o%uintptr(s) != 0 {
 501                         o++
 502                 }
 503         }
 504         return o
 505 }
 506
 507 // Return the field index of the named field.
 508 // Returns nil if there is no such field.
 509 func fieldIndex(t reflect.Type, name string) []int {
 510         if field, ok := t.FieldByName(name); ok {
 511                 return field.Index
 512         }
 513         return nil
 514 }
 515
 516 // Return the Properties object for the x[0]'th field of the structure.
 517 func propByIndex(t reflect.Type, x []int) *Properties {
 518         if len(x) != 1 {
 519                 fmt.Fprintf(os.Stderr, "proto: field index dimension %d (not 1) for type %s\n", len(x), t)
 520                 return nil
 521         }
 522         prop := GetProperties(t)
 523         return prop.Prop[x[0]]
 524 }
 525
 526 // Get the address and type of a pointer to a struct from an interface.
 527 // unsafe.Reflect can do this, but does multiple mallocs.
 528 func getbase(pb interface{}) (t reflect.Type, b uintptr, err os.Error) {
 529         // get pointer
 530         x := *(*[2]uintptr)(unsafe.Pointer(&pb))
 531         b = x[1]
 532         if b == 0 {
 533                 err = ErrNil
 534                 return
 535         }
 536
 537         // get the reflect type of the struct.
 538         t = reflect.TypeOf(pb)
 539         return
 540 }
 541
 542 // Allocate the aux space containing all the decoded data.  The structure
 543 // handed into Unmarshal is filled with pointers to this newly allocated
 544 // data.
 545 func getsbase(prop *StructProperties) uintptr {
 546         var vbyteptr *byte
 547         if prop.nscratch == 0 {
 548                 return 0
 549         }
 550
 551         // allocate the decode space as pointers
 552         // so that the GC will scan it for pointers
 553         n := uintptr(unsafe.Sizeof(vbyteptr))
 554         b := make([]*byte, (prop.nscratch+n-1)/n)
 555         sbase := uintptr(unsafe.Pointer(&b[0]))
 556         return sbase
 557 }
 558
 559 // A global registry of enum types.
 560 // The generated code will register the generated maps by calling RegisterEnum.
 561
 562 var enumNameMaps = make(map[string]map[int32]string)
 563 var enumValueMaps = make(map[string]map[string]int32)
 564
 565 // RegisterEnum is called from the generated code to install the enum descriptor
 566 // maps into the global table to aid parsing ASCII protocol buffers.
 567 func RegisterEnum(typeName string, nameMap map[int32]string, valueMap map[string]int32) {
 568         if _, ok := enumNameMaps[typeName]; ok {
 569                 panic("proto: duplicate enum registered: " + typeName)
 570         }
 571         enumNameMaps[typeName] = nameMap
 572         enumValueMaps[typeName] = valueMap
 573 }