rlp/rlpgen: RLP encoder code generator (#24251)

This change adds a code generator tool for creating EncodeRLP method
implementations. The generated methods will behave identically to the
reflect-based encoder, but run faster because there is no reflection overhead.

Package rlp now provides the EncoderBuffer type for incremental encoding. This
is used by generated code, but the new methods can also be useful for
hand-written encoders.

There is also experimental support for generating DecodeRLP, and some new
methods have been added to the existing Stream type to support this. Creating
decoders with rlpgen is not recommended at this time because the generated
methods create very poor error reporting.

More detail about package rlp changes:

* rlp: externalize struct field processing / validation

This adds a new package, rlp/internal/rlpstruct, in preparation for the
RLP encoder generator.

I think the struct field rules are subtle enough to warrant extracting
this into their own package, even though it means that a bunch of
adapter code is needed for converting to/from rlpstruct.Type.

* rlp: add more decoder methods (for rlpgen)

This adds new methods on rlp.Stream:

- Uint64, Uint32, Uint16, Uint8, BigInt
- ReadBytes for decoding into []byte
- MoreDataInList - useful for optional list elements

* rlp: expose encoder buffer (for rlpgen)

This exposes the internal encoder buffer type for use in EncodeRLP
implementations.

The new EncoderBuffer type is a sort-of 'opaque handle' for a pointer to
encBuffer. It is implemented this way to ensure the global encBuffer pool
is handled correctly.

rlp/rlpgen/testdata/bigint.in.txt

@@ -0,0 +1,10 @@
+// -*- mode: go -*-
+
+package test
+
+import "math/big"
+
+type Test struct {
+	Int      *big.Int
+	IntNoPtr big.Int
+}

rlp/rlpgen/testdata/bigint.out.txt

@@ -0,0 +1,49 @@
+package test
+
+import "github.com/ethereum/go-ethereum/rlp"
+import "io"
+
+func (obj *Test) EncodeRLP(_w io.Writer) error {
+	w := rlp.NewEncoderBuffer(_w)
+	_tmp0 := w.List()
+	if obj.Int == nil {
+		w.Write(rlp.EmptyString)
+	} else {
+		if obj.Int.Sign() == -1 {
+			return rlp.ErrNegativeBigInt
+		}
+		w.WriteBigInt(obj.Int)
+	}
+	if obj.IntNoPtr.Sign() == -1 {
+		return rlp.ErrNegativeBigInt
+	}
+	w.WriteBigInt(&obj.IntNoPtr)
+	w.ListEnd(_tmp0)
+	return w.Flush()
+}
+
+func (obj *Test) DecodeRLP(dec *rlp.Stream) error {
+	var _tmp0 Test
+	{
+		if _, err := dec.List(); err != nil {
+			return err
+		}
+		// Int:
+		_tmp1, err := dec.BigInt()
+		if err != nil {
+			return err
+		}
+		_tmp0.Int = _tmp1
+		// IntNoPtr:
+		_tmp2, err := dec.BigInt()
+		if err != nil {
+			return err
+		}
+		_tmp0.IntNoPtr = (*_tmp2)
+		if err := dec.ListEnd(); err != nil {
+			return err
+		}
+	}
+	*obj = _tmp0
+	return nil
+}

rlp/rlpgen/testdata/nil.in.txt

@@ -0,0 +1,30 @@
+// -*- mode: go -*-
+
+package test
+
+type Aux struct{
+    A uint32
+}
+
+type Test struct{
+    Uint8 *byte `rlp:"nil"`
+    Uint8List *byte `rlp:"nilList"`
+
+    Uint32 *uint32 `rlp:"nil"`
+    Uint32List *uint32 `rlp:"nilList"`
+
+    Uint64 *uint64 `rlp:"nil"`
+    Uint64List *uint64 `rlp:"nilList"`
+
+    String *string `rlp:"nil"`
+    StringList *string `rlp:"nilList"`
+
+    ByteArray *[3]byte `rlp:"nil"`
+    ByteArrayList *[3]byte `rlp:"nilList"`
+
+    ByteSlice *[]byte `rlp:"nil"`
+    ByteSliceList *[]byte `rlp:"nilList"`
+
+    Struct *Aux `rlp:"nil"`
+    StructString *Aux `rlp:"nilString"`
+}

rlp/rlpgen/testdata/nil.out.txt

@@ -0,0 +1,289 @@
+package test
+
+import "github.com/ethereum/go-ethereum/rlp"
+import "io"
+
+func (obj *Test) EncodeRLP(_w io.Writer) error {
+	w := rlp.NewEncoderBuffer(_w)
+	_tmp0 := w.List()
+	if obj.Uint8 == nil {
+		w.Write([]byte{0x80})
+	} else {
+		w.WriteUint64(uint64((*obj.Uint8)))
+	}
+	if obj.Uint8List == nil {
+		w.Write([]byte{0xC0})
+	} else {
+		w.WriteUint64(uint64((*obj.Uint8List)))
+	}
+	if obj.Uint32 == nil {
+		w.Write([]byte{0x80})
+	} else {
+		w.WriteUint64(uint64((*obj.Uint32)))
+	}
+	if obj.Uint32List == nil {
+		w.Write([]byte{0xC0})
+	} else {
+		w.WriteUint64(uint64((*obj.Uint32List)))
+	}
+	if obj.Uint64 == nil {
+		w.Write([]byte{0x80})
+	} else {
+		w.WriteUint64((*obj.Uint64))
+	}
+	if obj.Uint64List == nil {
+		w.Write([]byte{0xC0})
+	} else {
+		w.WriteUint64((*obj.Uint64List))
+	}
+	if obj.String == nil {
+		w.Write([]byte{0x80})
+	} else {
+		w.WriteString((*obj.String))
+	}
+	if obj.StringList == nil {
+		w.Write([]byte{0xC0})
+	} else {
+		w.WriteString((*obj.StringList))
+	}
+	if obj.ByteArray == nil {
+		w.Write([]byte{0x80})
+	} else {
+		w.WriteBytes(obj.ByteArray[:])
+	}
+	if obj.ByteArrayList == nil {
+		w.Write([]byte{0xC0})
+	} else {
+		w.WriteBytes(obj.ByteArrayList[:])
+	}
+	if obj.ByteSlice == nil {
+		w.Write([]byte{0x80})
+	} else {
+		w.WriteBytes((*obj.ByteSlice))
+	}
+	if obj.ByteSliceList == nil {
+		w.Write([]byte{0xC0})
+	} else {
+		w.WriteBytes((*obj.ByteSliceList))
+	}
+	if obj.Struct == nil {
+		w.Write([]byte{0xC0})
+	} else {
+		_tmp1 := w.List()
+		w.WriteUint64(uint64(obj.Struct.A))
+		w.ListEnd(_tmp1)
+	}
+	if obj.StructString == nil {
+		w.Write([]byte{0x80})
+	} else {
+		_tmp2 := w.List()
+		w.WriteUint64(uint64(obj.StructString.A))
+		w.ListEnd(_tmp2)
+	}
+	w.ListEnd(_tmp0)
+	return w.Flush()
+}
+
+func (obj *Test) DecodeRLP(dec *rlp.Stream) error {
+	var _tmp0 Test
+	{
+		if _, err := dec.List(); err != nil {
+			return err
+		}
+		// Uint8:
+		var _tmp2 *byte
+		if _tmp3, _tmp4, err := dec.Kind(); err != nil {
+			return err
+		} else if _tmp4 != 0 || _tmp3 != rlp.String {
+			_tmp1, err := dec.Uint8()
+			if err != nil {
+				return err
+			}
+			_tmp2 = &_tmp1
+		}
+		_tmp0.Uint8 = _tmp2
+		// Uint8List:
+		var _tmp6 *byte
+		if _tmp7, _tmp8, err := dec.Kind(); err != nil {
+			return err
+		} else if _tmp8 != 0 || _tmp7 != rlp.List {
+			_tmp5, err := dec.Uint8()
+			if err != nil {
+				return err
+			}
+			_tmp6 = &_tmp5
+		}
+		_tmp0.Uint8List = _tmp6
+		// Uint32:
+		var _tmp10 *uint32
+		if _tmp11, _tmp12, err := dec.Kind(); err != nil {
+			return err
+		} else if _tmp12 != 0 || _tmp11 != rlp.String {
+			_tmp9, err := dec.Uint32()
+			if err != nil {
+				return err
+			}
+			_tmp10 = &_tmp9
+		}
+		_tmp0.Uint32 = _tmp10
+		// Uint32List:
+		var _tmp14 *uint32
+		if _tmp15, _tmp16, err := dec.Kind(); err != nil {
+			return err
+		} else if _tmp16 != 0 || _tmp15 != rlp.List {
+			_tmp13, err := dec.Uint32()
+			if err != nil {
+				return err
+			}
+			_tmp14 = &_tmp13
+		}
+		_tmp0.Uint32List = _tmp14
+		// Uint64:
+		var _tmp18 *uint64
+		if _tmp19, _tmp20, err := dec.Kind(); err != nil {
+			return err
+		} else if _tmp20 != 0 || _tmp19 != rlp.String {
+			_tmp17, err := dec.Uint64()
+			if err != nil {
+				return err
+			}
+			_tmp18 = &_tmp17
+		}
+		_tmp0.Uint64 = _tmp18
+		// Uint64List:
+		var _tmp22 *uint64
+		if _tmp23, _tmp24, err := dec.Kind(); err != nil {
+			return err
+		} else if _tmp24 != 0 || _tmp23 != rlp.List {
+			_tmp21, err := dec.Uint64()
+			if err != nil {
+				return err
+			}
+			_tmp22 = &_tmp21
+		}
+		_tmp0.Uint64List = _tmp22
+		// String:
+		var _tmp26 *string
+		if _tmp27, _tmp28, err := dec.Kind(); err != nil {
+			return err
+		} else if _tmp28 != 0 || _tmp27 != rlp.String {
+			_tmp25, err := dec.String()
+			if err != nil {
+				return err
+			}
+			_tmp26 = &_tmp25
+		}
+		_tmp0.String = _tmp26
+		// StringList:
+		var _tmp30 *string
+		if _tmp31, _tmp32, err := dec.Kind(); err != nil {
+			return err
+		} else if _tmp32 != 0 || _tmp31 != rlp.List {
+			_tmp29, err := dec.String()
+			if err != nil {
+				return err
+			}
+			_tmp30 = &_tmp29
+		}
+		_tmp0.StringList = _tmp30
+		// ByteArray:
+		var _tmp34 *[3]byte
+		if _tmp35, _tmp36, err := dec.Kind(); err != nil {
+			return err
+		} else if _tmp36 != 0 || _tmp35 != rlp.String {
+			var _tmp33 [3]byte
+			if err := dec.ReadBytes(_tmp33[:]); err != nil {
+				return err
+			}
+			_tmp34 = &_tmp33
+		}
+		_tmp0.ByteArray = _tmp34
+		// ByteArrayList:
+		var _tmp38 *[3]byte
+		if _tmp39, _tmp40, err := dec.Kind(); err != nil {
+			return err
+		} else if _tmp40 != 0 || _tmp39 != rlp.List {
+			var _tmp37 [3]byte
+			if err := dec.ReadBytes(_tmp37[:]); err != nil {
+				return err
+			}
+			_tmp38 = &_tmp37
+		}
+		_tmp0.ByteArrayList = _tmp38
+		// ByteSlice:
+		var _tmp42 *[]byte
+		if _tmp43, _tmp44, err := dec.Kind(); err != nil {
+			return err
+		} else if _tmp44 != 0 || _tmp43 != rlp.String {
+			_tmp41, err := dec.Bytes()
+			if err != nil {
+				return err
+			}
+			_tmp42 = &_tmp41
+		}
+		_tmp0.ByteSlice = _tmp42
+		// ByteSliceList:
+		var _tmp46 *[]byte
+		if _tmp47, _tmp48, err := dec.Kind(); err != nil {
+			return err
+		} else if _tmp48 != 0 || _tmp47 != rlp.List {
+			_tmp45, err := dec.Bytes()
+			if err != nil {
+				return err
+			}
+			_tmp46 = &_tmp45
+		}
+		_tmp0.ByteSliceList = _tmp46
+		// Struct:
+		var _tmp51 *Aux
+		if _tmp52, _tmp53, err := dec.Kind(); err != nil {
+			return err
+		} else if _tmp53 != 0 || _tmp52 != rlp.List {
+			var _tmp49 Aux
+			{
+				if _, err := dec.List(); err != nil {
+					return err
+				}
+				// A:
+				_tmp50, err := dec.Uint32()
+				if err != nil {
+					return err
+				}
+				_tmp49.A = _tmp50
+				if err := dec.ListEnd(); err != nil {
+					return err
+				}
+			}
+			_tmp51 = &_tmp49
+		}
+		_tmp0.Struct = _tmp51
+		// StructString:
+		var _tmp56 *Aux
+		if _tmp57, _tmp58, err := dec.Kind(); err != nil {
+			return err
+		} else if _tmp58 != 0 || _tmp57 != rlp.String {
+			var _tmp54 Aux
+			{
+				if _, err := dec.List(); err != nil {
+					return err
+				}
+				// A:
+				_tmp55, err := dec.Uint32()
+				if err != nil {
+					return err
+				}
+				_tmp54.A = _tmp55
+				if err := dec.ListEnd(); err != nil {
+					return err
+				}
+			}
+			_tmp56 = &_tmp54
+		}
+		_tmp0.StructString = _tmp56
+		if err := dec.ListEnd(); err != nil {
+			return err
+		}
+	}
+	*obj = _tmp0
+	return nil
+}

rlp/rlpgen/testdata/optional.in.txt

@@ -0,0 +1,17 @@
+// -*- mode: go -*-
+
+package test
+
+type Aux struct {
+	A uint64
+}
+
+type Test struct {
+	Uint64      uint64             `rlp:"optional"`
+	Pointer     *uint64            `rlp:"optional"`
+	String      string             `rlp:"optional"`
+	Slice       []uint64           `rlp:"optional"`
+	Array       [3]byte            `rlp:"optional"`
+	NamedStruct Aux                `rlp:"optional"`
+	AnonStruct  struct{ A string } `rlp:"optional"`
+}

rlp/rlpgen/testdata/optional.out.txt

@@ -0,0 +1,153 @@
+package test
+
+import "github.com/ethereum/go-ethereum/rlp"
+import "io"
+
+func (obj *Test) EncodeRLP(_w io.Writer) error {
+	w := rlp.NewEncoderBuffer(_w)
+	_tmp0 := w.List()
+	_tmp1 := obj.Uint64 != 0
+	_tmp2 := obj.Pointer != nil
+	_tmp3 := obj.String != ""
+	_tmp4 := len(obj.Slice) > 0
+	_tmp5 := obj.Array != ([3]byte{})
+	_tmp6 := obj.NamedStruct != (Aux{})
+	_tmp7 := obj.AnonStruct != (struct{ A string }{})
+	if _tmp1 || _tmp2 || _tmp3 || _tmp4 || _tmp5 || _tmp6 || _tmp7 {
+		w.WriteUint64(obj.Uint64)
+	}
+	if _tmp2 || _tmp3 || _tmp4 || _tmp5 || _tmp6 || _tmp7 {
+		if obj.Pointer == nil {
+			w.Write([]byte{0x80})
+		} else {
+			w.WriteUint64((*obj.Pointer))
+		}
+	}
+	if _tmp3 || _tmp4 || _tmp5 || _tmp6 || _tmp7 {
+		w.WriteString(obj.String)
+	}
+	if _tmp4 || _tmp5 || _tmp6 || _tmp7 {
+		_tmp8 := w.List()
+		for _, _tmp9 := range obj.Slice {
+			w.WriteUint64(_tmp9)
+		}
+		w.ListEnd(_tmp8)
+	}
+	if _tmp5 || _tmp6 || _tmp7 {
+		w.WriteBytes(obj.Array[:])
+	}
+	if _tmp6 || _tmp7 {
+		_tmp10 := w.List()
+		w.WriteUint64(obj.NamedStruct.A)
+		w.ListEnd(_tmp10)
+	}
+	if _tmp7 {
+		_tmp11 := w.List()
+		w.WriteString(obj.AnonStruct.A)
+		w.ListEnd(_tmp11)
+	}
+	w.ListEnd(_tmp0)
+	return w.Flush()
+}
+
+func (obj *Test) DecodeRLP(dec *rlp.Stream) error {
+	var _tmp0 Test
+	{
+		if _, err := dec.List(); err != nil {
+			return err
+		}
+		// Uint64:
+		if dec.MoreDataInList() {
+			_tmp1, err := dec.Uint64()
+			if err != nil {
+				return err
+			}
+			_tmp0.Uint64 = _tmp1
+			// Pointer:
+			if dec.MoreDataInList() {
+				_tmp2, err := dec.Uint64()
+				if err != nil {
+					return err
+				}
+				_tmp0.Pointer = &_tmp2
+				// String:
+				if dec.MoreDataInList() {
+					_tmp3, err := dec.String()
+					if err != nil {
+						return err
+					}
+					_tmp0.String = _tmp3
+					// Slice:
+					if dec.MoreDataInList() {
+						var _tmp4 []uint64
+						if _, err := dec.List(); err != nil {
+							return err
+						}
+						for dec.MoreDataInList() {
+							_tmp5, err := dec.Uint64()
+							if err != nil {
+								return err
+							}
+							_tmp4 = append(_tmp4, _tmp5)
+						}
+						if err := dec.ListEnd(); err != nil {
+							return err
+						}
+						_tmp0.Slice = _tmp4
+						// Array:
+						if dec.MoreDataInList() {
+							var _tmp6 [3]byte
+							if err := dec.ReadBytes(_tmp6[:]); err != nil {
+								return err
+							}
+							_tmp0.Array = _tmp6
+							// NamedStruct:
+							if dec.MoreDataInList() {
+								var _tmp7 Aux
+								{
+									if _, err := dec.List(); err != nil {
+										return err
+									}
+									// A:
+									_tmp8, err := dec.Uint64()
+									if err != nil {
+										return err
+									}
+									_tmp7.A = _tmp8
+									if err := dec.ListEnd(); err != nil {
+										return err
+									}
+								}
+								_tmp0.NamedStruct = _tmp7
+								// AnonStruct:
+								if dec.MoreDataInList() {
+									var _tmp9 struct{ A string }
+									{
+										if _, err := dec.List(); err != nil {
+											return err
+										}
+										// A:
+										_tmp10, err := dec.String()
+										if err != nil {
+											return err
+										}
+										_tmp9.A = _tmp10
+										if err := dec.ListEnd(); err != nil {
+											return err
+										}
+									}
+									_tmp0.AnonStruct = _tmp9
+								}
+							}
+						}
+					}
+				}
+			}
+		}
+		if err := dec.ListEnd(); err != nil {
+			return err
+		}
+	}
+	*obj = _tmp0
+	return nil
+}

rlp/rlpgen/testdata/rawvalue.in.txt

@@ -0,0 +1,11 @@
+// -*- mode: go -*-
+
+package test
+
+import "github.com/ethereum/go-ethereum/rlp"
+
+type Test struct {
+	RawValue          rlp.RawValue
+	PointerToRawValue *rlp.RawValue
+	SliceOfRawValue   []rlp.RawValue
+}

rlp/rlpgen/testdata/rawvalue.out.txt

@@ -0,0 +1,64 @@
+package test
+
+import "github.com/ethereum/go-ethereum/rlp"
+import "io"
+
+func (obj *Test) EncodeRLP(_w io.Writer) error {
+	w := rlp.NewEncoderBuffer(_w)
+	_tmp0 := w.List()
+	w.Write(obj.RawValue)
+	if obj.PointerToRawValue == nil {
+		w.Write([]byte{0x80})
+	} else {
+		w.Write((*obj.PointerToRawValue))
+	}
+	_tmp1 := w.List()
+	for _, _tmp2 := range obj.SliceOfRawValue {
+		w.Write(_tmp2)
+	}
+	w.ListEnd(_tmp1)
+	w.ListEnd(_tmp0)
+	return w.Flush()
+}
+
+func (obj *Test) DecodeRLP(dec *rlp.Stream) error {
+	var _tmp0 Test
+	{
+		if _, err := dec.List(); err != nil {
+			return err
+		}
+		// RawValue:
+		_tmp1, err := dec.Raw()
+		if err != nil {
+			return err
+		}
+		_tmp0.RawValue = _tmp1
+		// PointerToRawValue:
+		_tmp2, err := dec.Raw()
+		if err != nil {
+			return err
+		}
+		_tmp0.PointerToRawValue = &_tmp2
+		// SliceOfRawValue:
+		var _tmp3 []rlp.RawValue
+		if _, err := dec.List(); err != nil {
+			return err
+		}
+		for dec.MoreDataInList() {
+			_tmp4, err := dec.Raw()
+			if err != nil {
+				return err
+			}
+			_tmp3 = append(_tmp3, _tmp4)
+		}
+		if err := dec.ListEnd(); err != nil {
+			return err
+		}
+		_tmp0.SliceOfRawValue = _tmp3
+		if err := dec.ListEnd(); err != nil {
+			return err
+		}
+	}
+	*obj = _tmp0
+	return nil
+}

rlp/rlpgen/testdata/uints.in.txt

@@ -0,0 +1,10 @@
+// -*- mode: go -*-
+
+package test
+
+type Test struct{
+     A uint8
+     B uint16
+     C uint32
+     D uint64
+}

rlp/rlpgen/testdata/uints.out.txt

@@ -0,0 +1,53 @@
+package test
+
+import "github.com/ethereum/go-ethereum/rlp"
+import "io"
+
+func (obj *Test) EncodeRLP(_w io.Writer) error {
+	w := rlp.NewEncoderBuffer(_w)
+	_tmp0 := w.List()
+	w.WriteUint64(uint64(obj.A))
+	w.WriteUint64(uint64(obj.B))
+	w.WriteUint64(uint64(obj.C))
+	w.WriteUint64(obj.D)
+	w.ListEnd(_tmp0)
+	return w.Flush()
+}
+
+func (obj *Test) DecodeRLP(dec *rlp.Stream) error {
+	var _tmp0 Test
+	{
+		if _, err := dec.List(); err != nil {
+			return err
+		}
+		// A:
+		_tmp1, err := dec.Uint8()
+		if err != nil {
+			return err
+		}
+		_tmp0.A = _tmp1
+		// B:
+		_tmp2, err := dec.Uint16()
+		if err != nil {
+			return err
+		}
+		_tmp0.B = _tmp2
+		// C:
+		_tmp3, err := dec.Uint32()
+		if err != nil {
+			return err
+		}
+		_tmp0.C = _tmp3
+		// D:
+		_tmp4, err := dec.Uint64()
+		if err != nil {
+			return err
+		}
+		_tmp0.D = _tmp4
+		if err := dec.ListEnd(); err != nil {
+			return err
+		}
+	}
+	*obj = _tmp0
+	return nil
+}