rlp: improve nil pointer handling (#20064)

* rlp: improve nil pointer handling

In both encoder and decoder, the rules for encoding nil pointers were a
bit hard to understand, and didn't leave much choice. Since RLP allows
two empty values (empty list, empty string), any protocol built on RLP
must choose either of these values to represent the null value in a
certain context.

This change adds choice in the form of two new struct tags, "nilString"
and "nilList". These can be used to specify how a nil pointer value is
encoded. The "nil" tag still exists, but its implementation is now
explicit and defines exactly how nil pointers are handled in a single
place.

Another important change in this commit is how nil pointers and the
Encoder interface interact. The EncodeRLP method was previously called
even on nil values, which was supposed to give users a choice of how
their value would be handled when nil. It turns out this is a stupid
idea. If you create a network protocol containing an object defined in
another package, it's better to be able to say that the object should be
a list or string when nil in the definition of the protocol message
rather than defining the encoding of nil on the object itself.

As of this commit, the encoding rules for pointers now take precedence
over the Encoder interface rule. I think the "nil" tag will work fine
for most cases. For special kinds of objects which are a struct in Go
but strings in RLP, code using the object can specify the desired
encoding of nil using the "nilString" and "nilList" tags.

* rlp: propagate struct field type errors

If a struct contained fields of undecodable type, the encoder and
decoder would panic instead of returning an error. Fix this by
propagating type errors in makeStruct{Writer,Decoder} and add a test.

rlp/decode_test.go

@@ -327,6 +327,10 @@ type recstruct struct {
 	Child *recstruct `rlp:"nil"`
 }
 
+type invalidNilTag struct {
+	X []byte `rlp:"nil"`
+}
+
 type invalidTail1 struct {
 	A uint `rlp:"tail"`
 	B string
@@ -353,6 +357,18 @@ type tailPrivateFields struct {
 	x, y bool
 }
 
+type nilListUint struct {
+	X *uint `rlp:"nilList"`
+}
+
+type nilStringSlice struct {
+	X *[]uint `rlp:"nilString"`
+}
+
+type intField struct {
+	X int
+}
+
 var (
 	veryBigInt = big.NewInt(0).Add(
 		big.NewInt(0).Lsh(big.NewInt(0xFFFFFFFFFFFFFF), 16),
@@ -485,20 +501,20 @@ var decodeTests = []decodeTest{
 		error: "rlp: expected input string or byte for uint, decoding into (rlp.recstruct).Child.I",
 	},
 	{
-		input: "C0",
-		ptr:   new(invalidTail1),
-		error: "rlp: invalid struct tag \"tail\" for rlp.invalidTail1.A (must be on last field)",
-	},
-	{
-		input: "C0",
-		ptr:   new(invalidTail2),
-		error: "rlp: invalid struct tag \"tail\" for rlp.invalidTail2.B (field type is not slice)",
+		input: "C103",
+		ptr:   new(intField),
+		error: "rlp: type int is not RLP-serializable (struct field rlp.intField.X)",
 	},
 	{
 		input: "C50102C20102",
 		ptr:   new(tailUint),
 		error: "rlp: expected input string or byte for uint, decoding into (rlp.tailUint).Tail[1]",
 	},
+	{
+		input: "C0",
+		ptr:   new(invalidNilTag),
+		error: `rlp: invalid struct tag "nil" for rlp.invalidNilTag.X (field is not a pointer)`,
+	},
 
 	// struct tag "tail"
 	{
@@ -521,6 +537,16 @@ var decodeTests = []decodeTest{
 		ptr:   new(tailPrivateFields),
 		value: tailPrivateFields{A: 1, Tail: []uint{2, 3}},
 	},
+	{
+		input: "C0",
+		ptr:   new(invalidTail1),
+		error: `rlp: invalid struct tag "tail" for rlp.invalidTail1.A (must be on last field)`,
+	},
+	{
+		input: "C0",
+		ptr:   new(invalidTail2),
+		error: `rlp: invalid struct tag "tail" for rlp.invalidTail2.B (field type is not slice)`,
+	},
 
 	// struct tag "-"
 	{
@@ -529,6 +555,43 @@ var decodeTests = []decodeTest{
 		value: hasIgnoredField{A: 1, C: 2},
 	},
 
+	// struct tag "nilList"
+	{
+		input: "C180",
+		ptr:   new(nilListUint),
+		error: "rlp: wrong kind of empty value (got String, want List) for *uint, decoding into (rlp.nilListUint).X",
+	},
+	{
+		input: "C1C0",
+		ptr:   new(nilListUint),
+		value: nilListUint{},
+	},
+	{
+		input: "C103",
+		ptr:   new(nilListUint),
+		value: func() interface{} {
+			v := uint(3)
+			return nilListUint{X: &v}
+		}(),
+	},
+
+	// struct tag "nilString"
+	{
+		input: "C1C0",
+		ptr:   new(nilStringSlice),
+		error: "rlp: wrong kind of empty value (got List, want String) for *[]uint, decoding into (rlp.nilStringSlice).X",
+	},
+	{
+		input: "C180",
+		ptr:   new(nilStringSlice),
+		value: nilStringSlice{},
+	},
+	{
+		input: "C2C103",
+		ptr:   new(nilStringSlice),
+		value: nilStringSlice{X: &[]uint{3}},
+	},
+
 	// RawValue
 	{input: "01", ptr: new(RawValue), value: RawValue(unhex("01"))},
 	{input: "82FFFF", ptr: new(RawValue), value: RawValue(unhex("82FFFF"))},
@@ -672,6 +735,22 @@ func TestDecodeDecoder(t *testing.T) {
 	}
 }
 
+func TestDecodeDecoderNilPointer(t *testing.T) {
+	var s struct {
+		T1 *testDecoder `rlp:"nil"`
+		T2 *testDecoder
+	}
+	if err := Decode(bytes.NewReader(unhex("C2C002")), &s); err != nil {
+		t.Fatalf("Decode error: %v", err)
+	}
+	if s.T1 != nil {
+		t.Errorf("decoder T1 allocated for empty input (called: %v)", s.T1.called)
+	}
+	if s.T2 == nil || !s.T2.called {
+		t.Errorf("decoder T2 not allocated/called")
+	}
+}
+
 type byteDecoder byte
 
 func (bd *byteDecoder) DecodeRLP(s *Stream) error {