| 
									
										
											  
											
												Swarm MRUs: Adaptive frequency / Predictable lookups / API simplification (#17559)
* swarm/storage/mru: Adaptive Frequency
swarm/storage/mru/lookup: fixed getBaseTime
Added NewEpoch constructor
swarm/api/client: better error handling in GetResource()
swarm/storage/mru: Renamed structures.
Renamed ResourceMetadata to ResourceID. 
Renamed ResourceID.Name to ResourceID.Topic
swarm/storage/mru: Added binarySerializer interface and test tools
swarm/storage/mru/lookup: Changed base time to time and + marshallers
swarm/storage/mru:  Added ResourceID (former resourceMetadata)
swarm/storage/mru: Added ResourceViewId and serialization tests
swarm/storage/mru/lookup: fixed epoch unmarshaller. Added Epoch Equals
swarm/storage/mru: Fixes as per review comments
cmd/swarm: reworded resource create/update help text regarding topic
swarm/storage/mru: Added UpdateLookup and serializer tests
swarm/storage/mru: Added UpdateHeader, serializers and tests
swarm/storage/mru: changed UpdateAddr / epoch to Base()
swarm/storage/mru: Added resourceUpdate serializer and tests
swarm/storage/mru: Added SignedResourceUpdate tests and serializers
swarm/storage/mru/lookup: fixed GetFirstEpoch bug
swarm/storage/mru: refactor, comments, cleanup
Also added tests for Topic
swarm/storage/mru: handler tests pass
swarm/storage/mru: all resource package tests pass
swarm/storage/mru: resource test pass after adding
timestamp checking support
swarm/storage/mru: Added JSON serializers to ResourceIDView structures
swarm/storage/mru: Sever, client, API test pass
swarm/storage/mru: server test pass
swarm/storage/mru: Added topic length check
swarm/storage/mru: removed some literals,
improved "previous lookup" test case
swarm/storage/mru: some fixes and comments as per review
swarm/storage/mru: first working version without metadata chunk
swarm/storage/mru: Various fixes as per review
swarm/storage/mru: client test pass
swarm/storage/mru: resource query strings and manifest-less queries
swarm/storage/mru: simplify naming
swarm/storage/mru: first autofreq working version
swarm/storage/mru: renamed ToValues to AppendValues
swarm/resource/mru: Added ToValues / FromValues for URL query strings
swarm/storage/mru: Changed POST resource to work with query strings.
No more JSON.
swarm/storage/mru: removed resourceid
swarm/storage/mru: Opened up structures
swarm/storage/mru: Merged Request and SignedResourceUpdate
swarm/storage/mru: removed initial data from CLI resource create
swarm/storage/mru: Refactor Topic as a direct fixed-length array
swarm/storage/mru/lookup: Comprehensive GetNextLevel tests
swarm/storage/mru: Added comments
Added length checks in Topic
swarm/storage/mru: fixes in tests and some code comments
swarm/storage/mru/lookup: new optimized lookup algorithm
swarm/api: moved getResourceView to api out of server
swarm/storage/mru: Lookup algorithm working
swarm/storage/mru: comments and renamed NewLookupParams
Deleted commented code
swarm/storage/mru/lookup: renamed Epoch.LaterThan to After
swarm/storage/mru/lookup: Comments and tidying naming
swarm/storage/mru: fix lookup algorithm
swarm/storage/mru: exposed lookup hint
removed updateheader
swarm/storage/mru/lookup: changed GetNextEpoch for initial values
swarm/storage/mru: resource tests pass
swarm/storage/mru: valueSerializer interface and tests
swarm/storage/mru/lookup: Comments, improvements, fixes, more tests
swarm/storage/mru: renamed UpdateLookup to ID, LookupParams to Query
swarm/storage/mru: renamed query receiver var
swarm/cmd: MRU CLI tests
* cmd/swarm: remove rogue fmt
* swarm/storage/mru: Add version / header for future use
* swarm/storage/mru: Fixes/comments as per review
cmd/swarm: remove rogue fmt
swarm/storage/mru: Add version / header for future use-
* swarm/storage/mru: fix linter errors
* cmd/swarm: Speeded up TestCLIResourceUpdate
											
										 
											2018-09-28 12:07:17 +02:00
										 |  |  | // Copyright 2018 The go-ethereum Authors | 
					
						
							|  |  |  | // This file is part of the go-ethereum library. | 
					
						
							|  |  |  | // | 
					
						
							|  |  |  | // The go-ethereum library is free software: you can redistribute it and/or modify | 
					
						
							|  |  |  | // it under the terms of the GNU Lesser General Public License as published by | 
					
						
							|  |  |  | // the Free Software Foundation, either version 3 of the License, or | 
					
						
							|  |  |  | // (at your option) any later version. | 
					
						
							|  |  |  | // | 
					
						
							|  |  |  | // The go-ethereum library is distributed in the hope that it will be useful, | 
					
						
							|  |  |  | // but WITHOUT ANY WARRANTY; without even the implied warranty of | 
					
						
							|  |  |  | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | 
					
						
							|  |  |  | // GNU Lesser General Public License for more details. | 
					
						
							|  |  |  | // | 
					
						
							|  |  |  | // You should have received a copy of the GNU Lesser General Public License | 
					
						
							|  |  |  | // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2018-10-03 09:15:17 +02:00
										 |  |  | package feed | 
					
						
							| 
									
										
											  
											
												Swarm MRUs: Adaptive frequency / Predictable lookups / API simplification (#17559)
* swarm/storage/mru: Adaptive Frequency
swarm/storage/mru/lookup: fixed getBaseTime
Added NewEpoch constructor
swarm/api/client: better error handling in GetResource()
swarm/storage/mru: Renamed structures.
Renamed ResourceMetadata to ResourceID. 
Renamed ResourceID.Name to ResourceID.Topic
swarm/storage/mru: Added binarySerializer interface and test tools
swarm/storage/mru/lookup: Changed base time to time and + marshallers
swarm/storage/mru:  Added ResourceID (former resourceMetadata)
swarm/storage/mru: Added ResourceViewId and serialization tests
swarm/storage/mru/lookup: fixed epoch unmarshaller. Added Epoch Equals
swarm/storage/mru: Fixes as per review comments
cmd/swarm: reworded resource create/update help text regarding topic
swarm/storage/mru: Added UpdateLookup and serializer tests
swarm/storage/mru: Added UpdateHeader, serializers and tests
swarm/storage/mru: changed UpdateAddr / epoch to Base()
swarm/storage/mru: Added resourceUpdate serializer and tests
swarm/storage/mru: Added SignedResourceUpdate tests and serializers
swarm/storage/mru/lookup: fixed GetFirstEpoch bug
swarm/storage/mru: refactor, comments, cleanup
Also added tests for Topic
swarm/storage/mru: handler tests pass
swarm/storage/mru: all resource package tests pass
swarm/storage/mru: resource test pass after adding
timestamp checking support
swarm/storage/mru: Added JSON serializers to ResourceIDView structures
swarm/storage/mru: Sever, client, API test pass
swarm/storage/mru: server test pass
swarm/storage/mru: Added topic length check
swarm/storage/mru: removed some literals,
improved "previous lookup" test case
swarm/storage/mru: some fixes and comments as per review
swarm/storage/mru: first working version without metadata chunk
swarm/storage/mru: Various fixes as per review
swarm/storage/mru: client test pass
swarm/storage/mru: resource query strings and manifest-less queries
swarm/storage/mru: simplify naming
swarm/storage/mru: first autofreq working version
swarm/storage/mru: renamed ToValues to AppendValues
swarm/resource/mru: Added ToValues / FromValues for URL query strings
swarm/storage/mru: Changed POST resource to work with query strings.
No more JSON.
swarm/storage/mru: removed resourceid
swarm/storage/mru: Opened up structures
swarm/storage/mru: Merged Request and SignedResourceUpdate
swarm/storage/mru: removed initial data from CLI resource create
swarm/storage/mru: Refactor Topic as a direct fixed-length array
swarm/storage/mru/lookup: Comprehensive GetNextLevel tests
swarm/storage/mru: Added comments
Added length checks in Topic
swarm/storage/mru: fixes in tests and some code comments
swarm/storage/mru/lookup: new optimized lookup algorithm
swarm/api: moved getResourceView to api out of server
swarm/storage/mru: Lookup algorithm working
swarm/storage/mru: comments and renamed NewLookupParams
Deleted commented code
swarm/storage/mru/lookup: renamed Epoch.LaterThan to After
swarm/storage/mru/lookup: Comments and tidying naming
swarm/storage/mru: fix lookup algorithm
swarm/storage/mru: exposed lookup hint
removed updateheader
swarm/storage/mru/lookup: changed GetNextEpoch for initial values
swarm/storage/mru: resource tests pass
swarm/storage/mru: valueSerializer interface and tests
swarm/storage/mru/lookup: Comments, improvements, fixes, more tests
swarm/storage/mru: renamed UpdateLookup to ID, LookupParams to Query
swarm/storage/mru: renamed query receiver var
swarm/cmd: MRU CLI tests
* cmd/swarm: remove rogue fmt
* swarm/storage/mru: Add version / header for future use
* swarm/storage/mru: Fixes/comments as per review
cmd/swarm: remove rogue fmt
swarm/storage/mru: Add version / header for future use-
* swarm/storage/mru: fix linter errors
* cmd/swarm: Speeded up TestCLIResourceUpdate
											
										 
											2018-09-28 12:07:17 +02:00
										 |  |  | 
 | 
					
						
							|  |  |  | import ( | 
					
						
							|  |  |  | 	"encoding/json" | 
					
						
							|  |  |  | 	"reflect" | 
					
						
							|  |  |  | 	"testing" | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | 	"github.com/ethereum/go-ethereum/common/hexutil" | 
					
						
							|  |  |  | ) | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | // KV mocks a key value store | 
					
						
							|  |  |  | type KV map[string]string | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | func (kv KV) Get(key string) string { | 
					
						
							|  |  |  | 	return kv[key] | 
					
						
							|  |  |  | } | 
					
						
							|  |  |  | func (kv KV) Set(key, value string) { | 
					
						
							|  |  |  | 	kv[key] = value | 
					
						
							|  |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | func compareByteSliceToExpectedHex(t *testing.T, variableName string, actualValue []byte, expectedHex string) { | 
					
						
							|  |  |  | 	if hexutil.Encode(actualValue) != expectedHex { | 
					
						
							|  |  |  | 		t.Fatalf("%s: Expected %s to be %s, got %s", t.Name(), variableName, expectedHex, hexutil.Encode(actualValue)) | 
					
						
							|  |  |  | 	} | 
					
						
							|  |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | func testBinarySerializerRecovery(t *testing.T, bin binarySerializer, expectedHex string) { | 
					
						
							|  |  |  | 	name := reflect.TypeOf(bin).Elem().Name() | 
					
						
							|  |  |  | 	serialized := make([]byte, bin.binaryLength()) | 
					
						
							|  |  |  | 	if err := bin.binaryPut(serialized); err != nil { | 
					
						
							|  |  |  | 		t.Fatalf("%s.binaryPut error when trying to serialize structure: %s", name, err) | 
					
						
							|  |  |  | 	} | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | 	compareByteSliceToExpectedHex(t, name, serialized, expectedHex) | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | 	recovered := reflect.New(reflect.TypeOf(bin).Elem()).Interface().(binarySerializer) | 
					
						
							|  |  |  | 	if err := recovered.binaryGet(serialized); err != nil { | 
					
						
							|  |  |  | 		t.Fatalf("%s.binaryGet error when trying to deserialize structure: %s", name, err) | 
					
						
							|  |  |  | 	} | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | 	if !reflect.DeepEqual(bin, recovered) { | 
					
						
							|  |  |  | 		t.Fatalf("Expected that the recovered %s equals the marshalled %s", name, name) | 
					
						
							|  |  |  | 	} | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | 	serializedWrongLength := make([]byte, 1) | 
					
						
							|  |  |  | 	copy(serializedWrongLength[:], serialized) | 
					
						
							|  |  |  | 	if err := recovered.binaryGet(serializedWrongLength); err == nil { | 
					
						
							|  |  |  | 		t.Fatalf("Expected %s.binaryGet to fail since data is too small", name) | 
					
						
							|  |  |  | 	} | 
					
						
							|  |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | func testBinarySerializerLengthCheck(t *testing.T, bin binarySerializer) { | 
					
						
							|  |  |  | 	name := reflect.TypeOf(bin).Elem().Name() | 
					
						
							|  |  |  | 	// make a slice that is too small to contain the metadata | 
					
						
							|  |  |  | 	serialized := make([]byte, bin.binaryLength()-1) | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | 	if err := bin.binaryPut(serialized); err == nil { | 
					
						
							|  |  |  | 		t.Fatalf("Expected %s.binaryPut to fail, since target slice is too small", name) | 
					
						
							|  |  |  | 	} | 
					
						
							|  |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | func testValueSerializer(t *testing.T, v valueSerializer, expected KV) { | 
					
						
							|  |  |  | 	name := reflect.TypeOf(v).Elem().Name() | 
					
						
							|  |  |  | 	kv := make(KV) | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | 	v.AppendValues(kv) | 
					
						
							|  |  |  | 	if !reflect.DeepEqual(expected, kv) { | 
					
						
							|  |  |  | 		expj, _ := json.Marshal(expected) | 
					
						
							|  |  |  | 		gotj, _ := json.Marshal(kv) | 
					
						
							|  |  |  | 		t.Fatalf("Expected %s.AppendValues to return %s, got %s", name, string(expj), string(gotj)) | 
					
						
							|  |  |  | 	} | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | 	recovered := reflect.New(reflect.TypeOf(v).Elem()).Interface().(valueSerializer) | 
					
						
							|  |  |  | 	err := recovered.FromValues(kv) | 
					
						
							|  |  |  | 	if err != nil { | 
					
						
							|  |  |  | 		t.Fatal(err) | 
					
						
							|  |  |  | 	} | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | 	if !reflect.DeepEqual(recovered, v) { | 
					
						
							|  |  |  | 		t.Fatalf("Expected recovered %s to be the same", name) | 
					
						
							|  |  |  | 	} | 
					
						
							|  |  |  | } |