swarm/network, swarm/storage: validate chunk size (#17397)
* swarm/network, swarm/storage: validate default chunk size * swarm/bmt, swarm/network, swarm/storage: update BMT hash initialisation * swarm/bmt: move segmentCount to tests * swarm/chunk: change chunk.DefaultSize to be untyped const * swarm/storage: add size validator * swarm/storage: add chunk size validation to localstore * swarm/storage: move validation from localstore to validator * swarm/storage: global chunk rules in MRU
This commit is contained in:
committed by
Balint Gabor
parent
8a040de60b
commit
97887d98da
@@ -25,6 +25,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/ethereum/go-ethereum/metrics"
|
||||
"github.com/ethereum/go-ethereum/swarm/chunk"
|
||||
"github.com/ethereum/go-ethereum/swarm/log"
|
||||
"github.com/ethereum/go-ethereum/swarm/spancontext"
|
||||
opentracing "github.com/opentracing/opentracing-go"
|
||||
@@ -69,10 +70,6 @@ var (
|
||||
errOperationTimedOut = errors.New("operation timed out")
|
||||
)
|
||||
|
||||
const (
|
||||
DefaultChunkSize int64 = 4096
|
||||
)
|
||||
|
||||
type ChunkerParams struct {
|
||||
chunkSize int64
|
||||
hashSize int64
|
||||
@@ -136,7 +133,7 @@ type TreeChunker struct {
|
||||
func TreeJoin(ctx context.Context, addr Address, getter Getter, depth int) *LazyChunkReader {
|
||||
jp := &JoinerParams{
|
||||
ChunkerParams: ChunkerParams{
|
||||
chunkSize: DefaultChunkSize,
|
||||
chunkSize: chunk.DefaultSize,
|
||||
hashSize: int64(len(addr)),
|
||||
},
|
||||
addr: addr,
|
||||
@@ -156,7 +153,7 @@ func TreeSplit(ctx context.Context, data io.Reader, size int64, putter Putter) (
|
||||
tsp := &TreeSplitterParams{
|
||||
SplitterParams: SplitterParams{
|
||||
ChunkerParams: ChunkerParams{
|
||||
chunkSize: DefaultChunkSize,
|
||||
chunkSize: chunk.DefaultSize,
|
||||
hashSize: putter.RefSize(),
|
||||
},
|
||||
reader: data,
|
||||
|
@@ -22,6 +22,7 @@ import (
|
||||
"sync"
|
||||
|
||||
"github.com/ethereum/go-ethereum/crypto/sha3"
|
||||
"github.com/ethereum/go-ethereum/swarm/chunk"
|
||||
"github.com/ethereum/go-ethereum/swarm/storage/encryption"
|
||||
)
|
||||
|
||||
@@ -57,7 +58,7 @@ func NewHasherStore(chunkStore ChunkStore, hashFunc SwarmHasher, toEncrypt bool)
|
||||
refSize := int64(hashSize)
|
||||
if toEncrypt {
|
||||
refSize += encryption.KeyLength
|
||||
chunkEncryption = newChunkEncryption(DefaultChunkSize, refSize)
|
||||
chunkEncryption = newChunkEncryption(chunk.DefaultSize, refSize)
|
||||
}
|
||||
|
||||
return &hasherStore{
|
||||
@@ -190,9 +191,9 @@ func (h *hasherStore) decryptChunkData(chunkData ChunkData, encryptionKey encryp
|
||||
|
||||
// removing extra bytes which were just added for padding
|
||||
length := ChunkData(decryptedSpan).Size()
|
||||
for length > DefaultChunkSize {
|
||||
length = length + (DefaultChunkSize - 1)
|
||||
length = length / DefaultChunkSize
|
||||
for length > chunk.DefaultSize {
|
||||
length = length + (chunk.DefaultSize - 1)
|
||||
length = length / chunk.DefaultSize
|
||||
length *= h.refSize
|
||||
}
|
||||
|
||||
|
@@ -27,6 +27,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/swarm/chunk"
|
||||
"github.com/ethereum/go-ethereum/swarm/log"
|
||||
"github.com/ethereum/go-ethereum/swarm/storage/mock/mem"
|
||||
|
||||
@@ -184,7 +185,7 @@ func testIterator(t *testing.T, mock bool) {
|
||||
t.Fatalf("init dbStore failed: %v", err)
|
||||
}
|
||||
|
||||
chunks := GenerateRandomChunks(DefaultChunkSize, chunkcount)
|
||||
chunks := GenerateRandomChunks(chunk.DefaultSize, chunkcount)
|
||||
|
||||
wg := &sync.WaitGroup{}
|
||||
wg.Add(len(chunks))
|
||||
@@ -294,7 +295,7 @@ func TestLDBStoreWithoutCollectGarbage(t *testing.T) {
|
||||
|
||||
chunks := []*Chunk{}
|
||||
for i := 0; i < n; i++ {
|
||||
c := GenerateRandomChunk(DefaultChunkSize)
|
||||
c := GenerateRandomChunk(chunk.DefaultSize)
|
||||
chunks = append(chunks, c)
|
||||
log.Trace("generate random chunk", "idx", i, "chunk", c)
|
||||
}
|
||||
@@ -344,7 +345,7 @@ func TestLDBStoreCollectGarbage(t *testing.T) {
|
||||
|
||||
chunks := []*Chunk{}
|
||||
for i := 0; i < n; i++ {
|
||||
c := GenerateRandomChunk(DefaultChunkSize)
|
||||
c := GenerateRandomChunk(chunk.DefaultSize)
|
||||
chunks = append(chunks, c)
|
||||
log.Trace("generate random chunk", "idx", i, "chunk", c)
|
||||
}
|
||||
@@ -398,7 +399,7 @@ func TestLDBStoreAddRemove(t *testing.T) {
|
||||
|
||||
chunks := []*Chunk{}
|
||||
for i := 0; i < n; i++ {
|
||||
c := GenerateRandomChunk(DefaultChunkSize)
|
||||
c := GenerateRandomChunk(chunk.DefaultSize)
|
||||
chunks = append(chunks, c)
|
||||
log.Trace("generate random chunk", "idx", i, "chunk", c)
|
||||
}
|
||||
@@ -460,7 +461,7 @@ func TestLDBStoreRemoveThenCollectGarbage(t *testing.T) {
|
||||
|
||||
chunks := []*Chunk{}
|
||||
for i := 0; i < capacity; i++ {
|
||||
c := GenerateRandomChunk(DefaultChunkSize)
|
||||
c := GenerateRandomChunk(chunk.DefaultSize)
|
||||
chunks = append(chunks, c)
|
||||
log.Trace("generate random chunk", "idx", i, "chunk", c)
|
||||
}
|
||||
|
@@ -98,20 +98,16 @@ func NewTestLocalStoreForAddr(params *LocalStoreParams) (*LocalStore, error) {
|
||||
// After the LDBStore.Put, it is ensured that the MemStore
|
||||
// contains the chunk with the same data, but nil ReqC channel.
|
||||
func (ls *LocalStore) Put(ctx context.Context, chunk *Chunk) {
|
||||
if l := len(chunk.SData); l < 9 {
|
||||
log.Debug("incomplete chunk data", "addr", chunk.Addr, "length", l)
|
||||
chunk.SetErrored(ErrChunkInvalid)
|
||||
chunk.markAsStored()
|
||||
return
|
||||
}
|
||||
valid := true
|
||||
// ls.Validators contains a list of one validator per chunk type.
|
||||
// if one validator succeeds, then the chunk is valid
|
||||
for _, v := range ls.Validators {
|
||||
if valid = v.Validate(chunk.Addr, chunk.SData); valid {
|
||||
break
|
||||
}
|
||||
}
|
||||
if !valid {
|
||||
log.Trace("invalid content address", "addr", chunk.Addr)
|
||||
log.Trace("invalid chunk", "addr", chunk.Addr, "len", len(chunk.SData))
|
||||
chunk.SetErrored(ErrChunkInvalid)
|
||||
chunk.markAsStored()
|
||||
return
|
||||
|
@@ -20,6 +20,8 @@ import (
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/ethereum/go-ethereum/swarm/chunk"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -61,7 +63,7 @@ func TestValidator(t *testing.T) {
|
||||
// add content address validator and check puts
|
||||
// bad should fail, good should pass
|
||||
store.Validators = append(store.Validators, NewContentAddressValidator(hashfunc))
|
||||
chunks = GenerateRandomChunks(DefaultChunkSize, 2)
|
||||
chunks = GenerateRandomChunks(chunk.DefaultSize, 2)
|
||||
goodChunk = chunks[0]
|
||||
badChunk = chunks[1]
|
||||
copy(badChunk.SData, goodChunk.SData)
|
||||
@@ -79,7 +81,7 @@ func TestValidator(t *testing.T) {
|
||||
var negV boolTestValidator
|
||||
store.Validators = append(store.Validators, negV)
|
||||
|
||||
chunks = GenerateRandomChunks(DefaultChunkSize, 2)
|
||||
chunks = GenerateRandomChunks(chunk.DefaultSize, 2)
|
||||
goodChunk = chunks[0]
|
||||
badChunk = chunks[1]
|
||||
copy(badChunk.SData, goodChunk.SData)
|
||||
@@ -97,7 +99,7 @@ func TestValidator(t *testing.T) {
|
||||
var posV boolTestValidator = true
|
||||
store.Validators = append(store.Validators, posV)
|
||||
|
||||
chunks = GenerateRandomChunks(DefaultChunkSize, 2)
|
||||
chunks = GenerateRandomChunks(chunk.DefaultSize, 2)
|
||||
goodChunk = chunks[0]
|
||||
badChunk = chunks[1]
|
||||
copy(badChunk.SData, goodChunk.SData)
|
||||
|
@@ -21,17 +21,15 @@ package mru
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
"unsafe"
|
||||
|
||||
"github.com/ethereum/go-ethereum/swarm/chunk"
|
||||
"github.com/ethereum/go-ethereum/swarm/log"
|
||||
"github.com/ethereum/go-ethereum/swarm/storage"
|
||||
)
|
||||
|
||||
const chunkSize = 4096 // temporary until we implement FileStore in the resourcehandler
|
||||
|
||||
type Handler struct {
|
||||
chunkStore *storage.NetStore
|
||||
HashSize int
|
||||
@@ -66,8 +64,7 @@ func init() {
|
||||
}
|
||||
|
||||
// NewHandler creates a new Mutable Resource API
|
||||
func NewHandler(params *HandlerParams) (*Handler, error) {
|
||||
|
||||
func NewHandler(params *HandlerParams) *Handler {
|
||||
rh := &Handler{
|
||||
resources: make(map[uint64]*resource),
|
||||
storeTimeout: defaultStoreTimeout,
|
||||
@@ -82,7 +79,7 @@ func NewHandler(params *HandlerParams) (*Handler, error) {
|
||||
hashPool.Put(hashfunc)
|
||||
}
|
||||
|
||||
return rh, nil
|
||||
return rh
|
||||
}
|
||||
|
||||
// SetStore sets the store backend for the Mutable Resource API
|
||||
@@ -94,9 +91,8 @@ func (h *Handler) SetStore(store *storage.NetStore) {
|
||||
// If it looks like a resource update, the chunk address is checked against the ownerAddr of the update's signature
|
||||
// It implements the storage.ChunkValidator interface
|
||||
func (h *Handler) Validate(chunkAddr storage.Address, data []byte) bool {
|
||||
|
||||
dataLength := len(data)
|
||||
if dataLength < minimumChunkLength {
|
||||
if dataLength < minimumChunkLength || dataLength > chunk.DefaultSize+8 {
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -106,7 +102,7 @@ func (h *Handler) Validate(chunkAddr storage.Address, data []byte) bool {
|
||||
rootAddr, _ := metadataHash(data)
|
||||
valid := bytes.Equal(chunkAddr, rootAddr)
|
||||
if !valid {
|
||||
log.Debug(fmt.Sprintf("Invalid root metadata chunk with address: %s", chunkAddr.Hex()))
|
||||
log.Debug("Invalid root metadata chunk with address", "addr", chunkAddr.Hex())
|
||||
}
|
||||
return valid
|
||||
}
|
||||
@@ -118,7 +114,7 @@ func (h *Handler) Validate(chunkAddr storage.Address, data []byte) bool {
|
||||
// First, deserialize the chunk
|
||||
var r SignedResourceUpdate
|
||||
if err := r.fromChunk(chunkAddr, data); err != nil {
|
||||
log.Debug("Invalid resource chunk with address %s: %s ", chunkAddr.Hex(), err.Error())
|
||||
log.Debug("Invalid resource chunk", "addr", chunkAddr.Hex(), "err", err.Error())
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -126,7 +122,7 @@ func (h *Handler) Validate(chunkAddr storage.Address, data []byte) bool {
|
||||
// that was used to retrieve this chunk
|
||||
// if this validation fails, someone forged a chunk.
|
||||
if !bytes.Equal(chunkAddr, r.updateHeader.UpdateAddr()) {
|
||||
log.Debug("period,version,rootAddr contained in update chunk do not match updateAddr %s", chunkAddr.Hex())
|
||||
log.Debug("period,version,rootAddr contained in update chunk do not match updateAddr", "addr", chunkAddr.Hex())
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -134,7 +130,7 @@ func (h *Handler) Validate(chunkAddr storage.Address, data []byte) bool {
|
||||
// If it fails, it means either the signature is not valid, data is corrupted
|
||||
// or someone is trying to update someone else's resource.
|
||||
if err := r.Verify(); err != nil {
|
||||
log.Debug("Invalid signature: %v", err)
|
||||
log.Debug("Invalid signature", "err", err)
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -172,11 +168,6 @@ func (h *Handler) GetVersion(rootAddr storage.Address) (uint32, error) {
|
||||
return rsrc.version, nil
|
||||
}
|
||||
|
||||
// \TODO should be hashsize * branches from the chosen chunker, implement with FileStore
|
||||
func (h *Handler) chunkSize() int64 {
|
||||
return chunkSize
|
||||
}
|
||||
|
||||
// New creates a new metadata chunk out of the request passed in.
|
||||
func (h *Handler) New(ctx context.Context, request *Request) error {
|
||||
|
||||
|
@@ -30,6 +30,7 @@ import (
|
||||
"github.com/ethereum/go-ethereum/contracts/ens"
|
||||
"github.com/ethereum/go-ethereum/crypto"
|
||||
"github.com/ethereum/go-ethereum/log"
|
||||
"github.com/ethereum/go-ethereum/swarm/chunk"
|
||||
"github.com/ethereum/go-ethereum/swarm/multihash"
|
||||
"github.com/ethereum/go-ethereum/swarm/storage"
|
||||
)
|
||||
@@ -776,14 +777,11 @@ func TestValidatorInStore(t *testing.T) {
|
||||
|
||||
// set up resource handler and add is as a validator to the localstore
|
||||
rhParams := &HandlerParams{}
|
||||
rh, err := NewHandler(rhParams)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rh := NewHandler(rhParams)
|
||||
store.Validators = append(store.Validators, rh)
|
||||
|
||||
// create content addressed chunks, one good, one faulty
|
||||
chunks := storage.GenerateRandomChunks(storage.DefaultChunkSize, 2)
|
||||
chunks := storage.GenerateRandomChunks(chunk.DefaultSize, 2)
|
||||
goodChunk := chunks[0]
|
||||
badChunk := chunks[1]
|
||||
badChunk.SData = goodChunk.SData
|
||||
|
@@ -38,10 +38,7 @@ func (t *TestHandler) Close() {
|
||||
// NewTestHandler creates Handler object to be used for testing purposes.
|
||||
func NewTestHandler(datadir string, params *HandlerParams) (*TestHandler, error) {
|
||||
path := filepath.Join(datadir, testDbDirName)
|
||||
rh, err := NewHandler(params)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("resource handler create fail: %v", err)
|
||||
}
|
||||
rh := NewHandler(params)
|
||||
localstoreparams := storage.NewDefaultLocalStoreParams()
|
||||
localstoreparams.Init(path)
|
||||
localStore, err := storage.NewLocalStore(localstoreparams, nil)
|
||||
|
@@ -20,6 +20,7 @@ import (
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
|
||||
"github.com/ethereum/go-ethereum/swarm/chunk"
|
||||
"github.com/ethereum/go-ethereum/swarm/log"
|
||||
"github.com/ethereum/go-ethereum/swarm/multihash"
|
||||
)
|
||||
@@ -42,7 +43,7 @@ const chunkPrefixLength = 2 + 2
|
||||
//
|
||||
// Minimum size is Header + 1 (minimum data length, enforced)
|
||||
const minimumUpdateDataLength = updateHeaderLength + 1
|
||||
const maxUpdateDataLength = chunkSize - signatureLength - updateHeaderLength - chunkPrefixLength
|
||||
const maxUpdateDataLength = chunk.DefaultSize - signatureLength - updateHeaderLength - chunkPrefixLength
|
||||
|
||||
// binaryPut serializes the resource update information into the given slice
|
||||
func (r *resourceUpdate) binaryPut(serializedData []byte) error {
|
||||
|
@@ -25,6 +25,7 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/ethereum/go-ethereum/swarm/chunk"
|
||||
"github.com/ethereum/go-ethereum/swarm/log"
|
||||
)
|
||||
|
||||
@@ -101,11 +102,11 @@ func NewPyramidSplitterParams(addr Address, reader io.Reader, putter Putter, get
|
||||
New chunks to store are store using the putter which the caller provides.
|
||||
*/
|
||||
func PyramidSplit(ctx context.Context, reader io.Reader, putter Putter, getter Getter) (Address, func(context.Context) error, error) {
|
||||
return NewPyramidSplitter(NewPyramidSplitterParams(nil, reader, putter, getter, DefaultChunkSize)).Split(ctx)
|
||||
return NewPyramidSplitter(NewPyramidSplitterParams(nil, reader, putter, getter, chunk.DefaultSize)).Split(ctx)
|
||||
}
|
||||
|
||||
func PyramidAppend(ctx context.Context, addr Address, reader io.Reader, putter Putter, getter Getter) (Address, func(context.Context) error, error) {
|
||||
return NewPyramidSplitter(NewPyramidSplitterParams(addr, reader, putter, getter, DefaultChunkSize)).Append(ctx)
|
||||
return NewPyramidSplitter(NewPyramidSplitterParams(addr, reader, putter, getter, chunk.DefaultSize)).Append(ctx)
|
||||
}
|
||||
|
||||
// Entry to create a tree node
|
||||
|
@@ -30,6 +30,7 @@ import (
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/crypto/sha3"
|
||||
"github.com/ethereum/go-ethereum/swarm/bmt"
|
||||
"github.com/ethereum/go-ethereum/swarm/chunk"
|
||||
)
|
||||
|
||||
const MaxPO = 16
|
||||
@@ -114,7 +115,9 @@ func MakeHashFunc(hash string) SwarmHasher {
|
||||
case "BMT":
|
||||
return func() SwarmHash {
|
||||
hasher := sha3.NewKeccak256
|
||||
pool := bmt.NewTreePool(hasher, bmt.SegmentCount, bmt.PoolSize)
|
||||
hasherSize := hasher().Size()
|
||||
segmentCount := chunk.DefaultSize / hasherSize
|
||||
pool := bmt.NewTreePool(hasher, segmentCount, bmt.PoolSize)
|
||||
return bmt.New(pool)
|
||||
}
|
||||
}
|
||||
@@ -230,8 +233,8 @@ func GenerateRandomChunk(dataSize int64) *Chunk {
|
||||
func GenerateRandomChunks(dataSize int64, count int) (chunks []*Chunk) {
|
||||
var i int
|
||||
hasher := MakeHashFunc(DefaultHash)()
|
||||
if dataSize > DefaultChunkSize {
|
||||
dataSize = DefaultChunkSize
|
||||
if dataSize > chunk.DefaultSize {
|
||||
dataSize = chunk.DefaultSize
|
||||
}
|
||||
|
||||
for i = 0; i < count; i++ {
|
||||
@@ -345,6 +348,10 @@ func NewContentAddressValidator(hasher SwarmHasher) *ContentAddressValidator {
|
||||
|
||||
// Validate that the given key is a valid content address for the given data
|
||||
func (v *ContentAddressValidator) Validate(addr Address, data []byte) bool {
|
||||
if l := len(data); l < 9 || l > chunk.DefaultSize+8 {
|
||||
return false
|
||||
}
|
||||
|
||||
hasher := v.Hasher()
|
||||
hasher.ResetWithLength(data[:8])
|
||||
hasher.Write(data[8:])
|
||||
|
Reference in New Issue
Block a user