* swarm/storage/localstore: most basic database * swarm/storage/localstore: fix typos and comments * swarm/shed: add uint64 field Dec and DecInBatch methods * swarm/storage/localstore: decrement size counter on ModeRemoval update * swarm/storage/localstore: unexport modeAccess and modeRemoval * swarm/storage/localstore: add WithRetrievalCompositeIndex * swarm/storage/localstore: add TestModeSyncing * swarm/storage/localstore: fix test name * swarm/storage/localstore: add TestModeUpload * swarm/storage/localstore: add TestModeRequest * swarm/storage/localstore: add TestModeSynced * swarm/storage/localstore: add TestModeAccess * swarm/storage/localstore: add TestModeRemoval * swarm/storage/localstore: add mock store option for chunk data * swarm/storage/localstore: add TestDB_pullIndex * swarm/storage/localstore: add TestDB_gcIndex * swarm/storage/localstore: change how batches are written * swarm/storage/localstore: add updateOnAccess function * swarm/storage/localhost: add DB.gcSize * swarm/storage/localstore: update comments * swarm/storage/localstore: add BenchmarkNew * swarm/storage/localstore: add retrieval tests benchmarks * swarm/storage/localstore: accessors redesign * swarm/storage/localstore: add semaphore for updateGC goroutine * swarm/storage/localstore: implement basic garbage collection * swarm/storage/localstore: optimize collectGarbage * swarm/storage/localstore: add more garbage collection tests cases * swarm/shed, swarm/storage/localstore: rename IndexItem to Item * swarm/shed: add Index.CountFrom * swarm/storage/localstore: persist gcSize * swarm/storage/localstore: remove composite retrieval index * swarm/shed: IterateWithPrefix and IterateWithPrefixFrom Index functions * swarm/storage/localstore: writeGCSize function with leveldb batch * swarm/storage/localstore: unexport modeSetRemove * swarm/storage/localstore: update writeGCSizeWorker comment * swarm/storage/localstore: add triggerGarbageCollection function * swarm/storage/localstore: call writeGCSize on DB Close * swarm/storage/localstore: additional comment in writeGCSizeWorker * swarm/storage/localstore: add MetricsPrefix option * swarm/storage/localstore: fix a typo * swamr/shed: only one Index Iterate function * swarm/storage/localstore: use shed Iterate function * swarm/shed: pass a new byte slice copy to index decode functions * swarm/storage/localstore: implement feed subscriptions * swarm/storage/localstore: add more subscriptions tests * swarm/storage/localsore: add parallel upload test * swarm/storage/localstore: use storage.MaxPO in subscription tests * swarm/storage/localstore: subscription of addresses instead chunks * swarm/storage/localstore: lock item address in collectGarbage iterator * swarm/storage/localstore: fix TestSubscribePull to include MaxPO * swarm/storage/localstore: improve subscriptions * swarm/storage/localstore: add TestDB_SubscribePull_sinceAndUntil test * swarm/storage/localstore: adjust pull sync tests * swarm/storage/localstore: remove writeGCSizeDelay and use literal * swarm/storage/localstore: adjust subscriptions tests delays and comments * swarm/storage/localstore: add godoc package overview * swarm/storage/localstore: fix a typo * swarm/storage/localstore: update package overview * swarm/storage/localstore: remove repeated index change * swarm/storage/localstore: rename ChunkInfo to ChunkDescriptor * swarm/storage/localstore: add comment in collectGarbageWorker * swarm/storage/localstore: replace atomics with mutexes for gcSize and tests * swarm/storage/localstore: protect addrs map in pull subs tests * swarm/storage/localstore: protect slices in push subs test * swarm/storage/localstore: protect chunks in TestModePutUpload_parallel * swarm/storage/localstore: fix a race in TestDB_updateGCSem defers * swarm/storage/localstore: remove parallel flag from tests * swarm/storage/localstore: fix a race in testDB_collectGarbageWorker * swarm/storage/localstore: remove unused code * swarm/storage/localstore: add more context to pull sub log messages * swarm/storage/localstore: BenchmarkPutUpload and global lock option * swarm/storage/localstore: pre-generate chunks in BenchmarkPutUpload * swarm/storage/localstore: correct useGlobalLock in collectGarbage * swarm/storage/localstore: fix typos and update comments * swarm/storage/localstore: update writeGCSize comment * swarm/storage/localstore: global batch write lock * swarm/storage/localstore: remove global lock option * swarm/storage/localstore: simplify DB.Close
		
			
				
	
	
		
			202 lines
		
	
	
		
			5.4 KiB
		
	
	
	
		
			Go
		
	
	
	
	
	
			
		
		
	
	
			202 lines
		
	
	
		
			5.4 KiB
		
	
	
	
		
			Go
		
	
	
	
	
	
| // Copyright 2018 The go-ethereum Authors
 | |
| // This file is part of the go-ethereum library.
 | |
| //
 | |
| // The go-ethereum library is free software: you can redistribute it and/or modify
 | |
| // it under the terms of the GNU Lesser General Public License as published by
 | |
| // the Free Software Foundation, either version 3 of the License, or
 | |
| // (at your option) any later version.
 | |
| //
 | |
| // The go-ethereum library is distributed in the hope that it will be useful,
 | |
| // but WITHOUT ANY WARRANTY; without even the implied warranty of
 | |
| // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 | |
| // GNU Lesser General Public License for more details.
 | |
| //
 | |
| // You should have received a copy of the GNU Lesser General Public License
 | |
| // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 | |
| 
 | |
| package localstore
 | |
| 
 | |
| import (
 | |
| 	"github.com/ethereum/go-ethereum/swarm/chunk"
 | |
| 	"github.com/syndtr/goleveldb/leveldb"
 | |
| )
 | |
| 
 | |
| // ModeSet enumerates different Setter modes.
 | |
| type ModeSet int
 | |
| 
 | |
| // Setter modes.
 | |
| const (
 | |
| 	// ModeSetAccess: when an update request is received for a chunk or chunk is retrieved for delivery
 | |
| 	ModeSetAccess ModeSet = iota
 | |
| 	// ModeSetSync: when push sync receipt is received
 | |
| 	ModeSetSync
 | |
| 	// modeSetRemove: when GC-d
 | |
| 	// unexported as no external packages should remove chunks from database
 | |
| 	modeSetRemove
 | |
| )
 | |
| 
 | |
| // Setter sets the state of a particular
 | |
| // Chunk in database by changing indexes.
 | |
| type Setter struct {
 | |
| 	db   *DB
 | |
| 	mode ModeSet
 | |
| }
 | |
| 
 | |
| // NewSetter returns a new Setter on database
 | |
| // with a specific Mode.
 | |
| func (db *DB) NewSetter(mode ModeSet) *Setter {
 | |
| 	return &Setter{
 | |
| 		mode: mode,
 | |
| 		db:   db,
 | |
| 	}
 | |
| }
 | |
| 
 | |
| // Set updates database indexes for a specific
 | |
| // chunk represented by the address.
 | |
| func (s *Setter) Set(addr chunk.Address) (err error) {
 | |
| 	return s.db.set(s.mode, addr)
 | |
| }
 | |
| 
 | |
| // set updates database indexes for a specific
 | |
| // chunk represented by the address.
 | |
| // It acquires lockAddr to protect two calls
 | |
| // of this function for the same address in parallel.
 | |
| func (db *DB) set(mode ModeSet, addr chunk.Address) (err error) {
 | |
| 	// protect parallel updates
 | |
| 	db.batchMu.Lock()
 | |
| 	defer db.batchMu.Unlock()
 | |
| 
 | |
| 	batch := new(leveldb.Batch)
 | |
| 
 | |
| 	// variables that provide information for operations
 | |
| 	// to be done after write batch function successfully executes
 | |
| 	var gcSizeChange int64   // number to add or subtract from gcSize
 | |
| 	var triggerPullFeed bool // signal pull feed subscriptions to iterate
 | |
| 
 | |
| 	item := addressToItem(addr)
 | |
| 
 | |
| 	switch mode {
 | |
| 	case ModeSetAccess:
 | |
| 		// add to pull, insert to gc
 | |
| 
 | |
| 		// need to get access timestamp here as it is not
 | |
| 		// provided by the access function, and it is not
 | |
| 		// a property of a chunk provided to Accessor.Put.
 | |
| 
 | |
| 		i, err := db.retrievalDataIndex.Get(item)
 | |
| 		switch err {
 | |
| 		case nil:
 | |
| 			item.StoreTimestamp = i.StoreTimestamp
 | |
| 		case leveldb.ErrNotFound:
 | |
| 			db.pushIndex.DeleteInBatch(batch, item)
 | |
| 			item.StoreTimestamp = now()
 | |
| 		default:
 | |
| 			return err
 | |
| 		}
 | |
| 
 | |
| 		i, err = db.retrievalAccessIndex.Get(item)
 | |
| 		switch err {
 | |
| 		case nil:
 | |
| 			item.AccessTimestamp = i.AccessTimestamp
 | |
| 			db.gcIndex.DeleteInBatch(batch, item)
 | |
| 			gcSizeChange--
 | |
| 		case leveldb.ErrNotFound:
 | |
| 			// the chunk is not accessed before
 | |
| 		default:
 | |
| 			return err
 | |
| 		}
 | |
| 		item.AccessTimestamp = now()
 | |
| 		db.retrievalAccessIndex.PutInBatch(batch, item)
 | |
| 		db.pullIndex.PutInBatch(batch, item)
 | |
| 		triggerPullFeed = true
 | |
| 		db.gcIndex.PutInBatch(batch, item)
 | |
| 		gcSizeChange++
 | |
| 
 | |
| 	case ModeSetSync:
 | |
| 		// delete from push, insert to gc
 | |
| 
 | |
| 		// need to get access timestamp here as it is not
 | |
| 		// provided by the access function, and it is not
 | |
| 		// a property of a chunk provided to Accessor.Put.
 | |
| 		i, err := db.retrievalDataIndex.Get(item)
 | |
| 		if err != nil {
 | |
| 			if err == leveldb.ErrNotFound {
 | |
| 				// chunk is not found,
 | |
| 				// no need to update gc index
 | |
| 				// just delete from the push index
 | |
| 				// if it is there
 | |
| 				db.pushIndex.DeleteInBatch(batch, item)
 | |
| 				return nil
 | |
| 			}
 | |
| 			return err
 | |
| 		}
 | |
| 		item.StoreTimestamp = i.StoreTimestamp
 | |
| 
 | |
| 		i, err = db.retrievalAccessIndex.Get(item)
 | |
| 		switch err {
 | |
| 		case nil:
 | |
| 			item.AccessTimestamp = i.AccessTimestamp
 | |
| 			db.gcIndex.DeleteInBatch(batch, item)
 | |
| 			gcSizeChange--
 | |
| 		case leveldb.ErrNotFound:
 | |
| 			// the chunk is not accessed before
 | |
| 		default:
 | |
| 			return err
 | |
| 		}
 | |
| 		item.AccessTimestamp = now()
 | |
| 		db.retrievalAccessIndex.PutInBatch(batch, item)
 | |
| 		db.pushIndex.DeleteInBatch(batch, item)
 | |
| 		db.gcIndex.PutInBatch(batch, item)
 | |
| 		gcSizeChange++
 | |
| 
 | |
| 	case modeSetRemove:
 | |
| 		// delete from retrieve, pull, gc
 | |
| 
 | |
| 		// need to get access timestamp here as it is not
 | |
| 		// provided by the access function, and it is not
 | |
| 		// a property of a chunk provided to Accessor.Put.
 | |
| 
 | |
| 		i, err := db.retrievalAccessIndex.Get(item)
 | |
| 		switch err {
 | |
| 		case nil:
 | |
| 			item.AccessTimestamp = i.AccessTimestamp
 | |
| 		case leveldb.ErrNotFound:
 | |
| 		default:
 | |
| 			return err
 | |
| 		}
 | |
| 		i, err = db.retrievalDataIndex.Get(item)
 | |
| 		if err != nil {
 | |
| 			return err
 | |
| 		}
 | |
| 		item.StoreTimestamp = i.StoreTimestamp
 | |
| 
 | |
| 		db.retrievalDataIndex.DeleteInBatch(batch, item)
 | |
| 		db.retrievalAccessIndex.DeleteInBatch(batch, item)
 | |
| 		db.pullIndex.DeleteInBatch(batch, item)
 | |
| 		db.gcIndex.DeleteInBatch(batch, item)
 | |
| 		// a check is needed for decrementing gcSize
 | |
| 		// as delete is not reporting if the key/value pair
 | |
| 		// is deleted or not
 | |
| 		if _, err := db.gcIndex.Get(item); err == nil {
 | |
| 			gcSizeChange = -1
 | |
| 		}
 | |
| 
 | |
| 	default:
 | |
| 		return ErrInvalidMode
 | |
| 	}
 | |
| 
 | |
| 	err = db.incGCSizeInBatch(batch, gcSizeChange)
 | |
| 	if err != nil {
 | |
| 		return err
 | |
| 	}
 | |
| 
 | |
| 	err = db.shed.WriteBatch(batch)
 | |
| 	if err != nil {
 | |
| 		return err
 | |
| 	}
 | |
| 	if triggerPullFeed {
 | |
| 		db.triggerPullSubscriptions(db.po(item.Address))
 | |
| 	}
 | |
| 	return nil
 | |
| }
 |