gaspersic/go-ethereum
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

swarm/storage: Support for uploading 100gb files (#1395)

Browse Source 
* swarm/storage: Fix gouroutine leak while uploading large files
* swarm/storage: Fix pyramid chunker to wrap properly level3 and above
This commit is contained in:
Zahoor Mohamed
2019-06-18 12:16:20 +05:30
committed by acud
parent 604960938b
commit f57d4f0802
5 changed files with 46 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
storage/pyramid.go
View File

@@ -204,7 +204,7 @@ func (pc *PyramidChunker) decrementWorkerCount() {
func (pc *PyramidChunker) Split(ctx context.Context) (k Address, wait func(context.Context) error, err error) {
pc.wg.Add(1)
pc.prepareChunks(ctx, false)
go pc.prepareChunks(ctx, false)
// closes internal error channel if all subprocesses in the workgroup finished
go func() {
@@ -239,7 +239,7 @@ func (pc *PyramidChunker) Append(ctx context.Context) (k Address, wait func(cont
pc.loadTree(ctx)
pc.wg.Add(1)
pc.prepareChunks(ctx, true)
go pc.prepareChunks(ctx, true)
// closes internal error channel if all subprocesses in the workgroup finished
go func() {
@@ -539,6 +539,15 @@ func (pc *PyramidChunker) buildTree(isAppend bool, ent *TreeEntry, chunkWG *sync
if lvlCount >= pc.branches {
endLvl = lvl + 1
compress = true
// Move up the chunk level to see if there is any boundary wrapping
for uprLvl := endLvl; uprLvl < pc.branches; uprLvl++ {
uprLvlCount := int64(len(pc.chunkLevel[uprLvl]))
if uprLvlCount >= pc.branches-1 {
endLvl = endLvl + 1
}
}
break
}
}