Fuzz local storage test

Includes org name in target ID.

.travis.yml

@@ -4,6 +4,9 @@ sudo: false
 os:
   - linux
 
+services:
+  - docker
+
 go:
   - 1.10.x
   - 1.11.x
@@ -19,6 +22,18 @@ script:
   - go vet ./...
   - go test ./...
 
+jobs:
+  include:
+    - stage: Fuzz regression
+      go: 1.12.x
+      dist: bionic
+      script: ./fuzzit.sh local-regression
+    - stage: Fuzz
+      if: branch = fuzz AND type IN (push)
+      go: 1.12.x
+      dist: bionic
+      script: ./fuzzit.sh fuzzing
+
 before_deploy:
   - mkdir -p release
   - "GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -ldflags -a -tags netgo -ldflags '-s -w -extldflags -static' -o release/transfersh-$TRAVIS_TAG-linux-amd64"
@@ -38,5 +53,5 @@ deploy:
   skip_cleanup: true
   on:
     tags: true
-    go: tip
+    go: 1.12.x
   overwrite: true

README.md

@@ -1,4 +1,4 @@
-# transfer.sh [![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/dutchcoders/transfer.sh?utm_source=badge&utm_medium=badge&utm_campaign=&utm_campaign=pr-badge&utm_content=badge) [![Go Report Card](https://goreportcard.com/badge/github.com/dutchcoders/transfer.sh)](https://goreportcard.com/report/github.com/dutchcoders/transfer.sh) [![Docker pulls](https://img.shields.io/docker/pulls/dutchcoders/transfer.sh.svg)](https://hub.docker.com/r/dutchcoders/transfer.sh/) [![Build Status](https://travis-ci.org/dutchcoders/transfer.sh.svg?branch=master)](https://travis-ci.org/dutchcoders/transfer.sh)
+# transfer.sh [![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/dutchcoders/transfer.sh?utm_source=badge&utm_medium=badge&utm_campaign=&utm_campaign=pr-badge&utm_content=badge) [![Go Report Card](https://goreportcard.com/badge/github.com/dutchcoders/transfer.sh)](https://goreportcard.com/report/github.com/dutchcoders/transfer.sh) [![Docker pulls](https://img.shields.io/docker/pulls/dutchcoders/transfer.sh.svg)](https://hub.docker.com/r/dutchcoders/transfer.sh/) [![Build Status](https://travis-ci.org/dutchcoders/transfer.sh.svg?branch=master)](https://travis-ci.org/dutchcoders/transfer.sh) [![Fuzzit Status](https://app.fuzzit.dev/badge?org_id=transfer.sh)](https://app.fuzzit.dev/orgs/transfer.sh/dashboard)
 
 Easy and fast file sharing from the command-line. This code contains the server with everything you need to create your own instance.
 
@@ -96,6 +96,7 @@ provider | which storage provider to use | (s3, gdrive or local) |
 aws-access-key | aws access key | | AWS_ACCESS_KEY
 aws-secret-key | aws access key | | AWS_SECRET_KEY
 bucket | aws bucket | | BUCKET
+s3-endpoint | Custom S3 endpoint. | |
 s3-region | region of the s3 bucket | eu-west-1 | S3_REGION
 s3-no-multipart | disables s3 multipart upload | false | |
 s3-path-style | Forces path style URLs, required for Minio. | false | |

fuzzit.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+set -xe
+
+# Validate arguments
+if [ "$#" -ne 1 ]; then
+    echo "Usage: $0 <fuzz-type>"
+    exit 1
+fi
+
+# Configure
+NAME=transfersh
+ROOT=./server
+TYPE=$1
+
+# Setup
+export GO111MODULE="off"
+go get -u github.com/dvyukov/go-fuzz/go-fuzz github.com/dvyukov/go-fuzz/go-fuzz-build
+go get -d -v -u ./...
+if [ ! -f fuzzit ]; then
+    wget -q -O fuzzit https://github.com/fuzzitdev/fuzzit/releases/download/v2.4.29/fuzzit_Linux_x86_64
+    chmod a+x fuzzit
+fi
+
+# Fuzz
+function fuzz {
+    FUNC=Fuzz$1
+    TARGET=$2
+    DIR=${3:-$ROOT}
+    go-fuzz-build -libfuzzer -func $FUNC -o fuzzer.a $DIR
+    clang -fsanitize=fuzzer fuzzer.a -o fuzzer
+    ./fuzzit create job --type $TYPE $NAME/$TARGET fuzzer
+}
+fuzz LocalStorage local-storage

server/server_fuzz.go

@@ -0,0 +1,90 @@
+// +build gofuzz
+
+package server
+
+import (
+	"bytes"
+	"io"
+	"math/rand"
+	"reflect"
+)
+
+const applicationOctetStream = "application/octet-stream"
+
+// FuzzLocalStorage tests the Local Storage.
+func FuzzLocalStorage(fuzz []byte) int {
+	var fuzzLength = uint64(len(fuzz))
+	if fuzzLength == 0 {
+		return -1
+	}
+
+	storage, err := NewLocalStorage("/tmp", nil)
+	if err != nil {
+		panic("unable to create local storage")
+	}
+
+	token := Encode(10000000 + int64(rand.Intn(1000000000)))
+	filename := Encode(10000000 + int64(rand.Intn(1000000000))) + ".bin"
+
+	input := bytes.NewReader(fuzz)
+	err = storage.Put(token, filename, input, applicationOctetStream, fuzzLength)
+	if err != nil {
+		panic("unable to save file")
+	}
+
+	contentType, contentLength, err := storage.Head(token, filename)
+	if err != nil {
+		panic("not visible through head")
+	}
+
+	if contentType != applicationOctetStream {
+		panic("incorrect content type")
+	}
+
+	if contentLength != fuzzLength {
+		panic("incorrect content length")
+	}
+
+	output, contentType, contentLength, err := storage.Get(token, filename)
+	if err != nil {
+		panic("not visible through get")
+	}
+
+	if contentType != applicationOctetStream {
+		panic("incorrect content type")
+	}
+
+	if contentLength != fuzzLength {
+		panic("incorrect content length")
+	}
+
+	var length uint64
+	b := make([]byte, len(fuzz))
+	for {
+		n, err := output.Read(b)
+		length += uint64(n)
+		if err == io.EOF {
+			break
+		}
+	}
+
+	if !reflect.DeepEqual(b, fuzz) {
+		panic("incorrect content body")
+	}
+
+	if length != fuzzLength {
+		panic("incorrect content length")
+	}
+
+	err = storage.Delete(token, filename)
+	if err != nil {
+		panic("unable to delete file")
+	}
+
+	_, _, err = storage.Head(token, filename)
+	if !storage.IsNotExist(err) {
+		panic("file not deleted")
+	}
+
+	return 1
+}

server/storage.go

@@ -243,9 +243,7 @@ func (s *S3Storage) Put(token string, filename string, reader io.Reader, content
 
 	// Create an uploader with the session and custom options
 	uploader := s3manager.NewUploader(s.session, func(u *s3manager.Uploader) {
-		u.PartSize = (1 << 20) * 5  // The minimum/default allowed part size is 5MB
 		u.Concurrency = concurrency // default is 5
-		u.MaxUploadParts = concurrency
 		u.LeavePartsOnError = false
 	})