Fuzz local storage test

Includes org name in target ID.

.travis.yml

@@ -4,6 +4,9 @@ sudo: false
 os:
   - linux
 
+services:
+  - docker
+
 go:
   - 1.10.x
   - 1.11.x
@@ -18,3 +21,37 @@ script:
   - go build -v .
   - go vet ./...
   - go test ./...
+
+jobs:
+  include:
+    - stage: Fuzz regression
+      go: 1.12.x
+      dist: bionic
+      script: ./fuzzit.sh local-regression
+    - stage: Fuzz
+      if: branch = fuzz AND type IN (push)
+      go: 1.12.x
+      dist: bionic
+      script: ./fuzzit.sh fuzzing
+
+before_deploy:
+  - mkdir -p release
+  - "GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -ldflags -a -tags netgo -ldflags '-s -w -extldflags -static' -o release/transfersh-$TRAVIS_TAG-linux-amd64"
+  - "GOOS=linux GOARCH=arm GOARM=7 CGO_ENABLED=0 go build -ldflags -a -tags netgo -ldflags '-s -w -extldflags -static' -o release/transfersh-$TRAVIS_TAG-linux-armv7"
+  - "GOOS=darwin GOARCH=amd64 CGO_ENABLED=0 go build -ldflags -a -tags netgo -ldflags '-s -w -extldflags -static' -o release/transfersh-$TRAVIS_TAG-darwin-amd64"
+  - "GOOS=windows GOARCH=amd64 CGO_ENABLED=0 go build -ldflags -a -tags netgo -ldflags '-s -w -extldflags -static' -o release/transfersh-$TRAVIS_TAG-win-amd64.exe"
+
+deploy:
+  provider: releases
+  api_key:
+    secure: cOuMGyvrl/9GX3TZFL+Vq++2Bv5Hlb3VfXSYONfeAj+1AXI3Y+tPruy/XnWpa1MUxkvFuIhea3sUAiKfwhHip9csCmMUhDJtaTU9apsxRkyF/OFrWb7/FlbnqYuAwnp91ImvtSlnubg2VHTjhBA6ycNQF7WZcJEMVMsAtC/nSY4=
+  file:
+    - "release/transfersh-$TRAVIS_TAG-linux-amd64"
+    - "release/transfersh-$TRAVIS_TAG-linux-armv7"
+    - "release/transfersh-$TRAVIS_TAG-darwin-amd64"
+    - "release/transfersh-$TRAVIS_TAG-win-amd64.exe"
+  skip_cleanup: true
+  on:
+    tags: true
+    go: 1.12.x
+  overwrite: true

README.md

@@ -1,4 +1,4 @@
-# transfer.sh [![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/dutchcoders/transfer.sh?utm_source=badge&utm_medium=badge&utm_campaign=&utm_campaign=pr-badge&utm_content=badge) [![Go Report Card](https://goreportcard.com/badge/github.com/dutchcoders/transfer.sh)](https://goreportcard.com/report/github.com/dutchcoders/transfer.sh) [![Docker pulls](https://img.shields.io/docker/pulls/dutchcoders/transfer.sh.svg)](https://hub.docker.com/r/dutchcoders/transfer.sh/) [![Build Status](https://travis-ci.org/dutchcoders/transfer.sh.svg?branch=master)](https://travis-ci.org/dutchcoders/transfer.sh)
+# transfer.sh [![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/dutchcoders/transfer.sh?utm_source=badge&utm_medium=badge&utm_campaign=&utm_campaign=pr-badge&utm_content=badge) [![Go Report Card](https://goreportcard.com/badge/github.com/dutchcoders/transfer.sh)](https://goreportcard.com/report/github.com/dutchcoders/transfer.sh) [![Docker pulls](https://img.shields.io/docker/pulls/dutchcoders/transfer.sh.svg)](https://hub.docker.com/r/dutchcoders/transfer.sh/) [![Build Status](https://travis-ci.org/dutchcoders/transfer.sh.svg?branch=master)](https://travis-ci.org/dutchcoders/transfer.sh) [![Fuzzit Status](https://app.fuzzit.dev/badge?org_id=transfer.sh)](https://app.fuzzit.dev/orgs/transfer.sh/dashboard)
 
 Easy and fast file sharing from the command-line. This code contains the server with everything you need to create your own instance.
 
@@ -58,76 +58,9 @@ curl -sD - --upload-file ./hello https://transfer.sh/hello.txt | grep 'X-Url-Del
 X-Url-Delete: https://transfer.sh/hello.txt/BAYh0/hello.txt/PDw0NHPcqU
 ```
 
-## Add alias to .bashrc or .zshrc
+## Examples
 
-### Using curl
-```bash
-transfer() {
-    curl --progress-bar --upload-file "$1" https://transfer.sh/$(basename "$1") | tee /dev/null;
-    echo
-}
-
-alias transfer=transfer
-```
-
-### Using wget
-```bash
-transfer() {
-    wget -t 1 -qO - --method=PUT --body-file="$1" --header="Content-Type: $(file -b --mime-type "$1")" https://transfer.sh/$(basename "$1");
-    echo
-}
-
-alias transfer=transfer
-```
-
-## Add alias for fish-shell
-
-### Using curl
-```fish
-function transfer --description 'Upload a file to transfer.sh'
-    if [ $argv[1] ]
-        # write to output to tmpfile because of progress bar
-        set -l tmpfile ( mktemp -t transferXXXXXX )
-        curl --progress-bar --upload-file "$argv[1]" https://transfer.sh/(basename $argv[1]) >> $tmpfile
-        cat $tmpfile
-        command rm -f $tmpfile
-    else
-        echo 'usage: transfer FILE_TO_TRANSFER'
-    end
-end
-
-funcsave transfer
-```
-
-### Using wget
-```fish
-function transfer --description 'Upload a file to transfer.sh'
-    if [ $argv[1] ]
-        wget -t 1 -qO - --method=PUT --body-file="$argv[1]" --header="Content-Type: (file -b --mime-type $argv[1])" https://transfer.sh/(basename $argv[1])
-    else
-        echo 'usage: transfer FILE_TO_TRANSFER'
-    end
-end
-
-funcsave transfer
-```
-
-Now run it like this:
-```bash
-$ transfer test.txt
-```
-
-## Add alias on Windows
-
-Put a file called `transfer.cmd` somewhere in your PATH with this inside it:
-```cmd
-@echo off
-setlocal
-:: use env vars to pass names to PS, to avoid escaping issues
-set FN=%~nx1
-set FULL=%1
-powershell -noprofile -command "$(Invoke-Webrequest -Method put -Infile $Env:FULL https://transfer.sh/$Env:FN).Content"
-```
+See good usage examples on [examples.md](examples.md)
 
 ## Link aliases
 
@@ -163,6 +96,7 @@ provider | which storage provider to use | (s3, gdrive or local) |
 aws-access-key | aws access key | | AWS_ACCESS_KEY
 aws-secret-key | aws access key | | AWS_SECRET_KEY
 bucket | aws bucket | | BUCKET
+s3-endpoint | Custom S3 endpoint. | |
 s3-region | region of the s3 bucket | eu-west-1 | S3_REGION
 s3-no-multipart | disables s3 multipart upload | false | |
 s3-path-style | Forces path style URLs, required for Minio. | false | |

cmd/cmd.go

@@ -12,7 +12,7 @@ import (
 	"google.golang.org/api/googleapi"
 )
 
-var Version = "1.1.0"
+var Version = "1.1.2"
 var helpTemplate = `NAME:
 {{.Name}} - {{.Usage}}
 

examples.md

@@ -0,0 +1,176 @@
+# Table of Contents
+
+* [Aliases](#aliases)
+* [Uploading and downloading](#uploading-and-downloading)
+* [Archiving and backups](#archiving-and-backups)
+* [Encrypting and decrypting](#encrypting-and-decrypting)
+* [Scanning for viruses](#scanning-for-viruses)
+
+## Aliases
+<a name="aliases"/>
+
+## Add alias to .bashrc or .zshrc
+
+### Using curl
+```bash
+transfer() {
+    curl --progress-bar --upload-file "$1" https://transfer.sh/$(basename "$1") | tee /dev/null;
+    echo
+}
+
+alias transfer=transfer
+```
+
+### Using wget
+```bash
+transfer() {
+    wget -t 1 -qO - --method=PUT --body-file="$1" --header="Content-Type: $(file -b --mime-type "$1")" https://transfer.sh/$(basename "$1");
+    echo
+}
+
+alias transfer=transfer
+```
+
+## Add alias for fish-shell
+
+### Using curl
+```fish
+function transfer --description 'Upload a file to transfer.sh'
+    if [ $argv[1] ]
+        # write to output to tmpfile because of progress bar
+        set -l tmpfile ( mktemp -t transferXXXXXX )
+        curl --progress-bar --upload-file "$argv[1]" https://transfer.sh/(basename $argv[1]) >> $tmpfile
+        cat $tmpfile
+        command rm -f $tmpfile
+    else
+        echo 'usage: transfer FILE_TO_TRANSFER'
+    end
+end
+
+funcsave transfer
+```
+
+### Using wget
+```fish
+function transfer --description 'Upload a file to transfer.sh'
+    if [ $argv[1] ]
+        wget -t 1 -qO - --method=PUT --body-file="$argv[1]" --header="Content-Type: (file -b --mime-type $argv[1])" https://transfer.sh/(basename $argv[1])
+    else
+        echo 'usage: transfer FILE_TO_TRANSFER'
+    end
+end
+
+funcsave transfer
+```
+
+Now run it like this:
+```bash
+$ transfer test.txt
+```
+
+## Add alias on Windows
+
+Put a file called `transfer.cmd` somewhere in your PATH with this inside it:
+```cmd
+@echo off
+setlocal
+:: use env vars to pass names to PS, to avoid escaping issues
+set FN=%~nx1
+set FULL=%1
+powershell -noprofile -command "$(Invoke-Webrequest -Method put -Infile $Env:FULL https://transfer.sh/$Env:FN).Content"
+```
+
+## Uploading and Downloading
+<a name="uploading-and-downloading"/>
+
+### Uploading with wget
+```bash
+$ wget --method PUT --body-file=/tmp/file.tar https://transfer.sh/file.tar -O - -nv 
+```
+
+### Uploading with PowerShell
+```posh
+PS H:\> invoke-webrequest -method put -infile .\file.txt https://transfer.sh/file.txt 
+```
+
+### Upload using HTTPie
+```bash
+$ http https://transfer.sh/ -vv < /tmp/test.log 
+```
+
+### Uploading a filtered text file
+```bash
+$ grep 'pound' /var/log/syslog | curl --upload-file - https://transfer.sh/pound.log 
+```
+
+### Downloading with curl
+```bash
+$ curl https://transfer.sh/1lDau/test.txt -o test.txt
+```
+
+### Downloading with wget
+```bash
+$ wget https://transfer.sh/1lDau/test.txt
+```
+
+## Archiving and backups
+<a name="archiving-and-backups"/>
+
+### Backup, encrypt and transfer a MySQL dump
+```bash
+$ mysqldump --all-databases | gzip | gpg -ac -o- | curl -X PUT --upload-file "-" https://transfer.sh/test.txt
+```
+
+### Archive and upload directory
+```bash
+$ tar -czf - /var/log/journal | curl --upload-file - https://transfer.sh/journal.tar.gz
+```
+
+### Uploading multiple files at once
+```bash
+$ curl -i -F filedata=@/tmp/hello.txt -F filedata=@/tmp/hello2.txt https://transfer.sh/
+```
+
+### Combining downloads as zip or tar.gz archive
+```bash
+$ curl https://transfer.sh/(15HKz/hello.txt,15HKz/hello.txt).tar.gz
+$ curl https://transfer.sh/(15HKz/hello.txt,15HKz/hello.txt).zip 
+```
+
+### Transfer and send email with link (using an alias)
+```bash
+$ transfer /tmp/hello.txt | mail -s "Hello World" user@yourmaildomain.com 
+```
+## Encrypting and decrypting
+<a name="encrypting-and-decrypting"/>
+
+### Encrypting files with password using gpg
+```bash
+$ cat /tmp/hello.txt | gpg -ac -o- | curl -X PUT --upload-file "-" https://transfer.sh/test.txt
+```
+
+### Downloading and decrypting
+```bash
+$ curl https://transfer.sh/1lDau/test.txt | gpg -o- > /tmp/hello.txt 
+```
+
+### Import keys from [keybase](https://keybase.io/)
+```bash
+$ keybase track [them] # Encrypt for recipient(s)
+$ cat somebackupfile.tar.gz | keybase encrypt [them] | curl --upload-file '-' https://transfer.sh/test.txt # Decrypt
+$ curl https://transfer.sh/sqUFi/test.md | keybase decrypt
+```
+
+## Scanning for viruses
+<a name="scanning-for-viruses"/>
+
+### Scan for malware or viruses using Clamav
+```bash
+$ wget http://www.eicar.org/download/eicar.com
+$ curl -X PUT --upload-file ./eicar.com https://transfer.sh/eicar.com/scan
+```
+
+### Upload malware to VirusTotal, get a permalink in return
+```bash
+$ curl -X PUT --upload-file nhgbhhj https://transfer.sh/test.txt/virustotal 
+```

fuzzit.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+set -xe
+
+# Validate arguments
+if [ "$#" -ne 1 ]; then
+    echo "Usage: $0 <fuzz-type>"
+    exit 1
+fi
+
+# Configure
+NAME=transfersh
+ROOT=./server
+TYPE=$1
+
+# Setup
+export GO111MODULE="off"
+go get -u github.com/dvyukov/go-fuzz/go-fuzz github.com/dvyukov/go-fuzz/go-fuzz-build
+go get -d -v -u ./...
+if [ ! -f fuzzit ]; then
+    wget -q -O fuzzit https://github.com/fuzzitdev/fuzzit/releases/download/v2.4.29/fuzzit_Linux_x86_64
+    chmod a+x fuzzit
+fi
+
+# Fuzz
+function fuzz {
+    FUNC=Fuzz$1
+    TARGET=$2
+    DIR=${3:-$ROOT}
+    go-fuzz-build -libfuzzer -func $FUNC -o fuzzer.a $DIR
+    clang -fsanitize=fuzzer fuzzer.a -o fuzzer
+    ./fuzzit create job --type $TYPE $NAME/$TARGET fuzzer
+}
+fuzz LocalStorage local-storage

server/server_fuzz.go

@@ -0,0 +1,90 @@
+// +build gofuzz
+
+package server
+
+import (
+	"bytes"
+	"io"
+	"math/rand"
+	"reflect"
+)
+
+const applicationOctetStream = "application/octet-stream"
+
+// FuzzLocalStorage tests the Local Storage.
+func FuzzLocalStorage(fuzz []byte) int {
+	var fuzzLength = uint64(len(fuzz))
+	if fuzzLength == 0 {
+		return -1
+	}
+
+	storage, err := NewLocalStorage("/tmp", nil)
+	if err != nil {
+		panic("unable to create local storage")
+	}
+
+	token := Encode(10000000 + int64(rand.Intn(1000000000)))
+	filename := Encode(10000000 + int64(rand.Intn(1000000000))) + ".bin"
+
+	input := bytes.NewReader(fuzz)
+	err = storage.Put(token, filename, input, applicationOctetStream, fuzzLength)
+	if err != nil {
+		panic("unable to save file")
+	}
+
+	contentType, contentLength, err := storage.Head(token, filename)
+	if err != nil {
+		panic("not visible through head")
+	}
+
+	if contentType != applicationOctetStream {
+		panic("incorrect content type")
+	}
+
+	if contentLength != fuzzLength {
+		panic("incorrect content length")
+	}
+
+	output, contentType, contentLength, err := storage.Get(token, filename)
+	if err != nil {
+		panic("not visible through get")
+	}
+
+	if contentType != applicationOctetStream {
+		panic("incorrect content type")
+	}
+
+	if contentLength != fuzzLength {
+		panic("incorrect content length")
+	}
+
+	var length uint64
+	b := make([]byte, len(fuzz))
+	for {
+		n, err := output.Read(b)
+		length += uint64(n)
+		if err == io.EOF {
+			break
+		}
+	}
+
+	if !reflect.DeepEqual(b, fuzz) {
+		panic("incorrect content body")
+	}
+
+	if length != fuzzLength {
+		panic("incorrect content length")
+	}
+
+	err = storage.Delete(token, filename)
+	if err != nil {
+		panic("unable to delete file")
+	}
+
+	_, _, err = storage.Head(token, filename)
+	if !storage.IsNotExist(err) {
+		panic("file not deleted")
+	}
+
+	return 1
+}

server/storage.go

@@ -106,7 +106,7 @@ func (s *LocalStorage) Put(token string, filename string, reader io.Reader, cont
 
 	path := filepath.Join(s.basedir, token)
 
-	if err = os.Mkdir(path, 0700); err != nil && !os.IsExist(err) {
+	if err = os.MkdirAll(path, 0700); err != nil && !os.IsExist(err) {
 		return err
 	}
 
@@ -243,9 +243,7 @@ func (s *S3Storage) Put(token string, filename string, reader io.Reader, content
 
 	// Create an uploader with the session and custom options
 	uploader := s3manager.NewUploader(s.session, func(u *s3manager.Uploader) {
-		u.PartSize = (1 << 20) * 5  // The minimum/default allowed part size is 5MB
 		u.Concurrency = concurrency // default is 5
-		u.MaxUploadParts = concurrency
 		u.LeavePartsOnError = false
 	})