gaspersic/transfer.sh
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Clamav prescan (#389)

Browse Source
This commit is contained in:
Andrea Spacca 2022-01-09 22:14:10 +01:00 committed by GitHub
parent 492731e31f
commit cff0a88bf3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 184 additions and 129 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
Makefile Normal file
View File

@ -0,0 +1,5 @@
.PHONY: lint
lint:
golangci-lint run --out-format=github-actions --config .golangci.yml

1
README.md
View File

@ -111,6 +111,7 @@ lets-encrypt-hosts | hosts to use for lets encrypt certificates (comma seperated
log | path to log file| | LOG |
cors-domains | comma separated list of domains for CORS, setting it enable CORS | | CORS_DOMAINS |
clamav-host | host for clamav feature | | CLAMAV_HOST |
perform-clamav-prescan | prescan every upload through clamav feature (clamav-host must be a local clamd unix socket) | | PERFORM_CLAMAV_PRESCAN |
rate-limit | request per minute | | RATE_LIMIT |
max-upload-size | max upload size in kilobytes | | MAX_UPLOAD_SIZE |
purge-days | number of days after the uploads are purged automatically | | PURGE_DAYS |

13
cmd/cmd.go
View File

@ -240,6 +240,11 @@ var globalFlags = []cli.Flag{
Value: "",
EnvVar: "CLAMAV_HOST",
},
cli.BoolFlag{
Name: "perform-clamav-prescan",
Usage: "perform-clamav-prescan",
EnvVar: "PERFORM_CLAMAV_PRESCAN",
},
cli.StringFlag{
Name: "virustotal-key",
Usage: "virustotal-key",
@ -388,6 +393,14 @@ func New() *Cmd {
options = append(options, server.ClamavHost(v))
}
if v := c.Bool("perform-clamav-prescan"); !v {
if c.String("clamav-host") == "" {
panic("clamav-host not set")
}
options = append(options, server.PerformClamavPrescan(v))
}
if v := c.Int64("max-upload-size"); v > 0 {
options = append(options, server.MaxUploadSize(v))
}

61
server/clamav.go
View File

@ -27,18 +27,19 @@ THE SOFTWARE.
package server
import (
// _ "transfer.sh/app/handlers"
// _ "transfer.sh/app/utils"
"errors"
"fmt"
"io"
"io/ioutil"
"net/http"
"time"
clamd "github.com/dutchcoders/go-clamd"
"github.com/gorilla/mux"
)
const clamavScanStatusOK = "OK"
func (s *Server) scanHandler(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
@ -49,23 +50,53 @@ func (s *Server) scanHandler(w http.ResponseWriter, r *http.Request) {
s.logger.Printf("Scanning %s %d %s", filename, contentLength, contentType)
reader := r.Body
c := clamd.NewClamd(s.ClamAVDaemonHost)
abort := make(chan bool)
defer close(abort)
response, err := c.ScanStream(reader, abort)
file, err := ioutil.TempFile(s.tempPath, "clamav-")
defer s.cleanTmpFile(file)
if err != nil {
s.logger.Printf("%s", err.Error())
http.Error(w, err.Error(), 500)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
_, err = io.Copy(file, r.Body)
if err != nil {
s.logger.Printf("%s", err.Error())
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
status, err := s.performScan(file.Name())
if err != nil {
s.logger.Printf("%s", err.Error())
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
_, _ = w.Write([]byte(fmt.Sprintf("%v\n", status)))
}
func (s *Server) performScan(path string) (string, error) {
c := clamd.NewClamd(s.ClamAVDaemonHost)
responseCh := make(chan chan *clamd.ScanResult)
errCh := make(chan error)
go func(responseCh chan chan *clamd.ScanResult, errCh chan error) {
response, err := c.ScanFile(path)
if err != nil {
errCh <- err
return
}
responseCh <- response
}(responseCh, errCh)
select {
case s := <-response:
_, _ = w.Write([]byte(fmt.Sprintf("%v\n", s.Status)))
case err := <-errCh:
return "", err
case response := <-responseCh:
st := <-response
return st.Status, nil
case <-time.After(time.Second * 60):
abort <- true
return "", errors.New("clamav scan timeout")
}
}

201
server/handlers.go
View File

@ -291,7 +291,7 @@ func sanitize(fileName string) string {
func (s *Server) postHandler(w http.ResponseWriter, r *http.Request) {
if err := r.ParseMultipartForm(_24K); nil != err {
s.logger.Printf("%s", err.Error())
http.Error(w, "Error occurred copying to output stream", 500)
http.Error(w, "Error occurred copying to output stream", http.StatusInternalServerError)
return
}
@ -309,74 +309,75 @@ func (s *Server) postHandler(w http.ResponseWriter, r *http.Request) {
if f, err = fheader.Open(); err != nil {
s.logger.Printf("%s", err.Error())
http.Error(w, err.Error(), 500)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
var b bytes.Buffer
file, err := ioutil.TempFile(s.tempPath, "transfer-")
defer s.cleanTmpFile(file)
n, err := io.CopyN(&b, f, _24K+1)
if err != nil && err != io.EOF {
if err != nil {
s.logger.Printf("%s", err.Error())
http.Error(w, err.Error(), 500)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
var file *os.File
var reader io.Reader
if n > _24K {
file, err = ioutil.TempFile(s.tempPath, "transfer-")
defer s.cleanTmpFile(file)
if err != nil {
s.logger.Fatal(err)
}
n, err = io.Copy(file, io.MultiReader(&b, f))
if err != nil {
s.logger.Printf("%s", err.Error())
http.Error(w, err.Error(), 500)
return
}
reader, err = os.Open(file.Name())
if err != nil {
s.logger.Printf("%s", err.Error())
http.Error(w, err.Error(), 500)
return
}
} else {
reader = bytes.NewReader(b.Bytes())
n, err := io.Copy(file, f)
if err != nil {
s.logger.Printf("%s", err.Error())
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
contentLength := n
_, err = file.Seek(0, io.SeekStart)
if err != nil {
s.logger.Printf("%s", err.Error())
return
}
if s.maxUploadSize > 0 && contentLength > s.maxUploadSize {
s.logger.Print("Entity too large")
http.Error(w, http.StatusText(http.StatusRequestEntityTooLarge), http.StatusRequestEntityTooLarge)
return
}
if s.performClamavPrescan {
status, err := s.performScan(file.Name())
if err != nil {
s.logger.Printf("%s", err.Error())
http.Error(w, "Could not perform prescan", http.StatusInternalServerError)
return
}
if status != clamavScanStatusOK {
s.logger.Printf("prescan positive: %s", status)
http.Error(w, "Clamav prescan found a virus", http.StatusPreconditionFailed)
return
}
}
metadata := metadataForRequest(contentType, s.randomTokenLength, r)
buffer := &bytes.Buffer{}
if err := json.NewEncoder(buffer).Encode(metadata); err != nil {
s.logger.Printf("%s", err.Error())
http.Error(w, "Could not encode metadata", 500)
http.Error(w, "Could not encode metadata", http.StatusInternalServerError)
return
} else if err := s.storage.Put(r.Context(), token, fmt.Sprintf("%s.metadata", filename), buffer, "text/json", uint64(buffer.Len())); err != nil {
s.logger.Printf("%s", err.Error())
http.Error(w, "Could not save metadata", 500)
http.Error(w, "Could not save metadata", http.StatusInternalServerError)
return
}
s.logger.Printf("Uploading %s %s %d %s", token, filename, contentLength, contentType)
if err = s.storage.Put(r.Context(), token, filename, reader, contentType, uint64(contentLength)); err != nil {
if err = s.storage.Put(r.Context(), token, filename, file, contentType, uint64(contentLength)); err != nil {
s.logger.Printf("Backend storage error: %s", err.Error())
http.Error(w, err.Error(), 500)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
@ -448,56 +449,35 @@ func (s *Server) putHandler(w http.ResponseWriter, r *http.Request) {
contentLength := r.ContentLength
var reader io.Reader
reader = r.Body
defer CloseCheck(r.Body.Close)
if contentLength == -1 {
// queue file to disk, because s3 needs content length
var err error
file, err := ioutil.TempFile(s.tempPath, "transfer-")
defer s.cleanTmpFile(file)
if err != nil {
s.logger.Printf("%s", err.Error())
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
f := reader
// queue file to disk, because s3 needs content length
// and clamav prescan scans a file
n, err := io.Copy(file, r.Body)
if err != nil {
s.logger.Printf("%s", err.Error())
http.Error(w, err.Error(), http.StatusInternalServerError)
var b bytes.Buffer
return
}
n, err := io.CopyN(&b, f, _24K+1)
if err != nil && err != io.EOF {
s.logger.Printf("Error putting new file: %s", err.Error())
http.Error(w, err.Error(), 500)
return
}
_, err = file.Seek(0, io.SeekStart)
if err != nil {
s.logger.Printf("%s", err.Error())
http.Error(w, "Cannot reset cache file", http.StatusInternalServerError)
var file *os.File
if n > _24K {
file, err = ioutil.TempFile(s.tempPath, "transfer-")
defer s.cleanTmpFile(file)
if err != nil {
s.logger.Printf("%s", err.Error())
http.Error(w, err.Error(), 500)
return
}
n, err = io.Copy(file, io.MultiReader(&b, f))
if err != nil {
s.logger.Printf("%s", err.Error())
http.Error(w, err.Error(), 500)
return
}
reader, err = os.Open(file.Name())
if err != nil {
s.logger.Printf("%s", err.Error())
http.Error(w, err.Error(), 500)
return
}
} else {
reader = bytes.NewReader(b.Bytes())
}
return
}
if contentLength < 1 {
contentLength = n
}
@ -509,10 +489,25 @@ func (s *Server) putHandler(w http.ResponseWriter, r *http.Request) {
if contentLength == 0 {
s.logger.Print("Empty content-length")
http.Error(w, "Could not upload empty file", 400)
http.Error(w, "Could not upload empty file", http.StatusBadRequest)
return
}
if s.performClamavPrescan {
status, err := s.performScan(file.Name())
if err != nil {
s.logger.Printf("%s", err.Error())
http.Error(w, "Could not perform prescan", http.StatusInternalServerError)
return
}
if status != clamavScanStatusOK {
s.logger.Printf("prescan positive: %s", status)
http.Error(w, "Clamav prescan found a virus", http.StatusPreconditionFailed)
return
}
}
contentType := mime.TypeByExtension(filepath.Ext(vars["filename"]))
token := token(s.randomTokenLength)
@ -522,25 +517,23 @@ func (s *Server) putHandler(w http.ResponseWriter, r *http.Request) {
buffer := &bytes.Buffer{}
if err := json.NewEncoder(buffer).Encode(metadata); err != nil {
s.logger.Printf("%s", err.Error())
http.Error(w, "Could not encode metadata", 500)
http.Error(w, "Could not encode metadata", http.StatusInternalServerError)
return
} else if !metadata.MaxDate.IsZero() && time.Now().After(metadata.MaxDate) {
s.logger.Print("Invalid MaxDate")
http.Error(w, "Invalid MaxDate, make sure Max-Days is smaller than 290 years", 400)
http.Error(w, "Invalid MaxDate, make sure Max-Days is smaller than 290 years", http.StatusBadRequest)
return
} else if err := s.storage.Put(r.Context(), token, fmt.Sprintf("%s.metadata", filename), buffer, "text/json", uint64(buffer.Len())); err != nil {
s.logger.Printf("%s", err.Error())
http.Error(w, "Could not save metadata", 500)
http.Error(w, "Could not save metadata", http.StatusInternalServerError)
return
}
s.logger.Printf("Uploading %s %s %d %s", token, filename, contentLength, contentType)
var err error
if err = s.storage.Put(r.Context(), token, filename, reader, contentType, uint64(contentLength)); err != nil {
if err = s.storage.Put(r.Context(), token, filename, file, contentType, uint64(contentLength)); err != nil {
s.logger.Printf("Error putting new file: %s", err.Error())
http.Error(w, "Could not save file", 500)
http.Error(w, "Could not save file", http.StatusInternalServerError)
return
}
@ -766,7 +759,7 @@ func (s *Server) deleteHandler(w http.ResponseWriter, r *http.Request) {
return
} else if err != nil {
s.logger.Printf("%s", err.Error())
http.Error(w, "Could not delete file.", 500)
http.Error(w, "Could not delete file.", http.StatusInternalServerError)
return
}
}
@ -805,7 +798,7 @@ func (s *Server) zipHandler(w http.ResponseWriter, r *http.Request) {
}
s.logger.Printf("%s", err.Error())
http.Error(w, "Could not retrieve file.", 500)
http.Error(w, "Could not retrieve file.", http.StatusInternalServerError)
return
}
@ -820,20 +813,20 @@ func (s *Server) zipHandler(w http.ResponseWriter, r *http.Request) {
if err != nil {
s.logger.Printf("%s", err.Error())
http.Error(w, "Internal server error.", 500)
http.Error(w, "Internal server error.", http.StatusInternalServerError)
return
}
if _, err = io.Copy(fw, reader); err != nil {
s.logger.Printf("%s", err.Error())
http.Error(w, "Internal server error.", 500)
http.Error(w, "Internal server error.", http.StatusInternalServerError)
return
}
}
if err := zw.Close(); err != nil {
s.logger.Printf("%s", err.Error())
http.Error(w, "Internal server error.", 500)
http.Error(w, "Internal server error.", http.StatusInternalServerError)
return
}
}
@ -876,7 +869,7 @@ func (s *Server) tarGzHandler(w http.ResponseWriter, r *http.Request) {
}
s.logger.Printf("%s", err.Error())
http.Error(w, "Could not retrieve file.", 500)
http.Error(w, "Could not retrieve file.", http.StatusInternalServerError)
return
}
@ -888,13 +881,13 @@ func (s *Server) tarGzHandler(w http.ResponseWriter, r *http.Request) {
err = zw.WriteHeader(header)
if err != nil {
s.logger.Printf("%s", err.Error())
http.Error(w, "Internal server error.", 500)
http.Error(w, "Internal server error.", http.StatusInternalServerError)
return
}
if _, err = io.Copy(zw, reader); err != nil {
s.logger.Printf("%s", err.Error())
http.Error(w, "Internal server error.", 500)
http.Error(w, "Internal server error.", http.StatusInternalServerError)
return
}
}
@ -935,7 +928,7 @@ func (s *Server) tarHandler(w http.ResponseWriter, r *http.Request) {
}
s.logger.Printf("%s", err.Error())
http.Error(w, "Could not retrieve file.", 500)
http.Error(w, "Could not retrieve file.", http.StatusInternalServerError)
return
}
@ -947,13 +940,13 @@ func (s *Server) tarHandler(w http.ResponseWriter, r *http.Request) {
err = zw.WriteHeader(header)
if err != nil {
s.logger.Printf("%s", err.Error())
http.Error(w, "Internal server error.", 500)
http.Error(w, "Internal server error.", http.StatusInternalServerError)
return
}
if _, err = io.Copy(zw, reader); err != nil {
s.logger.Printf("%s", err.Error())
http.Error(w, "Internal server error.", 500)
http.Error(w, "Internal server error.", http.StatusInternalServerError)
return
}
}
@ -980,7 +973,7 @@ func (s *Server) headHandler(w http.ResponseWriter, r *http.Request) {
return
} else if err != nil {
s.logger.Printf("%s", err.Error())
http.Error(w, "Could not retrieve file.", 500)
http.Error(w, "Could not retrieve file.", http.StatusInternalServerError)
return
}
@ -1017,7 +1010,7 @@ func (s *Server) getHandler(w http.ResponseWriter, r *http.Request) {
return
} else if err != nil {
s.logger.Printf("%s", err.Error())
http.Error(w, "Could not retrieve file.", 500)
http.Error(w, "Could not retrieve file.", http.StatusInternalServerError)
return
}
@ -1048,14 +1041,14 @@ func (s *Server) getHandler(w http.ResponseWriter, r *http.Request) {
if err != nil {
s.logger.Printf("%s", err.Error())
http.Error(w, "Error occurred copying to output stream", 500)
http.Error(w, "Error occurred copying to output stream", http.StatusInternalServerError)
return
}
_, err = io.Copy(file, reader)
if err != nil {
s.logger.Printf("%s", err.Error())
http.Error(w, "Error occurred copying to output stream", 500)
http.Error(w, "Error occurred copying to output stream", http.StatusInternalServerError)
return
}
@ -1065,7 +1058,7 @@ func (s *Server) getHandler(w http.ResponseWriter, r *http.Request) {
if _, err = io.Copy(w, reader); err != nil {
s.logger.Printf("%s", err.Error())
http.Error(w, "Error occurred copying to output stream", 500)
http.Error(w, "Error occurred copying to output stream", http.StatusInternalServerError)
return
}
}
@ -1136,12 +1129,12 @@ func (s *Server) basicAuthHandler(h http.Handler) http.HandlerFunc {
username, password, authOK := r.BasicAuth()
if !authOK {
http.Error(w, "Not authorized", 401)
http.Error(w, "Not authorized", http.StatusUnauthorized)
return
}
if username != s.AuthUser || password != s.AuthPass {
http.Error(w, "Not authorized", 401)
http.Error(w, "Not authorized", http.StatusUnauthorized)
return
}

28
server/server.go
View File

@ -76,6 +76,13 @@ func ClamavHost(s string) OptionFn {
}
}
// PerformClamavPrescan enables clamav prescan on upload
func PerformClamavPrescan(b bool) OptionFn {
return func(srvr *Server) {
srvr.performClamavPrescan = b
}
}
// VirustotalKey sets virus total key
func VirustotalKey(s string) OptionFn {
return func(srvr *Server) {
@ -338,8 +345,9 @@ type Server struct {
ipFilterOptions *IPFilterOptions
VirusTotalKey string
ClamAVDaemonHost string
VirusTotalKey string
ClamAVDaemonHost string
performClamavPrescan bool
tempPath string
@ -424,13 +432,17 @@ func (s *Server) Run() {
s.logger.Panicf("Unable to parse: path=%s, err=%s", path, err)
}
_, err = htmlTemplates.New(stripPrefix(path)).Parse(string(bytes))
if err != nil {
s.logger.Println("Unable to parse html template", err)
if strings.HasSuffix(path, ".html") {
_, err = htmlTemplates.New(stripPrefix(path)).Parse(string(bytes))
if err != nil {
s.logger.Println("Unable to parse html template", err)
}
}
_, err = textTemplates.New(stripPrefix(path)).Parse(string(bytes))
if err != nil {
s.logger.Println("Unable to parse text template", err)
if strings.HasSuffix(path, ".txt") {
_, err = textTemplates.New(stripPrefix(path)).Parse(string(bytes))
if err != nil {
s.logger.Println("Unable to parse text template", err)
}
}
}
}

4
server/virustotal.go
View File

@ -45,14 +45,14 @@ func (s *Server) virusTotalHandler(w http.ResponseWriter, r *http.Request) {
vt, err := virustotal.NewVirusTotal(s.VirusTotalKey)
if err != nil {
http.Error(w, err.Error(), 500)
http.Error(w, err.Error(), http.StatusInternalServerError)
}
reader := r.Body
result, err := vt.Scan(filename, reader)
if err != nil {
http.Error(w, err.Error(), 500)
http.Error(w, err.Error(), http.StatusInternalServerError)
}
s.logger.Println(result)