Clamav prescan (#389)

Makefile

@@ -0,0 +1,5 @@
+.PHONY: lint
+
+lint:
+	golangci-lint run --out-format=github-actions --config .golangci.yml 
+

README.md

@@ -111,6 +111,7 @@ lets-encrypt-hosts | hosts to use for lets encrypt certificates (comma seperated
 log | path to log file| | LOG |
 cors-domains | comma separated list of domains for CORS, setting it enable CORS | | CORS_DOMAINS |
 clamav-host | host for clamav feature  | | CLAMAV_HOST |
+perform-clamav-prescan | prescan every upload through clamav feature (clamav-host must be a local clamd unix socket) | | PERFORM_CLAMAV_PRESCAN |
 rate-limit | request per minute  | | RATE_LIMIT |
 max-upload-size | max upload size in kilobytes  | | MAX_UPLOAD_SIZE |
 purge-days | number of days after the uploads are purged automatically | | PURGE_DAYS |   

cmd/cmd.go

@@ -240,6 +240,11 @@ var globalFlags = []cli.Flag{
 		Value:  "",
 		EnvVar: "CLAMAV_HOST",
 	},
+	cli.BoolFlag{
+		Name:   "perform-clamav-prescan",
+		Usage:  "perform-clamav-prescan",
+		EnvVar: "PERFORM_CLAMAV_PRESCAN",
+	},
 	cli.StringFlag{
 		Name:   "virustotal-key",
 		Usage:  "virustotal-key",
@@ -388,6 +393,14 @@ func New() *Cmd {
 			options = append(options, server.ClamavHost(v))
 		}
 
+		if v := c.Bool("perform-clamav-prescan"); !v {
+			if c.String("clamav-host") == "" {
+				panic("clamav-host not set")
+			}
+
+			options = append(options, server.PerformClamavPrescan(v))
+		}
+
 		if v := c.Int64("max-upload-size"); v > 0 {
 			options = append(options, server.MaxUploadSize(v))
 		}

server/clamav.go

@@ -27,18 +27,19 @@ THE SOFTWARE.
 package server
 
 import (
-	// _ "transfer.sh/app/handlers"
+	"errors"
-	// _ "transfer.sh/app/utils"
-
 	"fmt"
+	"io"
+	"io/ioutil"
 	"net/http"
 	"time"
 
 	clamd "github.com/dutchcoders/go-clamd"
-
 	"github.com/gorilla/mux"
 )
 
+const clamavScanStatusOK = "OK"
+
 func (s *Server) scanHandler(w http.ResponseWriter, r *http.Request) {
 	vars := mux.Vars(r)
 
@@ -49,23 +50,53 @@ func (s *Server) scanHandler(w http.ResponseWriter, r *http.Request) {
 
 	s.logger.Printf("Scanning %s %d %s", filename, contentLength, contentType)
 
-	reader := r.Body
+	file, err := ioutil.TempFile(s.tempPath, "clamav-")
-
+	defer s.cleanTmpFile(file)
-	c := clamd.NewClamd(s.ClamAVDaemonHost)
-
-	abort := make(chan bool)
-	defer close(abort)
-	response, err := c.ScanStream(reader, abort)
 	if err != nil {
 		s.logger.Printf("%s", err.Error())
-		http.Error(w, err.Error(), 500)
+		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
 	}
 
+	_, err = io.Copy(file, r.Body)
+	if err != nil {
+		s.logger.Printf("%s", err.Error())
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	status, err := s.performScan(file.Name())
+	if err != nil {
+		s.logger.Printf("%s", err.Error())
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	_, _ = w.Write([]byte(fmt.Sprintf("%v\n", status)))
+}
+
+func (s *Server) performScan(path string) (string, error) {
+	c := clamd.NewClamd(s.ClamAVDaemonHost)
+
+	responseCh := make(chan chan *clamd.ScanResult)
+	errCh := make(chan error)
+	go func(responseCh chan chan *clamd.ScanResult, errCh chan error) {
+		response, err := c.ScanFile(path)
+		if err != nil {
+			errCh <- err
+			return
+		}
+
+		responseCh <- response
+	}(responseCh, errCh)
+
 	select {
-	case s := <-response:
+	case err := <-errCh:
-		_, _ = w.Write([]byte(fmt.Sprintf("%v\n", s.Status)))
+		return "", err
+	case response := <-responseCh:
+		st := <-response
+		return st.Status, nil
 	case <-time.After(time.Second * 60):
-		abort <- true
+		return "", errors.New("clamav scan timeout")
 	}
 }

server/handlers.go

@@ -291,7 +291,7 @@ func sanitize(fileName string) string {
 func (s *Server) postHandler(w http.ResponseWriter, r *http.Request) {
 	if err := r.ParseMultipartForm(_24K); nil != err {
 		s.logger.Printf("%s", err.Error())
-		http.Error(w, "Error occurred copying to output stream", 500)
+		http.Error(w, "Error occurred copying to output stream", http.StatusInternalServerError)
 		return
 	}
 
@@ -309,74 +309,75 @@ func (s *Server) postHandler(w http.ResponseWriter, r *http.Request) {
 
 			if f, err = fheader.Open(); err != nil {
 				s.logger.Printf("%s", err.Error())
-				http.Error(w, err.Error(), 500)
+				http.Error(w, err.Error(), http.StatusInternalServerError)
 				return
 			}
 
-			var b bytes.Buffer
+			file, err := ioutil.TempFile(s.tempPath, "transfer-")
+			defer s.cleanTmpFile(file)
 
-			n, err := io.CopyN(&b, f, _24K+1)
+			if err != nil {
-			if err != nil && err != io.EOF {
 				s.logger.Printf("%s", err.Error())
-				http.Error(w, err.Error(), 500)
+				http.Error(w, err.Error(), http.StatusInternalServerError)
 				return
 			}
 
-			var file *os.File
+			n, err := io.Copy(file, f)
-			var reader io.Reader
+			if err != nil {
-
+				s.logger.Printf("%s", err.Error())
-			if n > _24K {
+				http.Error(w, err.Error(), http.StatusInternalServerError)
-				file, err = ioutil.TempFile(s.tempPath, "transfer-")
+				return
-				defer s.cleanTmpFile(file)
-				if err != nil {
-					s.logger.Fatal(err)
-				}
-
-				n, err = io.Copy(file, io.MultiReader(&b, f))
-				if err != nil {
-					s.logger.Printf("%s", err.Error())
-					http.Error(w, err.Error(), 500)
-					return
-				}
-
-				reader, err = os.Open(file.Name())
-				if err != nil {
-					s.logger.Printf("%s", err.Error())
-					http.Error(w, err.Error(), 500)
-					return
-				}
-			} else {
-				reader = bytes.NewReader(b.Bytes())
 			}
 
 			contentLength := n
 
+			_, err = file.Seek(0, io.SeekStart)
+			if err != nil {
+				s.logger.Printf("%s", err.Error())
+				return
+			}
+
 			if s.maxUploadSize > 0 && contentLength > s.maxUploadSize {
 				s.logger.Print("Entity too large")
 				http.Error(w, http.StatusText(http.StatusRequestEntityTooLarge), http.StatusRequestEntityTooLarge)
 				return
 			}
 
+			if s.performClamavPrescan {
+				status, err := s.performScan(file.Name())
+				if err != nil {
+					s.logger.Printf("%s", err.Error())
+					http.Error(w, "Could not perform prescan", http.StatusInternalServerError)
+					return
+				}
+
+				if status != clamavScanStatusOK {
+					s.logger.Printf("prescan positive: %s", status)
+					http.Error(w, "Clamav prescan found a virus", http.StatusPreconditionFailed)
+					return
+				}
+			}
+
 			metadata := metadataForRequest(contentType, s.randomTokenLength, r)
 
 			buffer := &bytes.Buffer{}
 			if err := json.NewEncoder(buffer).Encode(metadata); err != nil {
 				s.logger.Printf("%s", err.Error())
-				http.Error(w, "Could not encode metadata", 500)
+				http.Error(w, "Could not encode metadata", http.StatusInternalServerError)
 
 				return
 			} else if err := s.storage.Put(r.Context(), token, fmt.Sprintf("%s.metadata", filename), buffer, "text/json", uint64(buffer.Len())); err != nil {
 				s.logger.Printf("%s", err.Error())
-				http.Error(w, "Could not save metadata", 500)
+				http.Error(w, "Could not save metadata", http.StatusInternalServerError)
 
 				return
 			}
 
 			s.logger.Printf("Uploading %s %s %d %s", token, filename, contentLength, contentType)
 
-			if err = s.storage.Put(r.Context(), token, filename, reader, contentType, uint64(contentLength)); err != nil {
+			if err = s.storage.Put(r.Context(), token, filename, file, contentType, uint64(contentLength)); err != nil {
 				s.logger.Printf("Backend storage error: %s", err.Error())
-				http.Error(w, err.Error(), 500)
+				http.Error(w, err.Error(), http.StatusInternalServerError)
 				return
 
 			}
@@ -448,56 +449,35 @@ func (s *Server) putHandler(w http.ResponseWriter, r *http.Request) {
 
 	contentLength := r.ContentLength
 
-	var reader io.Reader
-
-	reader = r.Body
-
 	defer CloseCheck(r.Body.Close)
 
-	if contentLength == -1 {
+	file, err := ioutil.TempFile(s.tempPath, "transfer-")
-		// queue file to disk, because s3 needs content length
+	defer s.cleanTmpFile(file)
-		var err error
+	if err != nil {
+		s.logger.Printf("%s", err.Error())
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
 
-		f := reader
+	// queue file to disk, because s3 needs content length
+	// and clamav prescan scans a file
+	n, err := io.Copy(file, r.Body)
+	if err != nil {
+		s.logger.Printf("%s", err.Error())
+		http.Error(w, err.Error(), http.StatusInternalServerError)
 
-		var b bytes.Buffer
+		return
+	}
 
-		n, err := io.CopyN(&b, f, _24K+1)
+	_, err = file.Seek(0, io.SeekStart)
-		if err != nil && err != io.EOF {
+	if err != nil {
-			s.logger.Printf("Error putting new file: %s", err.Error())
+		s.logger.Printf("%s", err.Error())
-			http.Error(w, err.Error(), 500)
+		http.Error(w, "Cannot reset cache file", http.StatusInternalServerError)
-			return
-		}
 
-		var file *os.File
+		return
-
+	}
-		if n > _24K {
-			file, err = ioutil.TempFile(s.tempPath, "transfer-")
-			defer s.cleanTmpFile(file)
-
-			if err != nil {
-				s.logger.Printf("%s", err.Error())
-				http.Error(w, err.Error(), 500)
-				return
-			}
-
-			n, err = io.Copy(file, io.MultiReader(&b, f))
-			if err != nil {
-				s.logger.Printf("%s", err.Error())
-				http.Error(w, err.Error(), 500)
-				return
-			}
-
-			reader, err = os.Open(file.Name())
-			if err != nil {
-				s.logger.Printf("%s", err.Error())
-				http.Error(w, err.Error(), 500)
-				return
-			}
-		} else {
-			reader = bytes.NewReader(b.Bytes())
-		}
 
+	if contentLength < 1 {
 		contentLength = n
 	}
 
@@ -509,10 +489,25 @@ func (s *Server) putHandler(w http.ResponseWriter, r *http.Request) {
 
 	if contentLength == 0 {
 		s.logger.Print("Empty content-length")
-		http.Error(w, "Could not upload empty file", 400)
+		http.Error(w, "Could not upload empty file", http.StatusBadRequest)
 		return
 	}
 
+	if s.performClamavPrescan {
+		status, err := s.performScan(file.Name())
+		if err != nil {
+			s.logger.Printf("%s", err.Error())
+			http.Error(w, "Could not perform prescan", http.StatusInternalServerError)
+			return
+		}
+
+		if status != clamavScanStatusOK {
+			s.logger.Printf("prescan positive: %s", status)
+			http.Error(w, "Clamav prescan found a virus", http.StatusPreconditionFailed)
+			return
+		}
+	}
+
 	contentType := mime.TypeByExtension(filepath.Ext(vars["filename"]))
 
 	token := token(s.randomTokenLength)
@@ -522,25 +517,23 @@ func (s *Server) putHandler(w http.ResponseWriter, r *http.Request) {
 	buffer := &bytes.Buffer{}
 	if err := json.NewEncoder(buffer).Encode(metadata); err != nil {
 		s.logger.Printf("%s", err.Error())
-		http.Error(w, "Could not encode metadata", 500)
+		http.Error(w, "Could not encode metadata", http.StatusInternalServerError)
 		return
 	} else if !metadata.MaxDate.IsZero() && time.Now().After(metadata.MaxDate) {
 		s.logger.Print("Invalid MaxDate")
-		http.Error(w, "Invalid MaxDate, make sure Max-Days is smaller than 290 years", 400)
+		http.Error(w, "Invalid MaxDate, make sure Max-Days is smaller than 290 years", http.StatusBadRequest)
 		return
 	} else if err := s.storage.Put(r.Context(), token, fmt.Sprintf("%s.metadata", filename), buffer, "text/json", uint64(buffer.Len())); err != nil {
 		s.logger.Printf("%s", err.Error())
-		http.Error(w, "Could not save metadata", 500)
+		http.Error(w, "Could not save metadata", http.StatusInternalServerError)
 		return
 	}
 
 	s.logger.Printf("Uploading %s %s %d %s", token, filename, contentLength, contentType)
 
-	var err error
+	if err = s.storage.Put(r.Context(), token, filename, file, contentType, uint64(contentLength)); err != nil {
-
-	if err = s.storage.Put(r.Context(), token, filename, reader, contentType, uint64(contentLength)); err != nil {
 		s.logger.Printf("Error putting new file: %s", err.Error())
-		http.Error(w, "Could not save file", 500)
+		http.Error(w, "Could not save file", http.StatusInternalServerError)
 		return
 	}
 
@@ -766,7 +759,7 @@ func (s *Server) deleteHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	} else if err != nil {
 		s.logger.Printf("%s", err.Error())
-		http.Error(w, "Could not delete file.", 500)
+		http.Error(w, "Could not delete file.", http.StatusInternalServerError)
 		return
 	}
 }
@@ -805,7 +798,7 @@ func (s *Server) zipHandler(w http.ResponseWriter, r *http.Request) {
 			}
 
 			s.logger.Printf("%s", err.Error())
-			http.Error(w, "Could not retrieve file.", 500)
+			http.Error(w, "Could not retrieve file.", http.StatusInternalServerError)
 			return
 		}
 
@@ -820,20 +813,20 @@ func (s *Server) zipHandler(w http.ResponseWriter, r *http.Request) {
 
 		if err != nil {
 			s.logger.Printf("%s", err.Error())
-			http.Error(w, "Internal server error.", 500)
+			http.Error(w, "Internal server error.", http.StatusInternalServerError)
 			return
 		}
 
 		if _, err = io.Copy(fw, reader); err != nil {
 			s.logger.Printf("%s", err.Error())
-			http.Error(w, "Internal server error.", 500)
+			http.Error(w, "Internal server error.", http.StatusInternalServerError)
 			return
 		}
 	}
 
 	if err := zw.Close(); err != nil {
 		s.logger.Printf("%s", err.Error())
-		http.Error(w, "Internal server error.", 500)
+		http.Error(w, "Internal server error.", http.StatusInternalServerError)
 		return
 	}
 }
@@ -876,7 +869,7 @@ func (s *Server) tarGzHandler(w http.ResponseWriter, r *http.Request) {
 			}
 
 			s.logger.Printf("%s", err.Error())
-			http.Error(w, "Could not retrieve file.", 500)
+			http.Error(w, "Could not retrieve file.", http.StatusInternalServerError)
 			return
 		}
 
@@ -888,13 +881,13 @@ func (s *Server) tarGzHandler(w http.ResponseWriter, r *http.Request) {
 		err = zw.WriteHeader(header)
 		if err != nil {
 			s.logger.Printf("%s", err.Error())
-			http.Error(w, "Internal server error.", 500)
+			http.Error(w, "Internal server error.", http.StatusInternalServerError)
 			return
 		}
 
 		if _, err = io.Copy(zw, reader); err != nil {
 			s.logger.Printf("%s", err.Error())
-			http.Error(w, "Internal server error.", 500)
+			http.Error(w, "Internal server error.", http.StatusInternalServerError)
 			return
 		}
 	}
@@ -935,7 +928,7 @@ func (s *Server) tarHandler(w http.ResponseWriter, r *http.Request) {
 			}
 
 			s.logger.Printf("%s", err.Error())
-			http.Error(w, "Could not retrieve file.", 500)
+			http.Error(w, "Could not retrieve file.", http.StatusInternalServerError)
 			return
 		}
 
@@ -947,13 +940,13 @@ func (s *Server) tarHandler(w http.ResponseWriter, r *http.Request) {
 		err = zw.WriteHeader(header)
 		if err != nil {
 			s.logger.Printf("%s", err.Error())
-			http.Error(w, "Internal server error.", 500)
+			http.Error(w, "Internal server error.", http.StatusInternalServerError)
 			return
 		}
 
 		if _, err = io.Copy(zw, reader); err != nil {
 			s.logger.Printf("%s", err.Error())
-			http.Error(w, "Internal server error.", 500)
+			http.Error(w, "Internal server error.", http.StatusInternalServerError)
 			return
 		}
 	}
@@ -980,7 +973,7 @@ func (s *Server) headHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	} else if err != nil {
 		s.logger.Printf("%s", err.Error())
-		http.Error(w, "Could not retrieve file.", 500)
+		http.Error(w, "Could not retrieve file.", http.StatusInternalServerError)
 		return
 	}
 
@@ -1017,7 +1010,7 @@ func (s *Server) getHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	} else if err != nil {
 		s.logger.Printf("%s", err.Error())
-		http.Error(w, "Could not retrieve file.", 500)
+		http.Error(w, "Could not retrieve file.", http.StatusInternalServerError)
 		return
 	}
 
@@ -1048,14 +1041,14 @@ func (s *Server) getHandler(w http.ResponseWriter, r *http.Request) {
 
 		if err != nil {
 			s.logger.Printf("%s", err.Error())
-			http.Error(w, "Error occurred copying to output stream", 500)
+			http.Error(w, "Error occurred copying to output stream", http.StatusInternalServerError)
 			return
 		}
 
 		_, err = io.Copy(file, reader)
 		if err != nil {
 			s.logger.Printf("%s", err.Error())
-			http.Error(w, "Error occurred copying to output stream", 500)
+			http.Error(w, "Error occurred copying to output stream", http.StatusInternalServerError)
 			return
 		}
 
@@ -1065,7 +1058,7 @@ func (s *Server) getHandler(w http.ResponseWriter, r *http.Request) {
 
 	if _, err = io.Copy(w, reader); err != nil {
 		s.logger.Printf("%s", err.Error())
-		http.Error(w, "Error occurred copying to output stream", 500)
+		http.Error(w, "Error occurred copying to output stream", http.StatusInternalServerError)
 		return
 	}
 }
@@ -1136,12 +1129,12 @@ func (s *Server) basicAuthHandler(h http.Handler) http.HandlerFunc {
 
 		username, password, authOK := r.BasicAuth()
 		if !authOK {
-			http.Error(w, "Not authorized", 401)
+			http.Error(w, "Not authorized", http.StatusUnauthorized)
 			return
 		}
 
 		if username != s.AuthUser || password != s.AuthPass {
-			http.Error(w, "Not authorized", 401)
+			http.Error(w, "Not authorized", http.StatusUnauthorized)
 			return
 		}
 

server/server.go

@@ -76,6 +76,13 @@ func ClamavHost(s string) OptionFn {
 	}
 }
 
+// PerformClamavPrescan enables clamav prescan on upload
+func PerformClamavPrescan(b bool) OptionFn {
+	return func(srvr *Server) {
+		srvr.performClamavPrescan = b
+	}
+}
+
 // VirustotalKey sets virus total key
 func VirustotalKey(s string) OptionFn {
 	return func(srvr *Server) {
@@ -338,8 +345,9 @@ type Server struct {
 
 	ipFilterOptions *IPFilterOptions
 
-	VirusTotalKey    string
+	VirusTotalKey        string
-	ClamAVDaemonHost string
+	ClamAVDaemonHost     string
+	performClamavPrescan bool
 
 	tempPath string
 
@@ -424,13 +432,17 @@ func (s *Server) Run() {
 				s.logger.Panicf("Unable to parse: path=%s, err=%s", path, err)
 			}
 
-			_, err = htmlTemplates.New(stripPrefix(path)).Parse(string(bytes))
+			if strings.HasSuffix(path, ".html") {
-			if err != nil {
+				_, err = htmlTemplates.New(stripPrefix(path)).Parse(string(bytes))
-				s.logger.Println("Unable to parse html template", err)
+				if err != nil {
+					s.logger.Println("Unable to parse html template", err)
+				}
 			}
-			_, err = textTemplates.New(stripPrefix(path)).Parse(string(bytes))
+			if strings.HasSuffix(path, ".txt") {
-			if err != nil {
+				_, err = textTemplates.New(stripPrefix(path)).Parse(string(bytes))
-				s.logger.Println("Unable to parse text template", err)
+				if err != nil {
+					s.logger.Println("Unable to parse text template", err)
+				}
 			}
 		}
 	}

server/virustotal.go

@@ -45,14 +45,14 @@ func (s *Server) virusTotalHandler(w http.ResponseWriter, r *http.Request) {
 
 	vt, err := virustotal.NewVirusTotal(s.VirusTotalKey)
 	if err != nil {
-		http.Error(w, err.Error(), 500)
+		http.Error(w, err.Error(), http.StatusInternalServerError)
 	}
 
 	reader := r.Body
 
 	result, err := vt.Scan(filename, reader)
 	if err != nil {
-		http.Error(w, err.Error(), 500)
+		http.Error(w, err.Error(), http.StatusInternalServerError)
 	}
 
 	s.logger.Println(result)