PURGE FEATURE

server/storage.go

@@ -4,14 +4,6 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"io"
-	"io/ioutil"
-	"log"
-	"net/http"
-	"os"
-	"path/filepath"
-	"strings"
-
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/aws/session"
@@ -22,6 +14,14 @@ import (
 	"golang.org/x/oauth2/google"
 	"google.golang.org/api/drive/v3"
 	"google.golang.org/api/googleapi"
+	"io"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
 
 	"storj.io/common/storj"
 	"storj.io/uplink"
@@ -33,6 +33,7 @@ type Storage interface {
 	Put(token string, filename string, reader io.Reader, contentType string, contentLength uint64) error
 	Delete(token string, filename string) error
 	IsNotExist(err error) bool
+	Purge(days time.Duration) error
 
 	Type() string
 }
@@ -91,6 +92,27 @@ func (s *LocalStorage) Delete(token string, filename string) (err error) {
 	return
 }
 
+func (s *LocalStorage) Purge(days time.Duration) (err error) {
+	err = filepath.Walk(s.basedir,
+		func(path string, info os.FileInfo, err error) error {
+			if err != nil {
+				return err
+			}
+			if info.IsDir() {
+				return nil
+			}
+
+			if info.ModTime().After(time.Now().Add(-1 * days)) {
+				err = os.Remove(path)
+				return err
+			}
+
+			return nil
+		})
+
+	return
+}
+
 func (s *LocalStorage) IsNotExist(err error) bool {
 	if err == nil {
 		return false
@@ -128,13 +150,21 @@ type S3Storage struct {
 	session     *session.Session
 	s3          *s3.S3
 	logger      *log.Logger
+	purgeDays   time.Duration
 	noMultipart bool
 }
 
-func NewS3Storage(accessKey, secretKey, bucketName, region, endpoint string, logger *log.Logger, disableMultipart bool, forcePathStyle bool) (*S3Storage, error) {
+func NewS3Storage(accessKey, secretKey, bucketName string, purgeDays int, region, endpoint string, disableMultipart bool, forcePathStyle bool, logger *log.Logger) (*S3Storage, error) {
 	sess := getAwsSession(accessKey, secretKey, region, endpoint, forcePathStyle)
 
-	return &S3Storage{bucket: bucketName, s3: s3.New(sess), session: sess, logger: logger, noMultipart: disableMultipart}, nil
+	return &S3Storage{
+		bucket: bucketName,
+		s3: s3.New(sess),
+		session: sess,
+		logger: logger,
+		noMultipart: disableMultipart,
+		purgeDays: time.Duration(purgeDays * 24) * time.Hour,
+	}, nil
 }
 
 func (s *S3Storage) Type() string {
@@ -162,6 +192,11 @@ func (s *S3Storage) Head(token string, filename string) (contentLength uint64, e
 	return
 }
 
+func (s *S3Storage) Purge(days time.Duration) (err error) {
+	// NOOP expiration is set at upload time
+	return nil
+}
+
 func (s *S3Storage) IsNotExist(err error) bool {
 	if err == nil {
 		return false
@@ -242,6 +277,7 @@ func (s *S3Storage) Put(token string, filename string, reader io.Reader, content
 		Bucket: aws.String(s.bucket),
 		Key:    aws.String(key),
 		Body:   reader,
+		Expires: aws.Time(time.Now().Add(s.purgeDays)),
 	})
 
 	return
@@ -448,6 +484,34 @@ func (s *GDrive) Delete(token string, filename string) (err error) {
 	return
 }
 
+func (s *GDrive) Purge(days time.Duration) (err error) {
+	nextPageToken := ""
+
+	expirationDate := time.Now().Add(-1 * days).Format(time.RFC3339)
+	q := fmt.Sprintf("'%s' in parents and modifiedTime > '%s' and mimeType!='%s' and trashed=false", s.rootId, expirationDate, GDriveDirectoryMimeType)
+	l, err := s.list(nextPageToken, q)
+	if err != nil {
+		return err
+	}
+
+	for 0 < len(l.Files) {
+		for _, fi := range l.Files {
+			err = s.service.Files.Delete(fi.Id).Do()
+			if err != nil {
+				return
+			}
+		}
+
+		if l.NextPageToken == "" {
+			break
+		}
+
+		l, err = s.list(l.NextPageToken, q)
+	}
+
+	return
+}
+
 func (s *GDrive) IsNotExist(err error) bool {
 	if err != nil {
 		if e, ok := err.(*googleapi.Error); ok {
@@ -554,10 +618,11 @@ type StorjStorage struct {
 	Storage
 	project *uplink.Project
 	bucket  *uplink.Bucket
+	purgeDays time.Duration
 	logger  *log.Logger
 }
 
-func NewStorjStorage(access, bucket string, logger *log.Logger) (*StorjStorage, error) {
+func NewStorjStorage(access, bucket string, purgeDays int, logger *log.Logger) (*StorjStorage, error) {
 	var instance StorjStorage
 	var err error
 
@@ -580,6 +645,8 @@ func NewStorjStorage(access, bucket string, logger *log.Logger) (*StorjStorage,
 		return nil, err
 	}
 
+	instance.purgeDays = time.Duration(purgeDays * 24) * time.Hour
+
 	instance.logger = logger
 
 	return &instance, nil
@@ -634,6 +701,11 @@ func (s *StorjStorage) Delete(token string, filename string) (err error) {
 	return
 }
 
+func (s *StorjStorage) Purge(days time.Duration) (err error) {
+	// NOOP expiration is set at upload time
+	return nil
+}
+
 func (s *StorjStorage) Put(token string, filename string, reader io.Reader, contentType string, contentLength uint64) (err error) {
 	key := storj.JoinPaths(token, filename)
 
@@ -641,7 +713,7 @@ func (s *StorjStorage) Put(token string, filename string, reader io.Reader, cont
 
 	ctx := context.TODO()
 
-	writer, err := s.project.UploadObject(ctx, s.bucket.Name, key, nil)
+	writer, err := s.project.UploadObject(ctx, s.bucket.Name, key, &uplink.UploadOptions{Expires: time.Now().Add(s.purgeDays)})
 	if err != nil {
 		return err
 	}