feat: Add blobstore package

This commit introduces a `blobstore` package and refactors the existing
upload mechanism. Upload is now handled by `providers` and the two
bundled providers are `S3` and `Filesystem`. `app.Blobstore` initialises
the correct provider based on the configuration and handles `Put`,
`Delete` and `Get` operations.

.dockerignore

@@ -0,0 +1,25 @@
+**/.classpath
+**/.dockerignore
+**/.env
+**/.git
+**/.gitignore
+**/.project
+**/.settings
+**/.toolstarget
+**/.vs
+**/.vscode
+**/*.*proj.user
+**/*.dbmdl
+**/*.jfm
+**/azds.yaml
+**/bin
+**/charts
+**/docker-compose*
+**/Dockerfile*
+**/node_modules
+**/npm-debug.log
+**/obj
+**/secrets.dev.yaml
+**/values.dev.yaml
+LICENSE
+README.md

.gitignore

@@ -6,3 +6,4 @@ frontend/yarn.lock
 config.toml
 node_modules
 listmonk
+dist/*

admin.go

@@ -28,7 +28,6 @@ func handleGetConfigScript(c echo.Context) error {
 		app = c.Get("app").(*App)
 		out = configScript{
 			RootURL:    app.Constants.RootURL,
-			UploadURI:  app.Constants.UploadURI,
 			FromEmail:  app.Constants.FromEmail,
 			Messengers: app.Manager.GetMessengerNames(),
 		}

config.toml.sample

@@ -24,14 +24,6 @@ from_email = "listmonk <from@mail.com>"
 # To disable notifications, set an empty list, eg: notify_emails = []
 notify_emails = ["admin1@mysite.com", "admin2@mysite.com"]
 
-# Path to the uploads directory where media will be uploaded.
-upload_path = "uploads"
-
-# Upload URI that's visible to the outside world. The media
-# uploaded to upload_path will be made available publicly
-# under this URI, for instance, list.yoursite.com/uploads.
-upload_uri = "/uploads"
-
 # Maximum concurrent workers that will attempt to send messages
 # simultaneously. This should depend on the number of CPUs the
 # machine has and also the number of simultaenous e-mails the
@@ -110,3 +102,33 @@ ssl_mode = "disable"
 
         # Maximum concurrent connections to the SMTP server.
         max_conns = 10
+
+# Upload settings
+[upload]
+# Provider which will be used to host uploaded media. Bundled providers are "filesystem" and "s3".
+provider = "filesystem"
+
+# S3 Provider settings
+[upload.s3]
+# (Optional). AWS Access Key and Secret Key for the user to access the bucket. Leaving it empty would default to use
+# instance IAM role.
+aws_access_key_id = ""
+aws_secret_access_key = ""
+# AWS Region where S3 bucket is hosted.
+aws_default_region="ap-south-1"
+# Specify bucket name.
+bucket=""
+# Path where the files will be stored inside bucket. Empty value ("") means the root of bucket.
+bucket_path=""
+# Bucket type can be "private" or "public".
+bucket_type="public"
+# (Optional) Specify TTL (in seconds) for the generated presigned URL. Expiry value is used only if the bucket is private.
+expiry="86400"
+
+# Filesystem provider settings
+[upload.filesystem]
+# Path to the uploads directory where media will be uploaded. Leaving it empty ("") means current working directory.
+upload_path=""
+# Upload URI that's visible to the outside world. The media uploaded to upload_path will be made available publicly
+# under this URI, for instance, list.yoursite.com/uploads.
+upload_uri = "/uploads"

go.mod

@@ -2,6 +2,7 @@ module github.com/knadh/listmonk
 
 require (
 	github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf
+	github.com/aws/aws-sdk-go v1.25.12
 	github.com/disintegration/imaging v1.5.0
 	github.com/jinzhu/gorm v1.9.1
 	github.com/jmoiron/sqlx v1.2.0
@@ -15,6 +16,7 @@ require (
 	github.com/lib/pq v1.0.0
 	github.com/mattn/go-colorable v0.0.9 // indirect
 	github.com/mattn/go-isatty v0.0.4 // indirect
+	github.com/rhnvrm/simples3 v0.2.4-0.20191018074503-3d5b071ef727
 	github.com/satori/go.uuid v1.2.0
 	github.com/spf13/pflag v1.0.3
 	github.com/stretchr/objx v0.2.0 // indirect
@@ -27,3 +29,5 @@ require (
 	gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
 	gopkg.in/volatiletech/null.v6 v6.0.0-20170828023728-0bef4e07ae1b
 )
+
+go 1.13

go.sum

@@ -2,6 +2,8 @@ github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf h1:eg0MeVzsP1G42dRafH3vf+al2vQIJU0YHX+1Tw87oco=
 github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
+github.com/aws/aws-sdk-go v1.25.12 h1:a4h2FxoUJq9h+hajSE/dsRiqoOniIh6BkzhxMjkepzY=
+github.com/aws/aws-sdk-go v1.25.12/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -16,6 +18,8 @@ github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/jinzhu/gorm v1.9.1 h1:lDSDtsCt5AGGSKTs8AHlSDbbgif4G4+CKJ8ETBDVHTA=
 github.com/jinzhu/gorm v1.9.1/go.mod h1:Vla75njaFJ8clLU1W44h34PjIkijhjHIYnZxMqCdxqo=
+github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
+github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA=
 github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
 github.com/jordan-wright/email v0.0.0-20181027021455-480bedc4908b h1:veTPVnbkOijplSJVywDYKDRPoZEN39kfuMDzzRKP0FA=
@@ -57,6 +61,10 @@ github.com/pelletier/go-toml v1.4.0 h1:u3Z1r+oOXJIkxqw34zVhyPgjBsm6X2wn21NWs/HfS
 github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rhnvrm/simples3 v0.2.3 h1:qNXPynabu8M3F4+69fspA5aWZR8jqVV1RQtv2xc1OVk=
+github.com/rhnvrm/simples3 v0.2.3/go.mod h1:iphavgjkW1uvoIiqLUX6D42XuuI9Cr+B/63xw3gb9qA=
+github.com/rhnvrm/simples3 v0.2.4-0.20191018074503-3d5b071ef727 h1:2josYcx2gm3CT0WMqi0jBagvg50V3UMWlYN/CnBEbSI=
+github.com/rhnvrm/simples3 v0.2.4-0.20191018074503-3d5b071ef727/go.mod h1:iphavgjkW1uvoIiqLUX6D42XuuI9Cr+B/63xw3gb9qA=
 github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
 github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
@@ -73,6 +81,7 @@ golang.org/x/crypto v0.0.0-20181106171534-e4dc69e5b2fd h1:VtIkGDhk0ph3t+THbvXHfM
 golang.org/x/crypto v0.0.0-20181106171534-e4dc69e5b2fd/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/image v0.0.0-20181116024801-cd38e8056d9b h1:VHyIDlv3XkfCa5/a81uzaoDkHH4rr81Z62g+xlnO8uM=
 golang.org/x/image v0.0.0-20181116024801-cd38e8056d9b/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225 h1:kNX+jCowfMYzvlSvJu5pQWEmyWFrBXJ3PBy10xKMXK8=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/sys v0.0.0-20180906133057-8cf3aee42992 h1:BH3eQWeGbwRU2+wxxuuPOdFBmaiBH81O8BugSjHeTFg=
 golang.org/x/sys v0.0.0-20180906133057-8cf3aee42992/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=

main.go

@@ -19,6 +19,9 @@ import (
 	"github.com/knadh/koanf/providers/file"
 	"github.com/knadh/koanf/providers/posflag"
 	"github.com/knadh/listmonk/manager"
+	"github.com/knadh/listmonk/media"
+	"github.com/knadh/listmonk/media/providers/filesystem"
+	"github.com/knadh/listmonk/media/providers/s3"
 	"github.com/knadh/listmonk/messenger"
 	"github.com/knadh/listmonk/subimporter"
 	"github.com/knadh/stuffbin"
@@ -30,8 +33,6 @@ type constants struct {
 	RootURL      string         `koanf:"root"`
 	LogoURL      string         `koanf:"logo_url"`
 	FaviconURL   string         `koanf:"favicon_url"`
-	UploadPath   string         `koanf:"upload_path"`
-	UploadURI    string         `koanf:"upload_uri"`
 	FromEmail    string         `koanf:"from_email"`
 	NotifyEmails []string       `koanf:"notify_emails"`
 	Privacy      privacyOptions `koanf:"privacy"`
@@ -56,6 +57,7 @@ type App struct {
 	Logger    *log.Logger
 	NotifTpls *template.Template
 	Messenger messenger.Messenger
+	Media     media.Store
 }
 
 var (
@@ -195,6 +197,33 @@ func initMessengers(r *manager.Manager) messenger.Messenger {
 	return msgr
 }
 
+// initMediaStore initializes Upload manager with a custom backend.
+func initMediaStore() media.Store {
+	switch provider := ko.String("upload.provider"); provider {
+	case "s3":
+		var opts s3.Opts
+		ko.Unmarshal("upload.s3", &opts)
+		uplder, err := s3.NewS3Store(opts)
+		if err != nil {
+			logger.Fatalf("error initializing s3 upload provider %s", err)
+		}
+		return uplder
+	case "filesystem":
+		var opts filesystem.Opts
+		ko.Unmarshal("upload.filesystem", &opts)
+		opts.UploadPath = filepath.Clean(opts.UploadPath)
+		opts.UploadURI = filepath.Clean(opts.UploadURI)
+		uplder, err := filesystem.NewDiskStore(opts)
+		if err != nil {
+			logger.Fatalf("error initializing filesystem upload provider %s", err)
+		}
+		return uplder
+	default:
+		logger.Fatalf("unknown provider. please select one of either filesystem or s3")
+	}
+	return nil
+}
+
 func main() {
 	// Connect to the DB.
 	db, err := connectDB(ko.String("db.host"),
@@ -216,8 +245,6 @@ func main() {
 		log.Fatalf("error loading app config: %v", err)
 	}
 	c.RootURL = strings.TrimRight(c.RootURL, "/")
-	c.UploadURI = filepath.Clean(c.UploadURI)
-	c.UploadPath = filepath.Clean(c.UploadPath)
 	c.Privacy.Exportable = maps.StringSliceToLookupMap(ko.Strings("privacy.exportable"))
 
 	// Initialize the static file system into which all
@@ -299,6 +326,9 @@ func main() {
 	// Add messengers.
 	app.Messenger = initMessengers(app.Manager)
 
+	// Add uploader
+	app.Media = initMediaStore()
+
 	// Initialize the workers that push out messages.
 	go m.Run(time.Second * 5)
 	m.SpawnWorkers()
@@ -330,7 +360,9 @@ func main() {
 	fSrv := app.FS.FileServer()
 	srv.GET("/public/*", echo.WrapHandler(fSrv))
 	srv.GET("/frontend/*", echo.WrapHandler(fSrv))
-	srv.Static(c.UploadURI, c.UploadURI)
+	if ko.String("upload.provider") == "filesystem" {
+		srv.Static(ko.String("upload.filesystem.upload_uri"), ko.String("upload.filesystem.upload_path"))
+	}
 	registerHandlers(srv)
 	srv.Logger.Fatal(srv.Start(ko.String("app.address")))
 }

media.go

@@ -3,12 +3,9 @@ package main
 import (
 	"fmt"
 	"net/http"
-	"os"
-	"path/filepath"
 	"strconv"
 
-	"github.com/disintegration/imaging"
+	"github.com/knadh/listmonk/media"
-	"github.com/knadh/listmonk/models"
 	"github.com/labstack/echo"
 	uuid "github.com/satori/go.uuid"
 )
@@ -26,45 +23,64 @@ func handleUploadMedia(c echo.Context) error {
 		app     = c.Get("app").(*App)
 		cleanUp = false
 	)
-
+	file, err := c.FormFile("file")
-	// Upload the file.
-	fName, err := uploadFile("file", app.Constants.UploadPath, "", imageMimes, c)
 	if err != nil {
+		return echo.NewHTTPError(http.StatusBadRequest,
+			fmt.Sprintf("Invalid file uploaded: %v", err))
+	}
+	// Validate MIME type with the list of allowed types.
+	var typ = file.Header.Get("Content-type")
+	ok := validateMIME(typ, imageMimes)
+	if !ok {
+		return echo.NewHTTPError(http.StatusBadRequest,
+			fmt.Sprintf("Unsupported file type (%s) uploaded.", typ))
+	}
+	// Generate filename
+	fName := generateFileName(file.Filename)
+	// Read file contents in memory
+	src, err := file.Open()
+	if err != nil {
+		return echo.NewHTTPError(http.StatusBadRequest,
+			fmt.Sprintf("Error reading file: %s", err))
+	}
+	defer src.Close()
+	// Upload the file.
+	fName, err = app.Media.Put(fName, typ, src)
+	if err != nil {
+		cleanUp = true
 		return echo.NewHTTPError(http.StatusInternalServerError,
 			fmt.Sprintf("Error uploading file: %s", err))
 	}
-	path := filepath.Join(app.Constants.UploadPath, fName)
 
 	defer func() {
 		// If any of the subroutines in this function fail,
 		// the uploaded image should be removed.
 		if cleanUp {
-			os.Remove(path)
+			app.Media.Delete(fName)
+			app.Media.Delete(thumbPrefix + fName)
 		}
 	}()
 
-	// Create a thumbnail.
+	// Create thumbnail from file.
-	src, err := imaging.Open(path)
+	thumbFile, err := createThumbnail(file)
 	if err != nil {
 		cleanUp = true
 		return echo.NewHTTPError(http.StatusInternalServerError,
 			fmt.Sprintf("Error opening image for resizing: %s", err))
 	}
-
+	// Upload thumbnail.
-	t := imaging.Resize(src, thumbnailSize, 0, imaging.Lanczos)
+	thumbfName, err := app.Media.Put(thumbPrefix+fName, typ, thumbFile)
-	if err := imaging.Save(t, fmt.Sprintf("%s/%s%s", app.Constants.UploadPath, thumbPrefix, fName)); err != nil {
+	if err != nil {
 		cleanUp = true
 		return echo.NewHTTPError(http.StatusInternalServerError,
 			fmt.Sprintf("Error saving thumbnail: %s", err))
 	}
-
 	// Write to the DB.
-	if _, err := app.Queries.InsertMedia.Exec(uuid.NewV4(), fName, fmt.Sprintf("%s%s", thumbPrefix, fName), 0, 0); err != nil {
+	if _, err := app.Queries.InsertMedia.Exec(uuid.NewV4(), fName, thumbfName, 0, 0); err != nil {
 		cleanUp = true
 		return echo.NewHTTPError(http.StatusInternalServerError,
-			fmt.Sprintf("Error saving uploaded file: %s", pqErrMsg(err)))
+			fmt.Sprintf("Error saving uploaded file to db: %s", pqErrMsg(err)))
 	}
-
 	return c.JSON(http.StatusOK, okResp{true})
 }
 
@@ -72,7 +88,7 @@ func handleUploadMedia(c echo.Context) error {
 func handleGetMedia(c echo.Context) error {
 	var (
 		app = c.Get("app").(*App)
-		out []models.Media
+		out []media.Media
 	)
 
 	if err := app.Queries.GetMedia.Select(&out); err != nil {
@@ -81,8 +97,8 @@ func handleGetMedia(c echo.Context) error {
 	}
 
 	for i := 0; i < len(out); i++ {
-		out[i].URI = fmt.Sprintf("%s/%s", app.Constants.UploadURI, out[i].Filename)
+		out[i].URI = app.Media.Get(out[i].Filename)
-		out[i].ThumbURI = fmt.Sprintf("%s/%s%s", app.Constants.UploadURI, thumbPrefix, out[i].Filename)
+		out[i].ThumbURI = app.Media.Get(thumbPrefix + out[i].Filename)
 	}
 
 	return c.JSON(http.StatusOK, okResp{out})
@@ -99,13 +115,14 @@ func handleDeleteMedia(c echo.Context) error {
 		return echo.NewHTTPError(http.StatusBadRequest, "Invalid ID.")
 	}
 
-	var m models.Media
+	var m media.Media
 	if err := app.Queries.DeleteMedia.Get(&m, id); err != nil {
 		return echo.NewHTTPError(http.StatusInternalServerError,
 			fmt.Sprintf("Error deleting media: %s", pqErrMsg(err)))
 	}
-	os.Remove(filepath.Join(app.Constants.UploadPath, m.Filename))
+
-	os.Remove(filepath.Join(app.Constants.UploadPath, fmt.Sprintf("%s%s", thumbPrefix, m.Filename)))
+	app.Media.Delete(m.Filename)
+	app.Media.Delete(thumbPrefix + m.Filename)
 
 	return c.JSON(http.StatusOK, okResp{true})
 }

media/media.go

@@ -0,0 +1,26 @@
+package media
+
+import (
+	"io"
+
+	"gopkg.in/volatiletech/null.v6"
+)
+
+// Media represents an uploaded object.
+type Media struct {
+	ID        int       `db:"id" json:"id"`
+	UUID      string    `db:"uuid" json:"uuid"`
+	Filename  string    `db:"filename" json:"filename"`
+	Width     int       `db:"width" json:"width"`
+	Height    int       `db:"height" json:"height"`
+	CreatedAt null.Time `db:"created_at" json:"created_at"`
+	ThumbURI  string    `json:"thumb_uri"`
+	URI       string    `json:"uri"`
+}
+
+// Store represents set of methods to perform upload/delete operations.
+type Store interface {
+	Put(string, string, io.ReadSeeker) (string, error)
+	Delete(string) error
+	Get(string) string
+}

media/providers/filesystem/filesystem.go

@@ -0,0 +1,133 @@
+package filesystem
+
+import (
+	"crypto/rand"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"github.com/knadh/listmonk/media"
+)
+
+const tmpFilePrefix = "listmonk"
+
+// Opts represents filesystem params
+type Opts struct {
+	UploadPath string `koanf:"upload_path"`
+	UploadURI  string `koanf:"upload_uri"`
+}
+
+// Client implements `media.Store`
+type Client struct {
+	opts Opts
+}
+
+// NewDiskStore initialises store for Filesystem provider.
+func NewDiskStore(opts Opts) (media.Store, error) {
+	return &Client{
+		opts: opts,
+	}, nil
+}
+
+// Put accepts the filename, the content type and file object itself and stores the file in disk.
+func (e *Client) Put(filename string, cType string, src io.ReadSeeker) (string, error) {
+	var out *os.File
+	// There's no explicit name. Use the one posted in the HTTP request.
+	if filename == "" {
+		filename = strings.TrimSpace(filename)
+		if filename == "" {
+			filename, _ = generateRandomString(10)
+		}
+	}
+	// Get the directory path
+	dir := getDir(e.opts.UploadPath)
+	filename = assertUniqueFilename(dir, filename)
+	o, err := os.OpenFile(filepath.Join(dir, filename), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0664)
+	if err != nil {
+		return "", err
+	}
+	out = o
+	defer out.Close()
+
+	if _, err := io.Copy(out, src); err != nil {
+		return "", err
+	}
+	return filename, nil
+}
+
+// Get accepts a filename and retrieves the full path from disk.
+func (e *Client) Get(name string) string {
+	return fmt.Sprintf("%s/%s", e.opts.UploadURI, name)
+}
+
+// Delete accepts a filename and removes it from disk.
+func (e *Client) Delete(file string) error {
+	dir := getDir(e.opts.UploadPath)
+	err := os.Remove(filepath.Join(dir, file))
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// This matches filenames, sans extensions, of the format
+// filename_(number). The number is incremented in case
+// new file uploads conflict with existing filenames
+// on the filesystem.
+var fnameRegexp = regexp.MustCompile(`(.+?)_([0-9]+)$`)
+
+// assertUniqueFilename takes a file path and check if it exists on the disk. If it doesn't,
+// it returns the same name and if it does, it adds a small random hash to the filename
+// and returns that.
+func assertUniqueFilename(dir, fileName string) string {
+	var (
+		ext  = filepath.Ext(fileName)
+		base = fileName[0 : len(fileName)-len(ext)]
+		num  = 0
+	)
+
+	for {
+		// There's no name conflict.
+		if _, err := os.Stat(filepath.Join(dir, fileName)); os.IsNotExist(err) {
+			return fileName
+		}
+
+		// Does the name match the _(num) syntax?
+		r := fnameRegexp.FindAllStringSubmatch(fileName, -1)
+		if len(r) == 1 && len(r[0]) == 3 {
+			num, _ = strconv.Atoi(r[0][2])
+		}
+		num++
+
+		fileName = fmt.Sprintf("%s_%d%s", base, num, ext)
+	}
+}
+
+// generateRandomString generates a cryptographically random, alphanumeric string of length n.
+func generateRandomString(n int) (string, error) {
+	const dictionary = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
+
+	var bytes = make([]byte, n)
+	if _, err := rand.Read(bytes); err != nil {
+		return "", err
+	}
+
+	for k, v := range bytes {
+		bytes[k] = dictionary[v%byte(len(dictionary))]
+	}
+
+	return string(bytes), nil
+}
+
+// getDir returns the current working directory path if no directory is specified,
+// else returns the directory path specified itself.
+func getDir(dir string) string {
+	if dir == "" {
+		dir, _ = os.Getwd()
+	}
+	return dir
+}

media/providers/s3/s3.go

@@ -0,0 +1,108 @@
+package s3
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"time"
+
+	"github.com/knadh/listmonk/media"
+	"github.com/rhnvrm/simples3"
+)
+
+const amznS3PublicURL = "https://%s.s3.%s.amazonaws.com/%s"
+
+// Opts represents AWS S3 specific params
+type Opts struct {
+	AccessKey  string `koanf:"aws_access_key_id"`
+	SecretKey  string `koanf:"aws_secret_access_key"`
+	Region     string `koanf:"aws_default_region"`
+	Bucket     string `koanf:"bucket"`
+	BucketPath string `koanf:"bucket_path"`
+	BucketType string `koanf:"bucket_type"`
+	Expiry     int    `koanf:"expiry"`
+}
+
+// Client implements `media.Store` for S3 provider
+type Client struct {
+	s3   *simples3.S3
+	opts Opts
+}
+
+// NewS3Store initialises store for S3 provider. It takes in the AWS configuration
+// and sets up the `simples3` client to interact with AWS APIs for all bucket operations.
+func NewS3Store(opts Opts) (media.Store, error) {
+	var s3svc *simples3.S3
+	var err error
+	if opts.Region == "" {
+		return nil, errors.New("Invalid AWS Region specified. Please check `upload.s3` config")
+	}
+	// Use Access Key/Secret Key if specified in config.
+	if opts.AccessKey != "" && opts.SecretKey != "" {
+		s3svc = simples3.New(opts.Region, opts.AccessKey, opts.SecretKey)
+	} else {
+		// fallback to IAM role if no access key/secret key is provided.
+		s3svc, err = simples3.NewUsingIAM(opts.Region)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return &Client{
+		s3:   s3svc,
+		opts: opts,
+	}, nil
+}
+
+// Put takes in the filename, the content type and file object itself and uploads to S3.
+func (e *Client) Put(name string, cType string, file io.ReadSeeker) (string, error) {
+	// Upload input parameters
+	upParams := simples3.UploadInput{
+		Bucket:      e.opts.Bucket,
+		ObjectKey:   getBucketPath(e.opts.BucketPath, name),
+		ContentType: cType,
+		FileName:    name,
+		Body:        file,
+	}
+	// Perform an upload.
+	_, err := e.s3.FileUpload(upParams)
+	if err != nil {
+		return "", err
+	}
+	return name, nil
+}
+
+// Get accepts the filename of the object stored and retrieves from S3.
+func (e *Client) Get(name string) string {
+	// Generate a private S3 pre-signed URL if it's a private bucket.
+	if e.opts.BucketType == "private" {
+		url := e.s3.GeneratePresignedURL(simples3.PresignedInput{
+			Bucket:        e.opts.Bucket,
+			ObjectKey:     getBucketPath(e.opts.BucketPath, name),
+			Method:        "GET",
+			Timestamp:     time.Now(),
+			ExpirySeconds: e.opts.Expiry,
+		})
+		return url
+	}
+	// Generate a public S3 URL if it's a public bucket.
+	url := fmt.Sprintf(amznS3PublicURL, e.opts.Bucket, e.opts.Region, getBucketPath(e.opts.BucketPath, name))
+	return url
+}
+
+// Delete accepts the filename of the object and deletes from S3.
+func (e *Client) Delete(name string) error {
+	err := e.s3.FileDelete(simples3.DeleteInput{
+		Bucket:    e.opts.Bucket,
+		ObjectKey: getBucketPath(e.opts.BucketPath, name),
+	})
+	return err
+}
+
+// getBucketPath constructs the key for the object stored in S3.
+// If path is empty, the key is the combination of root of S3 bucket and filename.
+func getBucketPath(path string, name string) string {
+	if path == "" {
+		return fmt.Sprintf("%s", name)
+	}
+	return fmt.Sprintf("%s/%s", path, name)
+}

models/models.go

@@ -174,19 +174,6 @@ type CampaignMeta struct {
 // Campaigns represents a slice of Campaigns.
 type Campaigns []Campaign
 
-// Media represents an uploaded media item.
-type Media struct {
-	ID        int       `db:"id" json:"id"`
-	UUID      string    `db:"uuid" json:"uuid"`
-	Filename  string    `db:"filename" json:"filename"`
-	Width     int       `db:"width" json:"width"`
-	Height    int       `db:"height" json:"height"`
-	CreatedAt null.Time `db:"created_at" json:"created_at"`
-
-	ThumbURI string `json:"thumb_uri"`
-	URI      string `json:"uri"`
-}
-
 // Template represents a reusable e-mail template.
 type Template struct {
 	Base

utils.go

@@ -4,30 +4,22 @@ import (
 	"bytes"
 	"crypto/rand"
 	"fmt"
-	"io"
+	"log"
-	"io/ioutil"
+	"mime/multipart"
 	"net/http"
-	"os"
-	"path/filepath"
 	"reflect"
 	"regexp"
 	"strconv"
 	"strings"
 
+	"github.com/disintegration/imaging"
 	"github.com/jmoiron/sqlx"
 	"github.com/knadh/goyesql"
 	"github.com/labstack/echo"
 	"github.com/lib/pq"
 )
 
-const tmpFilePrefix = "listmonk"
-
 var (
-	// This matches filenames, sans extensions, of the format
-	// filename_(number). The number is incremented in case
-	// new file uploads conflict with existing filenames
-	// on the filesystem.
-	fnameRegexp = regexp.MustCompile(`(.+?)_([0-9]+)$`)
 
 	// This replaces all special characters
 	tagRegexp       = regexp.MustCompile(`[^a-z0-9\-\s]`)
@@ -95,23 +87,12 @@ func scanQueriesToStruct(obj interface{}, q goyesql.Queries, db *sqlx.DB) error
 	return nil
 }
 
-// uploadFile is a helper function on top of echo.Context for processing file uploads.
+// validateMIME is a helper function to validate uploaded file's MIME type
-// It allows copying a single file given the incoming file field name.
+// against the slice of MIME types is given.
-// If the upload directory dir is empty, the file is copied to the system's temp directory.
+func validateMIME(typ string, mimes []string) (ok bool) {
-// If name is empty, the incoming file's name along with a small random hash is used.
-// When a slice of MIME types is given, the uploaded file's MIME type is validated against the list.
-func uploadFile(key string, dir, name string, mimes []string, c echo.Context) (string, error) {
-	file, err := c.FormFile(key)
-	if err != nil {
-		return "", echo.NewHTTPError(http.StatusBadRequest,
-			fmt.Sprintf("Invalid file uploaded: %v", err))
-	}
-
-	// Check MIME type.
 	if len(mimes) > 0 {
 		var (
-			typ = file.Header.Get("Content-type")
+			ok = false
-			ok  = false
 		)
 		for _, m := range mimes {
 			if typ == m {
@@ -119,55 +100,42 @@ func uploadFile(key string, dir, name string, mimes []string, c echo.Context) (s
 				break
 			}
 		}
-
 		if !ok {
-			return "", echo.NewHTTPError(http.StatusBadRequest,
+			return false
-				fmt.Sprintf("Unsupported file type (%s) uploaded.", typ))
 		}
 	}
+	return true
+}
 
+// generateFileName appends the incoming file's name with a small random hash.
+func generateFileName(fName string) string {
+	name := strings.TrimSpace(fName)
+	if name == "" {
+		name, _ = generateRandomString(10)
+	}
+	return name
+}
+
+// createThumbnail reads the file object and returns a smaller image
+func createThumbnail(file *multipart.FileHeader) (*bytes.Reader, error) {
 	src, err := file.Open()
 	if err != nil {
-		return "", err
+		return nil, err
 	}
 	defer src.Close()
-
+	img, err := imaging.Decode(src)
-	// There's no upload directory. Use a tempfile.
+	if err != nil {
-	var out *os.File
+		return nil, echo.NewHTTPError(http.StatusInternalServerError,
-	if dir == "" {
+			fmt.Sprintf("Error decoding image: %v", err))
-		o, err := ioutil.TempFile("", tmpFilePrefix)
-		if err != nil {
-			return "", echo.NewHTTPError(http.StatusInternalServerError,
-				fmt.Sprintf("Error copying uploaded file: %v", err))
-		}
-		out = o
-		name = o.Name()
-	} else {
-		// There's no explicit name. Use the one posted in the HTTP request.
-		if name == "" {
-			name = strings.TrimSpace(file.Filename)
-			if name == "" {
-				name, _ = generateRandomString(10)
-			}
-		}
-		name = assertUniqueFilename(dir, name)
-
-		o, err := os.OpenFile(filepath.Join(dir, name), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0664)
-		if err != nil {
-			return "", echo.NewHTTPError(http.StatusInternalServerError,
-				fmt.Sprintf("Error copying uploaded file: %v", err))
-		}
-
-		out = o
 	}
-	defer out.Close()
+	t := imaging.Resize(img, thumbnailSize, 0, imaging.Lanczos)
-
+	// Encode the image into a byte slice as PNG.
-	if _, err = io.Copy(out, src); err != nil {
+	var buf bytes.Buffer
-		return "", echo.NewHTTPError(http.StatusInternalServerError,
+	err = imaging.Encode(&buf, t, imaging.PNG)
-			fmt.Sprintf("Error copying uploaded file: %v", err))
+	if err != nil {
+		log.Fatal(err)
 	}
-
+	return bytes.NewReader(buf.Bytes()), nil
-	return name, nil
 }
 
 // Given an error, pqErrMsg will try to return pq error details
@@ -182,49 +150,6 @@ func pqErrMsg(err error) string {
 	return err.Error()
 }
 
-// generateRandomString generates a cryptographically random, alphanumeric string of length n.
-func generateRandomString(n int) (string, error) {
-	const dictionary = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
-
-	var bytes = make([]byte, n)
-	if _, err := rand.Read(bytes); err != nil {
-		return "", err
-	}
-
-	for k, v := range bytes {
-		bytes[k] = dictionary[v%byte(len(dictionary))]
-	}
-
-	return string(bytes), nil
-}
-
-// assertUniqueFilename takes a file path and check if it exists on the disk. If it doesn't,
-// it returns the same name and if it does, it adds a small random hash to the filename
-// and returns that.
-func assertUniqueFilename(dir, fileName string) string {
-	var (
-		ext  = filepath.Ext(fileName)
-		base = fileName[0 : len(fileName)-len(ext)]
-		num  = 0
-	)
-
-	for {
-		// There's no name conflict.
-		if _, err := os.Stat(filepath.Join(dir, fileName)); os.IsNotExist(err) {
-			return fileName
-		}
-
-		// Does the name match the _(num) syntax?
-		r := fnameRegexp.FindAllStringSubmatch(fileName, -1)
-		if len(r) == 1 && len(r[0]) == 3 {
-			num, _ = strconv.Atoi(r[0][2])
-		}
-		num++
-
-		fileName = fmt.Sprintf("%s_%d%s", base, num, ext)
-	}
-}
-
 // normalizeTags takes a list of string tags and normalizes them by
 // lowercasing and removing all special characters except for dashes.
 func normalizeTags(tags []string) []string {
@@ -282,3 +207,19 @@ func parseStringIDs(s []string) ([]int64, error) {
 
 	return vals, nil
 }
+
+// generateRandomString generates a cryptographically random, alphanumeric string of length n.
+func generateRandomString(n int) (string, error) {
+	const dictionary = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
+
+	var bytes = make([]byte, n)
+	if _, err := rand.Read(bytes); err != nil {
+		return "", err
+	}
+
+	for k, v := range bytes {
+		bytes[k] = dictionary[v%byte(len(dictionary))]
+	}
+
+	return string(bytes), nil
+}