hangar-tech
/
listmonk
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #31 from mr-karan/s3 

feat: Add media package that allows pluggable backends (filesystem / s3) for media file uploads.
This commit is contained in:
Kailash Nadh 2019-10-31 12:01:13 +05:30 committed by GitHub
parent 7ee71166fb e5c3196b31
commit ccd354df3f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 463 additions and 159 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
.dockerignore Normal file
View File

@ -0,0 +1,25 @@
**/.classpath
**/.dockerignore
**/.env
**/.git
**/.gitignore
**/.project
**/.settings
**/.toolstarget
**/.vs
**/.vscode
**/*.*proj.user
**/*.dbmdl
**/*.jfm
**/azds.yaml
**/bin
**/charts
**/docker-compose*
**/Dockerfile*
**/node_modules
**/npm-debug.log
**/obj
**/secrets.dev.yaml
**/values.dev.yaml
LICENSE
README.md

1
.gitignore vendored
View File

@ -6,3 +6,4 @@ frontend/yarn.lock
config.toml
node_modules
listmonk
dist/*

1
admin.go
View File

@ -28,7 +28,6 @@ func handleGetConfigScript(c echo.Context) error {
app = c.Get("app").(*App)
out = configScript{
RootURL: app.Constants.RootURL,
UploadURI: app.Constants.UploadURI,
FromEmail: app.Constants.FromEmail,
Messengers: app.Manager.GetMessengerNames(),
}

38
config.toml.sample
View File

@ -24,14 +24,6 @@ from_email = "listmonk <from@mail.com>"
# To disable notifications, set an empty list, eg: notify_emails = []
notify_emails = ["admin1@mysite.com", "admin2@mysite.com"]
# Path to the uploads directory where media will be uploaded.
upload_path = "uploads"
# Upload URI that's visible to the outside world. The media
# uploaded to upload_path will be made available publicly
# under this URI, for instance, list.yoursite.com/uploads.
upload_uri = "/uploads"
# Maximum concurrent workers that will attempt to send messages
# simultaneously. This should depend on the number of CPUs the
# machine has and also the number of simultaenous e-mails the
@ -110,3 +102,33 @@ ssl_mode = "disable"
# Maximum concurrent connections to the SMTP server.
max_conns = 10
# Upload settings
[upload]
# Provider which will be used to host uploaded media. Bundled providers are "filesystem" and "s3".
provider = "filesystem"
# S3 Provider settings
[upload.s3]
# (Optional). AWS Access Key and Secret Key for the user to access the bucket. Leaving it empty would default to use
# instance IAM role.
aws_access_key_id = ""
aws_secret_access_key = ""
# AWS Region where S3 bucket is hosted.
aws_default_region="ap-south-1"
# Specify bucket name.
bucket=""
# Path where the files will be stored inside bucket. Empty value ("") means the root of bucket.
bucket_path=""
# Bucket type can be "private" or "public".
bucket_type="public"
# (Optional) Specify TTL (in seconds) for the generated presigned URL. Expiry value is used only if the bucket is private.
expiry="86400"
# Filesystem provider settings
[upload.filesystem]
# Path to the uploads directory where media will be uploaded. Leaving it empty ("") means current working directory.
upload_path=""
# Upload URI that's visible to the outside world. The media uploaded to upload_path will be made available publicly
# under this URI, for instance, list.yoursite.com/uploads.
upload_uri = "/uploads"

4
go.mod
View File

@ -2,6 +2,7 @@ module github.com/knadh/listmonk
require (
github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf
github.com/aws/aws-sdk-go v1.25.12
github.com/disintegration/imaging v1.5.0
github.com/jinzhu/gorm v1.9.1
github.com/jmoiron/sqlx v1.2.0
@ -15,6 +16,7 @@ require (
github.com/lib/pq v1.0.0
github.com/mattn/go-colorable v0.0.9 // indirect
github.com/mattn/go-isatty v0.0.4 // indirect
github.com/rhnvrm/simples3 v0.2.4-0.20191018074503-3d5b071ef727
github.com/satori/go.uuid v1.2.0
github.com/spf13/pflag v1.0.3
github.com/stretchr/objx v0.2.0 // indirect
@ -27,3 +29,5 @@ require (
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
gopkg.in/volatiletech/null.v6 v6.0.0-20170828023728-0bef4e07ae1b
)
go 1.13

9
go.sum
View File

@ -2,6 +2,8 @@ github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf h1:eg0MeVzsP1G42dRafH3vf+al2vQIJU0YHX+1Tw87oco=
github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
github.com/aws/aws-sdk-go v1.25.12 h1:a4h2FxoUJq9h+hajSE/dsRiqoOniIh6BkzhxMjkepzY=
github.com/aws/aws-sdk-go v1.25.12/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@ -16,6 +18,8 @@ github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
github.com/jinzhu/gorm v1.9.1 h1:lDSDtsCt5AGGSKTs8AHlSDbbgif4G4+CKJ8ETBDVHTA=
github.com/jinzhu/gorm v1.9.1/go.mod h1:Vla75njaFJ8clLU1W44h34PjIkijhjHIYnZxMqCdxqo=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA=
github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
github.com/jordan-wright/email v0.0.0-20181027021455-480bedc4908b h1:veTPVnbkOijplSJVywDYKDRPoZEN39kfuMDzzRKP0FA=
@ -57,6 +61,10 @@ github.com/pelletier/go-toml v1.4.0 h1:u3Z1r+oOXJIkxqw34zVhyPgjBsm6X2wn21NWs/HfS
github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rhnvrm/simples3 v0.2.3 h1:qNXPynabu8M3F4+69fspA5aWZR8jqVV1RQtv2xc1OVk=
github.com/rhnvrm/simples3 v0.2.3/go.mod h1:iphavgjkW1uvoIiqLUX6D42XuuI9Cr+B/63xw3gb9qA=
github.com/rhnvrm/simples3 v0.2.4-0.20191018074503-3d5b071ef727 h1:2josYcx2gm3CT0WMqi0jBagvg50V3UMWlYN/CnBEbSI=
github.com/rhnvrm/simples3 v0.2.4-0.20191018074503-3d5b071ef727/go.mod h1:iphavgjkW1uvoIiqLUX6D42XuuI9Cr+B/63xw3gb9qA=
github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
@ -73,6 +81,7 @@ golang.org/x/crypto v0.0.0-20181106171534-e4dc69e5b2fd h1:VtIkGDhk0ph3t+THbvXHfM
golang.org/x/crypto v0.0.0-20181106171534-e4dc69e5b2fd/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/image v0.0.0-20181116024801-cd38e8056d9b h1:VHyIDlv3XkfCa5/a81uzaoDkHH4rr81Z62g+xlnO8uM=
golang.org/x/image v0.0.0-20181116024801-cd38e8056d9b/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225 h1:kNX+jCowfMYzvlSvJu5pQWEmyWFrBXJ3PBy10xKMXK8=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/sys v0.0.0-20180906133057-8cf3aee42992 h1:BH3eQWeGbwRU2+wxxuuPOdFBmaiBH81O8BugSjHeTFg=
golang.org/x/sys v0.0.0-20180906133057-8cf3aee42992/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=

42
main.go
View File

@ -19,6 +19,9 @@ import (
"github.com/knadh/koanf/providers/file"
"github.com/knadh/koanf/providers/posflag"
"github.com/knadh/listmonk/manager"
"github.com/knadh/listmonk/media"
"github.com/knadh/listmonk/media/providers/filesystem"
"github.com/knadh/listmonk/media/providers/s3"
"github.com/knadh/listmonk/messenger"
"github.com/knadh/listmonk/subimporter"
"github.com/knadh/stuffbin"
@ -30,8 +33,6 @@ type constants struct {
RootURL string `koanf:"root"`
LogoURL string `koanf:"logo_url"`
FaviconURL string `koanf:"favicon_url"`
UploadPath string `koanf:"upload_path"`
UploadURI string `koanf:"upload_uri"`
FromEmail string `koanf:"from_email"`
NotifyEmails []string `koanf:"notify_emails"`
Privacy privacyOptions `koanf:"privacy"`
@ -56,6 +57,7 @@ type App struct {
Logger *log.Logger
NotifTpls *template.Template
Messenger messenger.Messenger
Media media.Store
}
var (
@ -195,6 +197,33 @@ func initMessengers(r *manager.Manager) messenger.Messenger {
return msgr
}
// initMediaStore initializes Upload manager with a custom backend.
func initMediaStore() media.Store {
switch provider := ko.String("upload.provider"); provider {
case "s3":
var opts s3.Opts
ko.Unmarshal("upload.s3", &opts)
uplder, err := s3.NewS3Store(opts)
if err != nil {
logger.Fatalf("error initializing s3 upload provider %s", err)
}
return uplder
case "filesystem":
var opts filesystem.Opts
ko.Unmarshal("upload.filesystem", &opts)
opts.UploadPath = filepath.Clean(opts.UploadPath)
opts.UploadURI = filepath.Clean(opts.UploadURI)
uplder, err := filesystem.NewDiskStore(opts)
if err != nil {
logger.Fatalf("error initializing filesystem upload provider %s", err)
}
return uplder
default:
logger.Fatalf("unknown provider. please select one of either filesystem or s3")
}
return nil
}
func main() {
// Connect to the DB.
db, err := connectDB(ko.String("db.host"),
@ -216,8 +245,6 @@ func main() {
log.Fatalf("error loading app config: %v", err)
}
c.RootURL = strings.TrimRight(c.RootURL, "/")
c.UploadURI = filepath.Clean(c.UploadURI)
c.UploadPath = filepath.Clean(c.UploadPath)
c.Privacy.Exportable = maps.StringSliceToLookupMap(ko.Strings("privacy.exportable"))
// Initialize the static file system into which all
@ -299,6 +326,9 @@ func main() {
// Add messengers.
app.Messenger = initMessengers(app.Manager)
// Add uploader
app.Media = initMediaStore()
// Initialize the workers that push out messages.
go m.Run(time.Second * 5)
m.SpawnWorkers()
@ -330,7 +360,9 @@ func main() {
fSrv := app.FS.FileServer()
srv.GET("/public/*", echo.WrapHandler(fSrv))
srv.GET("/frontend/*", echo.WrapHandler(fSrv))
srv.Static(c.UploadURI, c.UploadURI)
if ko.String("upload.provider") == "filesystem" {
srv.Static(ko.String("upload.filesystem.upload_uri"), ko.String("upload.filesystem.upload_path"))
}
registerHandlers(srv)
srv.Logger.Fatal(srv.Start(ko.String("app.address")))
}

65
media.go
View File

@ -3,12 +3,9 @@ package main
import (
"fmt"
"net/http"
"os"
"path/filepath"
"strconv"
"github.com/disintegration/imaging"
"github.com/knadh/listmonk/models"
"github.com/knadh/listmonk/media"
"github.com/labstack/echo"
uuid "github.com/satori/go.uuid"
)
@ -26,45 +23,64 @@ func handleUploadMedia(c echo.Context) error {
app = c.Get("app").(*App)
cleanUp = false
)
// Upload the file.
fName, err := uploadFile("file", app.Constants.UploadPath, "", imageMimes, c)
file, err := c.FormFile("file")
if err != nil {
return echo.NewHTTPError(http.StatusBadRequest,
fmt.Sprintf("Invalid file uploaded: %v", err))
}
// Validate MIME type with the list of allowed types.
var typ = file.Header.Get("Content-type")
ok := validateMIME(typ, imageMimes)
if !ok {
return echo.NewHTTPError(http.StatusBadRequest,
fmt.Sprintf("Unsupported file type (%s) uploaded.", typ))
}
// Generate filename
fName := generateFileName(file.Filename)
// Read file contents in memory
src, err := file.Open()
if err != nil {
return echo.NewHTTPError(http.StatusBadRequest,
fmt.Sprintf("Error reading file: %s", err))
}
defer src.Close()
// Upload the file.
fName, err = app.Media.Put(fName, typ, src)
if err != nil {
cleanUp = true
return echo.NewHTTPError(http.StatusInternalServerError,
fmt.Sprintf("Error uploading file: %s", err))
}
path := filepath.Join(app.Constants.UploadPath, fName)
defer func() {
// If any of the subroutines in this function fail,
// the uploaded image should be removed.
if cleanUp {
os.Remove(path)
app.Media.Delete(fName)
app.Media.Delete(thumbPrefix + fName)
}
}()
// Create a thumbnail.
src, err := imaging.Open(path)
// Create thumbnail from file.
thumbFile, err := createThumbnail(file)
if err != nil {
cleanUp = true
return echo.NewHTTPError(http.StatusInternalServerError,
fmt.Sprintf("Error opening image for resizing: %s", err))
}
t := imaging.Resize(src, thumbnailSize, 0, imaging.Lanczos)
if err := imaging.Save(t, fmt.Sprintf("%s/%s%s", app.Constants.UploadPath, thumbPrefix, fName)); err != nil {
// Upload thumbnail.
thumbfName, err := app.Media.Put(thumbPrefix+fName, typ, thumbFile)
if err != nil {
cleanUp = true
return echo.NewHTTPError(http.StatusInternalServerError,
fmt.Sprintf("Error saving thumbnail: %s", err))
}
// Write to the DB.
if _, err := app.Queries.InsertMedia.Exec(uuid.NewV4(), fName, fmt.Sprintf("%s%s", thumbPrefix, fName), 0, 0); err != nil {
if _, err := app.Queries.InsertMedia.Exec(uuid.NewV4(), fName, thumbfName, 0, 0); err != nil {
cleanUp = true
return echo.NewHTTPError(http.StatusInternalServerError,
fmt.Sprintf("Error saving uploaded file: %s", pqErrMsg(err)))
fmt.Sprintf("Error saving uploaded file to db: %s", pqErrMsg(err)))
}
return c.JSON(http.StatusOK, okResp{true})
}
@ -72,7 +88,7 @@ func handleUploadMedia(c echo.Context) error {
func handleGetMedia(c echo.Context) error {
var (
app = c.Get("app").(*App)
out []models.Media
out []media.Media
)
if err := app.Queries.GetMedia.Select(&out); err != nil {
@ -81,8 +97,8 @@ func handleGetMedia(c echo.Context) error {
}
for i := 0; i < len(out); i++ {
out[i].URI = fmt.Sprintf("%s/%s", app.Constants.UploadURI, out[i].Filename)
out[i].ThumbURI = fmt.Sprintf("%s/%s%s", app.Constants.UploadURI, thumbPrefix, out[i].Filename)
out[i].URI = app.Media.Get(out[i].Filename)
out[i].ThumbURI = app.Media.Get(thumbPrefix + out[i].Filename)
}
return c.JSON(http.StatusOK, okResp{out})
@ -99,13 +115,14 @@ func handleDeleteMedia(c echo.Context) error {
return echo.NewHTTPError(http.StatusBadRequest, "Invalid ID.")
}
var m models.Media
var m media.Media
if err := app.Queries.DeleteMedia.Get(&m, id); err != nil {
return echo.NewHTTPError(http.StatusInternalServerError,
fmt.Sprintf("Error deleting media: %s", pqErrMsg(err)))
}
os.Remove(filepath.Join(app.Constants.UploadPath, m.Filename))
os.Remove(filepath.Join(app.Constants.UploadPath, fmt.Sprintf("%s%s", thumbPrefix, m.Filename)))
app.Media.Delete(m.Filename)
app.Media.Delete(thumbPrefix + m.Filename)
return c.JSON(http.StatusOK, okResp{true})
}

26
media/media.go Normal file
View File

@ -0,0 +1,26 @@
package media
import (
"io"
"gopkg.in/volatiletech/null.v6"
)
// Media represents an uploaded object.
type Media struct {
ID int `db:"id" json:"id"`
UUID string `db:"uuid" json:"uuid"`
Filename string `db:"filename" json:"filename"`
Width int `db:"width" json:"width"`
Height int `db:"height" json:"height"`
CreatedAt null.Time `db:"created_at" json:"created_at"`
ThumbURI string `json:"thumb_uri"`
URI string `json:"uri"`
}
// Store represents set of methods to perform upload/delete operations.
type Store interface {
Put(string, string, io.ReadSeeker) (string, error)
Delete(string) error
Get(string) string
}

133
media/providers/filesystem/filesystem.go Normal file
View File

@ -0,0 +1,133 @@
package filesystem
import (
"crypto/rand"
"fmt"
"io"
"os"
"path/filepath"
"regexp"
"strconv"
"strings"
"github.com/knadh/listmonk/media"
)
const tmpFilePrefix = "listmonk"
// Opts represents filesystem params
type Opts struct {
UploadPath string `koanf:"upload_path"`
UploadURI string `koanf:"upload_uri"`
}
// Client implements `media.Store`
type Client struct {
opts Opts
}
// NewDiskStore initialises store for Filesystem provider.
func NewDiskStore(opts Opts) (media.Store, error) {
return &Client{
opts: opts,
}, nil
}
// Put accepts the filename, the content type and file object itself and stores the file in disk.
func (e *Client) Put(filename string, cType string, src io.ReadSeeker) (string, error) {
var out *os.File
// There's no explicit name. Use the one posted in the HTTP request.
if filename == "" {
filename = strings.TrimSpace(filename)
if filename == "" {
filename, _ = generateRandomString(10)
}
}
// Get the directory path
dir := getDir(e.opts.UploadPath)
filename = assertUniqueFilename(dir, filename)
o, err := os.OpenFile(filepath.Join(dir, filename), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0664)
if err != nil {
return "", err
}
out = o
defer out.Close()
if _, err := io.Copy(out, src); err != nil {
return "", err
}
return filename, nil
}
// Get accepts a filename and retrieves the full path from disk.
func (e *Client) Get(name string) string {
return fmt.Sprintf("%s/%s", e.opts.UploadURI, name)
}
// Delete accepts a filename and removes it from disk.
func (e *Client) Delete(file string) error {
dir := getDir(e.opts.UploadPath)
err := os.Remove(filepath.Join(dir, file))
if err != nil {
return err
}
return nil
}
// This matches filenames, sans extensions, of the format
// filename_(number). The number is incremented in case
// new file uploads conflict with existing filenames
// on the filesystem.
var fnameRegexp = regexp.MustCompile(`(.+?)_([0-9]+)$`)
// assertUniqueFilename takes a file path and check if it exists on the disk. If it doesn't,
// it returns the same name and if it does, it adds a small random hash to the filename
// and returns that.
func assertUniqueFilename(dir, fileName string) string {
var (
ext = filepath.Ext(fileName)
base = fileName[0 : len(fileName)-len(ext)]
num = 0
)
for {
// There's no name conflict.
if _, err := os.Stat(filepath.Join(dir, fileName)); os.IsNotExist(err) {
return fileName
}
// Does the name match the _(num) syntax?
r := fnameRegexp.FindAllStringSubmatch(fileName, -1)
if len(r) == 1 && len(r[0]) == 3 {
num, _ = strconv.Atoi(r[0][2])
}
num++
fileName = fmt.Sprintf("%s_%d%s", base, num, ext)
}
}
// generateRandomString generates a cryptographically random, alphanumeric string of length n.
func generateRandomString(n int) (string, error) {
const dictionary = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
var bytes = make([]byte, n)
if _, err := rand.Read(bytes); err != nil {
return "", err
}
for k, v := range bytes {
bytes[k] = dictionary[v%byte(len(dictionary))]
}
return string(bytes), nil
}
// getDir returns the current working directory path if no directory is specified,
// else returns the directory path specified itself.
func getDir(dir string) string {
if dir == "" {
dir, _ = os.Getwd()
}
return dir
}

108
media/providers/s3/s3.go Normal file
View File

@ -0,0 +1,108 @@
package s3
import (
"errors"
"fmt"
"io"
"time"
"github.com/knadh/listmonk/media"
"github.com/rhnvrm/simples3"
)
const amznS3PublicURL = "https://%s.s3.%s.amazonaws.com/%s"
// Opts represents AWS S3 specific params
type Opts struct {
AccessKey string `koanf:"aws_access_key_id"`
SecretKey string `koanf:"aws_secret_access_key"`
Region string `koanf:"aws_default_region"`
Bucket string `koanf:"bucket"`
BucketPath string `koanf:"bucket_path"`
BucketType string `koanf:"bucket_type"`
Expiry int `koanf:"expiry"`
}
// Client implements `media.Store` for S3 provider
type Client struct {
s3 *simples3.S3
opts Opts
}
// NewS3Store initialises store for S3 provider. It takes in the AWS configuration
// and sets up the `simples3` client to interact with AWS APIs for all bucket operations.
func NewS3Store(opts Opts) (media.Store, error) {
var s3svc *simples3.S3
var err error
if opts.Region == "" {
return nil, errors.New("Invalid AWS Region specified. Please check `upload.s3` config")
}
// Use Access Key/Secret Key if specified in config.
if opts.AccessKey != "" && opts.SecretKey != "" {
s3svc = simples3.New(opts.Region, opts.AccessKey, opts.SecretKey)
} else {
// fallback to IAM role if no access key/secret key is provided.
s3svc, err = simples3.NewUsingIAM(opts.Region)
if err != nil {
return nil, err
}
}
return &Client{
s3: s3svc,
opts: opts,
}, nil
}
// Put takes in the filename, the content type and file object itself and uploads to S3.
func (e *Client) Put(name string, cType string, file io.ReadSeeker) (string, error) {
// Upload input parameters
upParams := simples3.UploadInput{
Bucket: e.opts.Bucket,
ObjectKey: getBucketPath(e.opts.BucketPath, name),
ContentType: cType,
FileName: name,
Body: file,
}
// Perform an upload.
_, err := e.s3.FileUpload(upParams)
if err != nil {
return "", err
}
return name, nil
}
// Get accepts the filename of the object stored and retrieves from S3.
func (e *Client) Get(name string) string {
// Generate a private S3 pre-signed URL if it's a private bucket.
if e.opts.BucketType == "private" {
url := e.s3.GeneratePresignedURL(simples3.PresignedInput{
Bucket: e.opts.Bucket,
ObjectKey: getBucketPath(e.opts.BucketPath, name),
Method: "GET",
Timestamp: time.Now(),
ExpirySeconds: e.opts.Expiry,
})
return url
}
// Generate a public S3 URL if it's a public bucket.
url := fmt.Sprintf(amznS3PublicURL, e.opts.Bucket, e.opts.Region, getBucketPath(e.opts.BucketPath, name))
return url
}
// Delete accepts the filename of the object and deletes from S3.
func (e *Client) Delete(name string) error {
err := e.s3.FileDelete(simples3.DeleteInput{
Bucket: e.opts.Bucket,
ObjectKey: getBucketPath(e.opts.BucketPath, name),
})
return err
}
// getBucketPath constructs the key for the object stored in S3.
// If path is empty, the key is the combination of root of S3 bucket and filename.
func getBucketPath(path string, name string) string {
if path == "" {
return fmt.Sprintf("%s", name)
}
return fmt.Sprintf("%s/%s", path, name)
}

13
models/models.go
View File

@ -174,19 +174,6 @@ type CampaignMeta struct {
// Campaigns represents a slice of Campaigns.
type Campaigns []Campaign
// Media represents an uploaded media item.
type Media struct {
ID int `db:"id" json:"id"`
UUID string `db:"uuid" json:"uuid"`
Filename string `db:"filename" json:"filename"`
Width int `db:"width" json:"width"`
Height int `db:"height" json:"height"`
CreatedAt null.Time `db:"created_at" json:"created_at"`
ThumbURI string `json:"thumb_uri"`
URI string `json:"uri"`
}
// Template represents a reusable e-mail template.
type Template struct {
Base

157
utils.go
View File

@ -4,30 +4,22 @@ import (
"bytes"
"crypto/rand"
"fmt"
"io"
"io/ioutil"
"log"
"mime/multipart"
"net/http"
"os"
"path/filepath"
"reflect"
"regexp"
"strconv"
"strings"
"github.com/disintegration/imaging"
"github.com/jmoiron/sqlx"
"github.com/knadh/goyesql"
"github.com/labstack/echo"
"github.com/lib/pq"
)
const tmpFilePrefix = "listmonk"
var (
// This matches filenames, sans extensions, of the format
// filename_(number). The number is incremented in case
// new file uploads conflict with existing filenames
// on the filesystem.
fnameRegexp = regexp.MustCompile(`(.+?)_([0-9]+)$`)
// This replaces all special characters
tagRegexp = regexp.MustCompile(`[^a-z0-9\-\s]`)
@ -95,23 +87,12 @@ func scanQueriesToStruct(obj interface{}, q goyesql.Queries, db *sqlx.DB) error
return nil
}
// uploadFile is a helper function on top of echo.Context for processing file uploads.
// It allows copying a single file given the incoming file field name.
// If the upload directory dir is empty, the file is copied to the system's temp directory.
// If name is empty, the incoming file's name along with a small random hash is used.
// When a slice of MIME types is given, the uploaded file's MIME type is validated against the list.
func uploadFile(key string, dir, name string, mimes []string, c echo.Context) (string, error) {
file, err := c.FormFile(key)
if err != nil {
return "", echo.NewHTTPError(http.StatusBadRequest,
fmt.Sprintf("Invalid file uploaded: %v", err))
}
// Check MIME type.
// validateMIME is a helper function to validate uploaded file's MIME type
// against the slice of MIME types is given.
func validateMIME(typ string, mimes []string) (ok bool) {
if len(mimes) > 0 {
var (
typ = file.Header.Get("Content-type")
ok = false
ok = false
)
for _, m := range mimes {
if typ == m {
@ -119,55 +100,42 @@ func uploadFile(key string, dir, name string, mimes []string, c echo.Context) (s
break
}
}
if !ok {
return "", echo.NewHTTPError(http.StatusBadRequest,
fmt.Sprintf("Unsupported file type (%s) uploaded.", typ))
return false
}
}
return true
}
// generateFileName appends the incoming file's name with a small random hash.
func generateFileName(fName string) string {
name := strings.TrimSpace(fName)
if name == "" {
name, _ = generateRandomString(10)
}
return name
}
// createThumbnail reads the file object and returns a smaller image
func createThumbnail(file *multipart.FileHeader) (*bytes.Reader, error) {
src, err := file.Open()
if err != nil {
return "", err
return nil, err
}
defer src.Close()
// There's no upload directory. Use a tempfile.
var out *os.File
if dir == "" {
o, err := ioutil.TempFile("", tmpFilePrefix)
if err != nil {
return "", echo.NewHTTPError(http.StatusInternalServerError,
fmt.Sprintf("Error copying uploaded file: %v", err))
}
out = o
name = o.Name()
} else {
// There's no explicit name. Use the one posted in the HTTP request.
if name == "" {
name = strings.TrimSpace(file.Filename)
if name == "" {
name, _ = generateRandomString(10)
}
}
name = assertUniqueFilename(dir, name)
o, err := os.OpenFile(filepath.Join(dir, name), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0664)
if err != nil {
return "", echo.NewHTTPError(http.StatusInternalServerError,
fmt.Sprintf("Error copying uploaded file: %v", err))
}
out = o
img, err := imaging.Decode(src)
if err != nil {
return nil, echo.NewHTTPError(http.StatusInternalServerError,
fmt.Sprintf("Error decoding image: %v", err))
}
defer out.Close()
if _, err = io.Copy(out, src); err != nil {
return "", echo.NewHTTPError(http.StatusInternalServerError,
fmt.Sprintf("Error copying uploaded file: %v", err))
t := imaging.Resize(img, thumbnailSize, 0, imaging.Lanczos)
// Encode the image into a byte slice as PNG.
var buf bytes.Buffer
err = imaging.Encode(&buf, t, imaging.PNG)
if err != nil {
log.Fatal(err)
}
return name, nil
return bytes.NewReader(buf.Bytes()), nil
}
// Given an error, pqErrMsg will try to return pq error details
@ -182,49 +150,6 @@ func pqErrMsg(err error) string {
return err.Error()
}
// generateRandomString generates a cryptographically random, alphanumeric string of length n.
func generateRandomString(n int) (string, error) {
const dictionary = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
var bytes = make([]byte, n)
if _, err := rand.Read(bytes); err != nil {
return "", err
}
for k, v := range bytes {
bytes[k] = dictionary[v%byte(len(dictionary))]
}
return string(bytes), nil
}
// assertUniqueFilename takes a file path and check if it exists on the disk. If it doesn't,
// it returns the same name and if it does, it adds a small random hash to the filename
// and returns that.
func assertUniqueFilename(dir, fileName string) string {
var (
ext = filepath.Ext(fileName)
base = fileName[0 : len(fileName)-len(ext)]
num = 0
)
for {
// There's no name conflict.
if _, err := os.Stat(filepath.Join(dir, fileName)); os.IsNotExist(err) {
return fileName
}
// Does the name match the _(num) syntax?
r := fnameRegexp.FindAllStringSubmatch(fileName, -1)
if len(r) == 1 && len(r[0]) == 3 {
num, _ = strconv.Atoi(r[0][2])
}
num++
fileName = fmt.Sprintf("%s_%d%s", base, num, ext)
}
}
// normalizeTags takes a list of string tags and normalizes them by
// lowercasing and removing all special characters except for dashes.
func normalizeTags(tags []string) []string {
@ -282,3 +207,19 @@ func parseStringIDs(s []string) ([]int64, error) {
return vals, nil
}
// generateRandomString generates a cryptographically random, alphanumeric string of length n.
func generateRandomString(n int) (string, error) {
const dictionary = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
var bytes = make([]byte, n)
if _, err := rand.Read(bytes); err != nil {
return "", err
}
for k, v := range bytes {
bytes[k] = dictionary[v%byte(len(dictionary))]
}
return string(bytes), nil
}