feat: Add blobstore package
This commit introduces a `blobstore` package and refactors the existing upload mechanism. Upload is now handled by `providers` and the two bundled providers are `S3` and `Filesystem`. `app.Blobstore` initialises the correct provider based on the configuration and handles `Put`, `Delete` and `Get` operations.
This commit is contained in:
Karan Sharma
parent
7ee71166fb
commit
e5c3196b31
|
|
@ -0,0 +1,25 @@
|
|||
**/.classpath
|
||||
**/.dockerignore
|
||||
**/.env
|
||||
**/.git
|
||||
**/.gitignore
|
||||
**/.project
|
||||
**/.settings
|
||||
**/.toolstarget
|
||||
**/.vs
|
||||
**/.vscode
|
||||
**/*.*proj.user
|
||||
**/*.dbmdl
|
||||
**/*.jfm
|
||||
**/azds.yaml
|
||||
**/bin
|
||||
**/charts
|
||||
**/docker-compose*
|
||||
**/Dockerfile*
|
||||
**/node_modules
|
||||
**/npm-debug.log
|
||||
**/obj
|
||||
**/secrets.dev.yaml
|
||||
**/values.dev.yaml
|
||||
LICENSE
|
||||
README.md
|
||||
|
|
@ -6,3 +6,4 @@ frontend/yarn.lock
|
|||
config.toml
|
||||
node_modules
|
||||
listmonk
|
||||
dist/*
|
||||
1
admin.go
1
admin.go
|
|
@ -28,7 +28,6 @@ func handleGetConfigScript(c echo.Context) error {
|
|||
app = c.Get("app").(*App)
|
||||
out = configScript{
|
||||
RootURL: app.Constants.RootURL,
|
||||
UploadURI: app.Constants.UploadURI,
|
||||
FromEmail: app.Constants.FromEmail,
|
||||
Messengers: app.Manager.GetMessengerNames(),
|
||||
}
|
||||
|
|
|
|||
|
|
@ -24,14 +24,6 @@ from_email = "listmonk <from@mail.com>"
|
|||
# To disable notifications, set an empty list, eg: notify_emails = []
|
||||
notify_emails = ["admin1@mysite.com", "admin2@mysite.com"]
|
||||
|
||||
# Path to the uploads directory where media will be uploaded.
|
||||
upload_path = "uploads"
|
||||
|
||||
# Upload URI that's visible to the outside world. The media
|
||||
# uploaded to upload_path will be made available publicly
|
||||
# under this URI, for instance, list.yoursite.com/uploads.
|
||||
upload_uri = "/uploads"
|
||||
|
||||
# Maximum concurrent workers that will attempt to send messages
|
||||
# simultaneously. This should depend on the number of CPUs the
|
||||
# machine has and also the number of simultaenous e-mails the
|
||||
|
|
@ -110,3 +102,33 @@ ssl_mode = "disable"
|
|||
|
||||
# Maximum concurrent connections to the SMTP server.
|
||||
max_conns = 10
|
||||
|
||||
# Upload settings
|
||||
[upload]
|
||||
# Provider which will be used to host uploaded media. Bundled providers are "filesystem" and "s3".
|
||||
provider = "filesystem"
|
||||
|
||||
# S3 Provider settings
|
||||
[upload.s3]
|
||||
# (Optional). AWS Access Key and Secret Key for the user to access the bucket. Leaving it empty would default to use
|
||||
# instance IAM role.
|
||||
aws_access_key_id = ""
|
||||
aws_secret_access_key = ""
|
||||
# AWS Region where S3 bucket is hosted.
|
||||
aws_default_region="ap-south-1"
|
||||
# Specify bucket name.
|
||||
bucket=""
|
||||
# Path where the files will be stored inside bucket. Empty value ("") means the root of bucket.
|
||||
bucket_path=""
|
||||
# Bucket type can be "private" or "public".
|
||||
bucket_type="public"
|
||||
# (Optional) Specify TTL (in seconds) for the generated presigned URL. Expiry value is used only if the bucket is private.
|
||||
expiry="86400"
|
||||
|
||||
# Filesystem provider settings
|
||||
[upload.filesystem]
|
||||
# Path to the uploads directory where media will be uploaded. Leaving it empty ("") means current working directory.
|
||||
upload_path=""
|
||||
# Upload URI that's visible to the outside world. The media uploaded to upload_path will be made available publicly
|
||||
# under this URI, for instance, list.yoursite.com/uploads.
|
||||
upload_uri = "/uploads"
|
||||
|
|
|
|||
4
go.mod
4
go.mod
|
|
@ -2,6 +2,7 @@ module github.com/knadh/listmonk
|
|||
|
||||
require (
|
||||
github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf
|
||||
github.com/aws/aws-sdk-go v1.25.12
|
||||
github.com/disintegration/imaging v1.5.0
|
||||
github.com/jinzhu/gorm v1.9.1
|
||||
github.com/jmoiron/sqlx v1.2.0
|
||||
|
|
@ -15,6 +16,7 @@ require (
|
|||
github.com/lib/pq v1.0.0
|
||||
github.com/mattn/go-colorable v0.0.9 // indirect
|
||||
github.com/mattn/go-isatty v0.0.4 // indirect
|
||||
github.com/rhnvrm/simples3 v0.2.4-0.20191018074503-3d5b071ef727
|
||||
github.com/satori/go.uuid v1.2.0
|
||||
github.com/spf13/pflag v1.0.3
|
||||
github.com/stretchr/objx v0.2.0 // indirect
|
||||
|
|
@ -27,3 +29,5 @@ require (
|
|||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
|
||||
gopkg.in/volatiletech/null.v6 v6.0.0-20170828023728-0bef4e07ae1b
|
||||
)
|
||||
|
||||
go 1.13
|
||||
|
|
|
|||
9
go.sum
9
go.sum
|
|
@ -2,6 +2,8 @@ github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ
|
|||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf h1:eg0MeVzsP1G42dRafH3vf+al2vQIJU0YHX+1Tw87oco=
|
||||
github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
|
||||
github.com/aws/aws-sdk-go v1.25.12 h1:a4h2FxoUJq9h+hajSE/dsRiqoOniIh6BkzhxMjkepzY=
|
||||
github.com/aws/aws-sdk-go v1.25.12/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
|
|
@ -16,6 +18,8 @@ github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
|
|||
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
|
||||
github.com/jinzhu/gorm v1.9.1 h1:lDSDtsCt5AGGSKTs8AHlSDbbgif4G4+CKJ8ETBDVHTA=
|
||||
github.com/jinzhu/gorm v1.9.1/go.mod h1:Vla75njaFJ8clLU1W44h34PjIkijhjHIYnZxMqCdxqo=
|
||||
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
|
||||
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
|
||||
github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA=
|
||||
github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
|
||||
github.com/jordan-wright/email v0.0.0-20181027021455-480bedc4908b h1:veTPVnbkOijplSJVywDYKDRPoZEN39kfuMDzzRKP0FA=
|
||||
|
|
@ -57,6 +61,10 @@ github.com/pelletier/go-toml v1.4.0 h1:u3Z1r+oOXJIkxqw34zVhyPgjBsm6X2wn21NWs/HfS
|
|||
github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/rhnvrm/simples3 v0.2.3 h1:qNXPynabu8M3F4+69fspA5aWZR8jqVV1RQtv2xc1OVk=
|
||||
github.com/rhnvrm/simples3 v0.2.3/go.mod h1:iphavgjkW1uvoIiqLUX6D42XuuI9Cr+B/63xw3gb9qA=
|
||||
github.com/rhnvrm/simples3 v0.2.4-0.20191018074503-3d5b071ef727 h1:2josYcx2gm3CT0WMqi0jBagvg50V3UMWlYN/CnBEbSI=
|
||||
github.com/rhnvrm/simples3 v0.2.4-0.20191018074503-3d5b071ef727/go.mod h1:iphavgjkW1uvoIiqLUX6D42XuuI9Cr+B/63xw3gb9qA=
|
||||
github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
|
||||
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
|
||||
github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
|
||||
|
|
@ -73,6 +81,7 @@ golang.org/x/crypto v0.0.0-20181106171534-e4dc69e5b2fd h1:VtIkGDhk0ph3t+THbvXHfM
|
|||
golang.org/x/crypto v0.0.0-20181106171534-e4dc69e5b2fd/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/image v0.0.0-20181116024801-cd38e8056d9b h1:VHyIDlv3XkfCa5/a81uzaoDkHH4rr81Z62g+xlnO8uM=
|
||||
golang.org/x/image v0.0.0-20181116024801-cd38e8056d9b/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225 h1:kNX+jCowfMYzvlSvJu5pQWEmyWFrBXJ3PBy10xKMXK8=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/sys v0.0.0-20180906133057-8cf3aee42992 h1:BH3eQWeGbwRU2+wxxuuPOdFBmaiBH81O8BugSjHeTFg=
|
||||
golang.org/x/sys v0.0.0-20180906133057-8cf3aee42992/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
|
|
|
|||
42
main.go
42
main.go
|
|
@ -19,6 +19,9 @@ import (
|
|||
"github.com/knadh/koanf/providers/file"
|
||||
"github.com/knadh/koanf/providers/posflag"
|
||||
"github.com/knadh/listmonk/manager"
|
||||
"github.com/knadh/listmonk/media"
|
||||
"github.com/knadh/listmonk/media/providers/filesystem"
|
||||
"github.com/knadh/listmonk/media/providers/s3"
|
||||
"github.com/knadh/listmonk/messenger"
|
||||
"github.com/knadh/listmonk/subimporter"
|
||||
"github.com/knadh/stuffbin"
|
||||
|
|
@ -30,8 +33,6 @@ type constants struct {
|
|||
RootURL string `koanf:"root"`
|
||||
LogoURL string `koanf:"logo_url"`
|
||||
FaviconURL string `koanf:"favicon_url"`
|
||||
UploadPath string `koanf:"upload_path"`
|
||||
UploadURI string `koanf:"upload_uri"`
|
||||
FromEmail string `koanf:"from_email"`
|
||||
NotifyEmails []string `koanf:"notify_emails"`
|
||||
Privacy privacyOptions `koanf:"privacy"`
|
||||
|
|
@ -56,6 +57,7 @@ type App struct {
|
|||
Logger *log.Logger
|
||||
NotifTpls *template.Template
|
||||
Messenger messenger.Messenger
|
||||
Media media.Store
|
||||
}
|
||||
|
||||
var (
|
||||
|
|
@ -195,6 +197,33 @@ func initMessengers(r *manager.Manager) messenger.Messenger {
|
|||
return msgr
|
||||
}
|
||||
|
||||
// initMediaStore initializes Upload manager with a custom backend.
|
||||
func initMediaStore() media.Store {
|
||||
switch provider := ko.String("upload.provider"); provider {
|
||||
case "s3":
|
||||
var opts s3.Opts
|
||||
ko.Unmarshal("upload.s3", &opts)
|
||||
uplder, err := s3.NewS3Store(opts)
|
||||
if err != nil {
|
||||
logger.Fatalf("error initializing s3 upload provider %s", err)
|
||||
}
|
||||
return uplder
|
||||
case "filesystem":
|
||||
var opts filesystem.Opts
|
||||
ko.Unmarshal("upload.filesystem", &opts)
|
||||
opts.UploadPath = filepath.Clean(opts.UploadPath)
|
||||
opts.UploadURI = filepath.Clean(opts.UploadURI)
|
||||
uplder, err := filesystem.NewDiskStore(opts)
|
||||
if err != nil {
|
||||
logger.Fatalf("error initializing filesystem upload provider %s", err)
|
||||
}
|
||||
return uplder
|
||||
default:
|
||||
logger.Fatalf("unknown provider. please select one of either filesystem or s3")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
// Connect to the DB.
|
||||
db, err := connectDB(ko.String("db.host"),
|
||||
|
|
@ -216,8 +245,6 @@ func main() {
|
|||
log.Fatalf("error loading app config: %v", err)
|
||||
}
|
||||
c.RootURL = strings.TrimRight(c.RootURL, "/")
|
||||
c.UploadURI = filepath.Clean(c.UploadURI)
|
||||
c.UploadPath = filepath.Clean(c.UploadPath)
|
||||
c.Privacy.Exportable = maps.StringSliceToLookupMap(ko.Strings("privacy.exportable"))
|
||||
|
||||
// Initialize the static file system into which all
|
||||
|
|
@ -299,6 +326,9 @@ func main() {
|
|||
// Add messengers.
|
||||
app.Messenger = initMessengers(app.Manager)
|
||||
|
||||
// Add uploader
|
||||
app.Media = initMediaStore()
|
||||
|
||||
// Initialize the workers that push out messages.
|
||||
go m.Run(time.Second * 5)
|
||||
m.SpawnWorkers()
|
||||
|
|
@ -330,7 +360,9 @@ func main() {
|
|||
fSrv := app.FS.FileServer()
|
||||
srv.GET("/public/*", echo.WrapHandler(fSrv))
|
||||
srv.GET("/frontend/*", echo.WrapHandler(fSrv))
|
||||
srv.Static(c.UploadURI, c.UploadURI)
|
||||
if ko.String("upload.provider") == "filesystem" {
|
||||
srv.Static(ko.String("upload.filesystem.upload_uri"), ko.String("upload.filesystem.upload_path"))
|
||||
}
|
||||
registerHandlers(srv)
|
||||
srv.Logger.Fatal(srv.Start(ko.String("app.address")))
|
||||
}
|
||||
|
|
|
|||
65
media.go
65
media.go
|
|
@ -3,12 +3,9 @@ package main
|
|||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
|
||||
"github.com/disintegration/imaging"
|
||||
"github.com/knadh/listmonk/models"
|
||||
"github.com/knadh/listmonk/media"
|
||||
"github.com/labstack/echo"
|
||||
uuid "github.com/satori/go.uuid"
|
||||
)
|
||||
|
|
@ -26,45 +23,64 @@ func handleUploadMedia(c echo.Context) error {
|
|||
app = c.Get("app").(*App)
|
||||
cleanUp = false
|
||||
)
|
||||
|
||||
// Upload the file.
|
||||
fName, err := uploadFile("file", app.Constants.UploadPath, "", imageMimes, c)
|
||||
file, err := c.FormFile("file")
|
||||
if err != nil {
|
||||
return echo.NewHTTPError(http.StatusBadRequest,
|
||||
fmt.Sprintf("Invalid file uploaded: %v", err))
|
||||
}
|
||||
// Validate MIME type with the list of allowed types.
|
||||
var typ = file.Header.Get("Content-type")
|
||||
ok := validateMIME(typ, imageMimes)
|
||||
if !ok {
|
||||
return echo.NewHTTPError(http.StatusBadRequest,
|
||||
fmt.Sprintf("Unsupported file type (%s) uploaded.", typ))
|
||||
}
|
||||
// Generate filename
|
||||
fName := generateFileName(file.Filename)
|
||||
// Read file contents in memory
|
||||
src, err := file.Open()
|
||||
if err != nil {
|
||||
return echo.NewHTTPError(http.StatusBadRequest,
|
||||
fmt.Sprintf("Error reading file: %s", err))
|
||||
}
|
||||
defer src.Close()
|
||||
// Upload the file.
|
||||
fName, err = app.Media.Put(fName, typ, src)
|
||||
if err != nil {
|
||||
cleanUp = true
|
||||
return echo.NewHTTPError(http.StatusInternalServerError,
|
||||
fmt.Sprintf("Error uploading file: %s", err))
|
||||
}
|
||||
path := filepath.Join(app.Constants.UploadPath, fName)
|
||||
|
||||
defer func() {
|
||||
// If any of the subroutines in this function fail,
|
||||
// the uploaded image should be removed.
|
||||
if cleanUp {
|
||||
os.Remove(path)
|
||||
app.Media.Delete(fName)
|
||||
app.Media.Delete(thumbPrefix + fName)
|
||||
}
|
||||
}()
|
||||
|
||||
// Create a thumbnail.
|
||||
src, err := imaging.Open(path)
|
||||
// Create thumbnail from file.
|
||||
thumbFile, err := createThumbnail(file)
|
||||
if err != nil {
|
||||
cleanUp = true
|
||||
return echo.NewHTTPError(http.StatusInternalServerError,
|
||||
fmt.Sprintf("Error opening image for resizing: %s", err))
|
||||
}
|
||||
|
||||
t := imaging.Resize(src, thumbnailSize, 0, imaging.Lanczos)
|
||||
if err := imaging.Save(t, fmt.Sprintf("%s/%s%s", app.Constants.UploadPath, thumbPrefix, fName)); err != nil {
|
||||
// Upload thumbnail.
|
||||
thumbfName, err := app.Media.Put(thumbPrefix+fName, typ, thumbFile)
|
||||
if err != nil {
|
||||
cleanUp = true
|
||||
return echo.NewHTTPError(http.StatusInternalServerError,
|
||||
fmt.Sprintf("Error saving thumbnail: %s", err))
|
||||
}
|
||||
|
||||
// Write to the DB.
|
||||
if _, err := app.Queries.InsertMedia.Exec(uuid.NewV4(), fName, fmt.Sprintf("%s%s", thumbPrefix, fName), 0, 0); err != nil {
|
||||
if _, err := app.Queries.InsertMedia.Exec(uuid.NewV4(), fName, thumbfName, 0, 0); err != nil {
|
||||
cleanUp = true
|
||||
return echo.NewHTTPError(http.StatusInternalServerError,
|
||||
fmt.Sprintf("Error saving uploaded file: %s", pqErrMsg(err)))
|
||||
fmt.Sprintf("Error saving uploaded file to db: %s", pqErrMsg(err)))
|
||||
}
|
||||
|
||||
return c.JSON(http.StatusOK, okResp{true})
|
||||
}
|
||||
|
||||
|
|
@ -72,7 +88,7 @@ func handleUploadMedia(c echo.Context) error {
|
|||
func handleGetMedia(c echo.Context) error {
|
||||
var (
|
||||
app = c.Get("app").(*App)
|
||||
out []models.Media
|
||||
out []media.Media
|
||||
)
|
||||
|
||||
if err := app.Queries.GetMedia.Select(&out); err != nil {
|
||||
|
|
@ -81,8 +97,8 @@ func handleGetMedia(c echo.Context) error {
|
|||
}
|
||||
|
||||
for i := 0; i < len(out); i++ {
|
||||
out[i].URI = fmt.Sprintf("%s/%s", app.Constants.UploadURI, out[i].Filename)
|
||||
out[i].ThumbURI = fmt.Sprintf("%s/%s%s", app.Constants.UploadURI, thumbPrefix, out[i].Filename)
|
||||
out[i].URI = app.Media.Get(out[i].Filename)
|
||||
out[i].ThumbURI = app.Media.Get(thumbPrefix + out[i].Filename)
|
||||
}
|
||||
|
||||
return c.JSON(http.StatusOK, okResp{out})
|
||||
|
|
@ -99,13 +115,14 @@ func handleDeleteMedia(c echo.Context) error {
|
|||
return echo.NewHTTPError(http.StatusBadRequest, "Invalid ID.")
|
||||
}
|
||||
|
||||
var m models.Media
|
||||
var m media.Media
|
||||
if err := app.Queries.DeleteMedia.Get(&m, id); err != nil {
|
||||
return echo.NewHTTPError(http.StatusInternalServerError,
|
||||
fmt.Sprintf("Error deleting media: %s", pqErrMsg(err)))
|
||||
}
|
||||
os.Remove(filepath.Join(app.Constants.UploadPath, m.Filename))
|
||||
os.Remove(filepath.Join(app.Constants.UploadPath, fmt.Sprintf("%s%s", thumbPrefix, m.Filename)))
|
||||
|
||||
app.Media.Delete(m.Filename)
|
||||
app.Media.Delete(thumbPrefix + m.Filename)
|
||||
|
||||
return c.JSON(http.StatusOK, okResp{true})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,26 @@
|
|||
package media
|
||||
|
||||
import (
|
||||
"io"
|
||||
|
||||
"gopkg.in/volatiletech/null.v6"
|
||||
)
|
||||
|
||||
// Media represents an uploaded object.
|
||||
type Media struct {
|
||||
ID int `db:"id" json:"id"`
|
||||
UUID string `db:"uuid" json:"uuid"`
|
||||
Filename string `db:"filename" json:"filename"`
|
||||
Width int `db:"width" json:"width"`
|
||||
Height int `db:"height" json:"height"`
|
||||
CreatedAt null.Time `db:"created_at" json:"created_at"`
|
||||
ThumbURI string `json:"thumb_uri"`
|
||||
URI string `json:"uri"`
|
||||
}
|
||||
|
||||
// Store represents set of methods to perform upload/delete operations.
|
||||
type Store interface {
|
||||
Put(string, string, io.ReadSeeker) (string, error)
|
||||
Delete(string) error
|
||||
Get(string) string
|
||||
}
|
||||
|
|
@ -0,0 +1,133 @@
|
|||
package filesystem
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/knadh/listmonk/media"
|
||||
)
|
||||
|
||||
const tmpFilePrefix = "listmonk"
|
||||
|
||||
// Opts represents filesystem params
|
||||
type Opts struct {
|
||||
UploadPath string `koanf:"upload_path"`
|
||||
UploadURI string `koanf:"upload_uri"`
|
||||
}
|
||||
|
||||
// Client implements `media.Store`
|
||||
type Client struct {
|
||||
opts Opts
|
||||
}
|
||||
|
||||
// NewDiskStore initialises store for Filesystem provider.
|
||||
func NewDiskStore(opts Opts) (media.Store, error) {
|
||||
return &Client{
|
||||
opts: opts,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Put accepts the filename, the content type and file object itself and stores the file in disk.
|
||||
func (e *Client) Put(filename string, cType string, src io.ReadSeeker) (string, error) {
|
||||
var out *os.File
|
||||
// There's no explicit name. Use the one posted in the HTTP request.
|
||||
if filename == "" {
|
||||
filename = strings.TrimSpace(filename)
|
||||
if filename == "" {
|
||||
filename, _ = generateRandomString(10)
|
||||
}
|
||||
}
|
||||
// Get the directory path
|
||||
dir := getDir(e.opts.UploadPath)
|
||||
filename = assertUniqueFilename(dir, filename)
|
||||
o, err := os.OpenFile(filepath.Join(dir, filename), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0664)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
out = o
|
||||
defer out.Close()
|
||||
|
||||
if _, err := io.Copy(out, src); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return filename, nil
|
||||
}
|
||||
|
||||
// Get accepts a filename and retrieves the full path from disk.
|
||||
func (e *Client) Get(name string) string {
|
||||
return fmt.Sprintf("%s/%s", e.opts.UploadURI, name)
|
||||
}
|
||||
|
||||
// Delete accepts a filename and removes it from disk.
|
||||
func (e *Client) Delete(file string) error {
|
||||
dir := getDir(e.opts.UploadPath)
|
||||
err := os.Remove(filepath.Join(dir, file))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// This matches filenames, sans extensions, of the format
|
||||
// filename_(number). The number is incremented in case
|
||||
// new file uploads conflict with existing filenames
|
||||
// on the filesystem.
|
||||
var fnameRegexp = regexp.MustCompile(`(.+?)_([0-9]+)$`)
|
||||
|
||||
// assertUniqueFilename takes a file path and check if it exists on the disk. If it doesn't,
|
||||
// it returns the same name and if it does, it adds a small random hash to the filename
|
||||
// and returns that.
|
||||
func assertUniqueFilename(dir, fileName string) string {
|
||||
var (
|
||||
ext = filepath.Ext(fileName)
|
||||
base = fileName[0 : len(fileName)-len(ext)]
|
||||
num = 0
|
||||
)
|
||||
|
||||
for {
|
||||
// There's no name conflict.
|
||||
if _, err := os.Stat(filepath.Join(dir, fileName)); os.IsNotExist(err) {
|
||||
return fileName
|
||||
}
|
||||
|
||||
// Does the name match the _(num) syntax?
|
||||
r := fnameRegexp.FindAllStringSubmatch(fileName, -1)
|
||||
if len(r) == 1 && len(r[0]) == 3 {
|
||||
num, _ = strconv.Atoi(r[0][2])
|
||||
}
|
||||
num++
|
||||
|
||||
fileName = fmt.Sprintf("%s_%d%s", base, num, ext)
|
||||
}
|
||||
}
|
||||
|
||||
// generateRandomString generates a cryptographically random, alphanumeric string of length n.
|
||||
func generateRandomString(n int) (string, error) {
|
||||
const dictionary = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
|
||||
|
||||
var bytes = make([]byte, n)
|
||||
if _, err := rand.Read(bytes); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
for k, v := range bytes {
|
||||
bytes[k] = dictionary[v%byte(len(dictionary))]
|
||||
}
|
||||
|
||||
return string(bytes), nil
|
||||
}
|
||||
|
||||
// getDir returns the current working directory path if no directory is specified,
|
||||
// else returns the directory path specified itself.
|
||||
func getDir(dir string) string {
|
||||
if dir == "" {
|
||||
dir, _ = os.Getwd()
|
||||
}
|
||||
return dir
|
||||
}
|
||||
|
|
@ -0,0 +1,108 @@
|
|||
package s3
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"github.com/knadh/listmonk/media"
|
||||
"github.com/rhnvrm/simples3"
|
||||
)
|
||||
|
||||
const amznS3PublicURL = "https://%s.s3.%s.amazonaws.com/%s"
|
||||
|
||||
// Opts represents AWS S3 specific params
|
||||
type Opts struct {
|
||||
AccessKey string `koanf:"aws_access_key_id"`
|
||||
SecretKey string `koanf:"aws_secret_access_key"`
|
||||
Region string `koanf:"aws_default_region"`
|
||||
Bucket string `koanf:"bucket"`
|
||||
BucketPath string `koanf:"bucket_path"`
|
||||
BucketType string `koanf:"bucket_type"`
|
||||
Expiry int `koanf:"expiry"`
|
||||
}
|
||||
|
||||
// Client implements `media.Store` for S3 provider
|
||||
type Client struct {
|
||||
s3 *simples3.S3
|
||||
opts Opts
|
||||
}
|
||||
|
||||
// NewS3Store initialises store for S3 provider. It takes in the AWS configuration
|
||||
// and sets up the `simples3` client to interact with AWS APIs for all bucket operations.
|
||||
func NewS3Store(opts Opts) (media.Store, error) {
|
||||
var s3svc *simples3.S3
|
||||
var err error
|
||||
if opts.Region == "" {
|
||||
return nil, errors.New("Invalid AWS Region specified. Please check `upload.s3` config")
|
||||
}
|
||||
// Use Access Key/Secret Key if specified in config.
|
||||
if opts.AccessKey != "" && opts.SecretKey != "" {
|
||||
s3svc = simples3.New(opts.Region, opts.AccessKey, opts.SecretKey)
|
||||
} else {
|
||||
// fallback to IAM role if no access key/secret key is provided.
|
||||
s3svc, err = simples3.NewUsingIAM(opts.Region)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return &Client{
|
||||
s3: s3svc,
|
||||
opts: opts,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Put takes in the filename, the content type and file object itself and uploads to S3.
|
||||
func (e *Client) Put(name string, cType string, file io.ReadSeeker) (string, error) {
|
||||
// Upload input parameters
|
||||
upParams := simples3.UploadInput{
|
||||
Bucket: e.opts.Bucket,
|
||||
ObjectKey: getBucketPath(e.opts.BucketPath, name),
|
||||
ContentType: cType,
|
||||
FileName: name,
|
||||
Body: file,
|
||||
}
|
||||
// Perform an upload.
|
||||
_, err := e.s3.FileUpload(upParams)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return name, nil
|
||||
}
|
||||
|
||||
// Get accepts the filename of the object stored and retrieves from S3.
|
||||
func (e *Client) Get(name string) string {
|
||||
// Generate a private S3 pre-signed URL if it's a private bucket.
|
||||
if e.opts.BucketType == "private" {
|
||||
url := e.s3.GeneratePresignedURL(simples3.PresignedInput{
|
||||
Bucket: e.opts.Bucket,
|
||||
ObjectKey: getBucketPath(e.opts.BucketPath, name),
|
||||
Method: "GET",
|
||||
Timestamp: time.Now(),
|
||||
ExpirySeconds: e.opts.Expiry,
|
||||
})
|
||||
return url
|
||||
}
|
||||
// Generate a public S3 URL if it's a public bucket.
|
||||
url := fmt.Sprintf(amznS3PublicURL, e.opts.Bucket, e.opts.Region, getBucketPath(e.opts.BucketPath, name))
|
||||
return url
|
||||
}
|
||||
|
||||
// Delete accepts the filename of the object and deletes from S3.
|
||||
func (e *Client) Delete(name string) error {
|
||||
err := e.s3.FileDelete(simples3.DeleteInput{
|
||||
Bucket: e.opts.Bucket,
|
||||
ObjectKey: getBucketPath(e.opts.BucketPath, name),
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
// getBucketPath constructs the key for the object stored in S3.
|
||||
// If path is empty, the key is the combination of root of S3 bucket and filename.
|
||||
func getBucketPath(path string, name string) string {
|
||||
if path == "" {
|
||||
return fmt.Sprintf("%s", name)
|
||||
}
|
||||
return fmt.Sprintf("%s/%s", path, name)
|
||||
}
|
||||
|
|
@ -174,19 +174,6 @@ type CampaignMeta struct {
|
|||
// Campaigns represents a slice of Campaigns.
|
||||
type Campaigns []Campaign
|
||||
|
||||
// Media represents an uploaded media item.
|
||||
type Media struct {
|
||||
ID int `db:"id" json:"id"`
|
||||
UUID string `db:"uuid" json:"uuid"`
|
||||
Filename string `db:"filename" json:"filename"`
|
||||
Width int `db:"width" json:"width"`
|
||||
Height int `db:"height" json:"height"`
|
||||
CreatedAt null.Time `db:"created_at" json:"created_at"`
|
||||
|
||||
ThumbURI string `json:"thumb_uri"`
|
||||
URI string `json:"uri"`
|
||||
}
|
||||
|
||||
// Template represents a reusable e-mail template.
|
||||
type Template struct {
|
||||
Base
|
||||
|
|
|
|||
157
utils.go
157
utils.go
|
|
@ -4,30 +4,22 @@ import (
|
|||
"bytes"
|
||||
"crypto/rand"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/disintegration/imaging"
|
||||
"github.com/jmoiron/sqlx"
|
||||
"github.com/knadh/goyesql"
|
||||
"github.com/labstack/echo"
|
||||
"github.com/lib/pq"
|
||||
)
|
||||
|
||||
const tmpFilePrefix = "listmonk"
|
||||
|
||||
var (
|
||||
// This matches filenames, sans extensions, of the format
|
||||
// filename_(number). The number is incremented in case
|
||||
// new file uploads conflict with existing filenames
|
||||
// on the filesystem.
|
||||
fnameRegexp = regexp.MustCompile(`(.+?)_([0-9]+)$`)
|
||||
|
||||
// This replaces all special characters
|
||||
tagRegexp = regexp.MustCompile(`[^a-z0-9\-\s]`)
|
||||
|
|
@ -95,23 +87,12 @@ func scanQueriesToStruct(obj interface{}, q goyesql.Queries, db *sqlx.DB) error
|
|||
return nil
|
||||
}
|
||||
|
||||
// uploadFile is a helper function on top of echo.Context for processing file uploads.
|
||||
// It allows copying a single file given the incoming file field name.
|
||||
// If the upload directory dir is empty, the file is copied to the system's temp directory.
|
||||
// If name is empty, the incoming file's name along with a small random hash is used.
|
||||
// When a slice of MIME types is given, the uploaded file's MIME type is validated against the list.
|
||||
func uploadFile(key string, dir, name string, mimes []string, c echo.Context) (string, error) {
|
||||
file, err := c.FormFile(key)
|
||||
if err != nil {
|
||||
return "", echo.NewHTTPError(http.StatusBadRequest,
|
||||
fmt.Sprintf("Invalid file uploaded: %v", err))
|
||||
}
|
||||
|
||||
// Check MIME type.
|
||||
// validateMIME is a helper function to validate uploaded file's MIME type
|
||||
// against the slice of MIME types is given.
|
||||
func validateMIME(typ string, mimes []string) (ok bool) {
|
||||
if len(mimes) > 0 {
|
||||
var (
|
||||
typ = file.Header.Get("Content-type")
|
||||
ok = false
|
||||
ok = false
|
||||
)
|
||||
for _, m := range mimes {
|
||||
if typ == m {
|
||||
|
|
@ -119,55 +100,42 @@ func uploadFile(key string, dir, name string, mimes []string, c echo.Context) (s
|
|||
break
|
||||
}
|
||||
}
|
||||
|
||||
if !ok {
|
||||
return "", echo.NewHTTPError(http.StatusBadRequest,
|
||||
fmt.Sprintf("Unsupported file type (%s) uploaded.", typ))
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// generateFileName appends the incoming file's name with a small random hash.
|
||||
func generateFileName(fName string) string {
|
||||
name := strings.TrimSpace(fName)
|
||||
if name == "" {
|
||||
name, _ = generateRandomString(10)
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
// createThumbnail reads the file object and returns a smaller image
|
||||
func createThumbnail(file *multipart.FileHeader) (*bytes.Reader, error) {
|
||||
src, err := file.Open()
|
||||
if err != nil {
|
||||
return "", err
|
||||
return nil, err
|
||||
}
|
||||
defer src.Close()
|
||||
|
||||
// There's no upload directory. Use a tempfile.
|
||||
var out *os.File
|
||||
if dir == "" {
|
||||
o, err := ioutil.TempFile("", tmpFilePrefix)
|
||||
if err != nil {
|
||||
return "", echo.NewHTTPError(http.StatusInternalServerError,
|
||||
fmt.Sprintf("Error copying uploaded file: %v", err))
|
||||
}
|
||||
out = o
|
||||
name = o.Name()
|
||||
} else {
|
||||
// There's no explicit name. Use the one posted in the HTTP request.
|
||||
if name == "" {
|
||||
name = strings.TrimSpace(file.Filename)
|
||||
if name == "" {
|
||||
name, _ = generateRandomString(10)
|
||||
}
|
||||
}
|
||||
name = assertUniqueFilename(dir, name)
|
||||
|
||||
o, err := os.OpenFile(filepath.Join(dir, name), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0664)
|
||||
if err != nil {
|
||||
return "", echo.NewHTTPError(http.StatusInternalServerError,
|
||||
fmt.Sprintf("Error copying uploaded file: %v", err))
|
||||
}
|
||||
|
||||
out = o
|
||||
img, err := imaging.Decode(src)
|
||||
if err != nil {
|
||||
return nil, echo.NewHTTPError(http.StatusInternalServerError,
|
||||
fmt.Sprintf("Error decoding image: %v", err))
|
||||
}
|
||||
defer out.Close()
|
||||
|
||||
if _, err = io.Copy(out, src); err != nil {
|
||||
return "", echo.NewHTTPError(http.StatusInternalServerError,
|
||||
fmt.Sprintf("Error copying uploaded file: %v", err))
|
||||
t := imaging.Resize(img, thumbnailSize, 0, imaging.Lanczos)
|
||||
// Encode the image into a byte slice as PNG.
|
||||
var buf bytes.Buffer
|
||||
err = imaging.Encode(&buf, t, imaging.PNG)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
return name, nil
|
||||
return bytes.NewReader(buf.Bytes()), nil
|
||||
}
|
||||
|
||||
// Given an error, pqErrMsg will try to return pq error details
|
||||
|
|
@ -182,49 +150,6 @@ func pqErrMsg(err error) string {
|
|||
return err.Error()
|
||||
}
|
||||
|
||||
// generateRandomString generates a cryptographically random, alphanumeric string of length n.
|
||||
func generateRandomString(n int) (string, error) {
|
||||
const dictionary = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
|
||||
|
||||
var bytes = make([]byte, n)
|
||||
if _, err := rand.Read(bytes); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
for k, v := range bytes {
|
||||
bytes[k] = dictionary[v%byte(len(dictionary))]
|
||||
}
|
||||
|
||||
return string(bytes), nil
|
||||
}
|
||||
|
||||
// assertUniqueFilename takes a file path and check if it exists on the disk. If it doesn't,
|
||||
// it returns the same name and if it does, it adds a small random hash to the filename
|
||||
// and returns that.
|
||||
func assertUniqueFilename(dir, fileName string) string {
|
||||
var (
|
||||
ext = filepath.Ext(fileName)
|
||||
base = fileName[0 : len(fileName)-len(ext)]
|
||||
num = 0
|
||||
)
|
||||
|
||||
for {
|
||||
// There's no name conflict.
|
||||
if _, err := os.Stat(filepath.Join(dir, fileName)); os.IsNotExist(err) {
|
||||
return fileName
|
||||
}
|
||||
|
||||
// Does the name match the _(num) syntax?
|
||||
r := fnameRegexp.FindAllStringSubmatch(fileName, -1)
|
||||
if len(r) == 1 && len(r[0]) == 3 {
|
||||
num, _ = strconv.Atoi(r[0][2])
|
||||
}
|
||||
num++
|
||||
|
||||
fileName = fmt.Sprintf("%s_%d%s", base, num, ext)
|
||||
}
|
||||
}
|
||||
|
||||
// normalizeTags takes a list of string tags and normalizes them by
|
||||
// lowercasing and removing all special characters except for dashes.
|
||||
func normalizeTags(tags []string) []string {
|
||||
|
|
@ -282,3 +207,19 @@ func parseStringIDs(s []string) ([]int64, error) {
|
|||
|
||||
return vals, nil
|
||||
}
|
||||
|
||||
// generateRandomString generates a cryptographically random, alphanumeric string of length n.
|
||||
func generateRandomString(n int) (string, error) {
|
||||
const dictionary = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
|
||||
|
||||
var bytes = make([]byte, n)
|
||||
if _, err := rand.Read(bytes); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
for k, v := range bytes {
|
||||
bytes[k] = dictionary[v%byte(len(dictionary))]
|
||||
}
|
||||
|
||||
return string(bytes), nil
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue