Refactor and fix media uploads.

- Fix path related issues in filesystem and S3.
- Add checks for S3 "/" path prefix.
- Add support for custom S3 domain names.
- Remove obsolete `width` and `height` columns from media table (breaking)
- Add `provider` field to media table (breaking)
This commit is contained in:
Kailash Nadh 2020-07-05 17:35:05 +05:30
parent 7f9a811897
commit 24192a327f
15 changed files with 91 additions and 69 deletions

View File

@ -14,6 +14,7 @@ type configScript struct {
RootURL string `json:"rootURL"` RootURL string `json:"rootURL"`
FromEmail string `json:"fromEmail"` FromEmail string `json:"fromEmail"`
Messengers []string `json:"messengers"` Messengers []string `json:"messengers"`
MediaProvider string `json:"media_provider"`
} }
// handleGetConfigScript returns general configuration as a Javascript // handleGetConfigScript returns general configuration as a Javascript
@ -25,6 +26,7 @@ func handleGetConfigScript(c echo.Context) error {
RootURL: app.constants.RootURL, RootURL: app.constants.RootURL,
FromEmail: app.constants.FromEmail, FromEmail: app.constants.FromEmail,
Messengers: app.manager.GetMessengerNames(), Messengers: app.manager.GetMessengerNames(),
MediaProvider: app.constants.MediaProvider,
} }
b = bytes.Buffer{} b = bytes.Buffer{}

View File

@ -166,20 +166,23 @@ provider = "filesystem"
aws_secret_access_key = "" aws_secret_access_key = ""
# AWS Region where S3 bucket is hosted. # AWS Region where S3 bucket is hosted.
aws_default_region="ap-south-1" aws_default_region = "ap-south-1"
# Bucket name. # Bucket name.
bucket="" bucket = ""
# Path where the files will be stored inside bucket. Empty for root. # Path where the files will be stored inside bucket. Default is "/".
bucket_path="" bucket_path = "/"
# Optional full URL to the bucket. eg: https://files.mycustom.com
bucket_url = ""
# "private" or "public". # "private" or "public".
bucket_type="public" bucket_type = "public"
# (Optional) Specify TTL (in seconds) for the generated presigned URL. # (Optional) Specify TTL (in seconds) for the generated presigned URL.
# Expiry value is used only if the bucket is private. # Expiry value is used only if the bucket is private.
expiry="86400" expiry = 86400
[upload.filesystem] [upload.filesystem]
# Path to the uploads directory where media will be uploaded. # Path to the uploads directory where media will be uploaded.

2
frontend/README.md vendored
View File

@ -5,7 +5,7 @@ It's best if the `listmonk/frontend` directory is opened in an IDE as a separate
For developer setup instructions, refer to the main project's README. For developer setup instructions, refer to the main project's README.
## Globals ## Globals
`main.js` is where Buefy is injected globally into Vue. In addition two controllers, `$api` (collection of API calls from `api/index.js`) and `$utils` (util functions from `util.js`), are also attached globaly to Vue. They are accessible within Vue as `this.$api` and `this.$utils`. `main.js` is where Buefy is injected globally into Vue. In addition two controllers, `$api` (collection of API calls from `api/index.js`), `$utils` (util functions from `util.js`), `$serverConfig` (loaded form /api/config.js) are also attached globaly to Vue. They are accessible within Vue as `this.$api` and `this.$utils`.
Some constants are defined in `constants.js`. Some constants are defined in `constants.js`.

View File

@ -22,8 +22,7 @@ const http = axios.create({
return resp; return resp;
} }
const data = humps.camelizeKeys(resp.data); return humps.camelizeKeys(resp.data);
return data;
}, },
], ],

View File

@ -164,7 +164,7 @@ export default {
}, },
onMediaSelect(m) { onMediaSelect(m) {
this.$refs.quill.quill.insertEmbed(10, 'image', m.uri); this.$refs.quill.quill.insertEmbed(10, 'image', m.url);
}, },
}, },

View File

@ -1,5 +1,6 @@
import Vue from 'vue'; import Vue from 'vue';
import Buefy from 'buefy'; import Buefy from 'buefy';
import humps from 'humps';
import App from './App.vue'; import App from './App.vue';
import router from './router'; import router from './router';
@ -14,6 +15,9 @@ Vue.config.productionTip = false;
Vue.prototype.$api = api; Vue.prototype.$api = api;
Vue.prototype.$utils = utils; Vue.prototype.$utils = utils;
// window.CONFIG is loaded from /api/config.js directly in a <script> tag.
Vue.prototype.$serverConfig = humps.camelizeKeys(window.CONFIG);
new Vue({ new Vue({
router, router,
store, store,

View File

@ -91,6 +91,7 @@ export default class utils {
message: msg, message: msg,
type: !typ ? 'is-success' : typ, type: !typ ? 'is-success' : typ,
queue: false, queue: false,
duration: 3000,
}); });
}; };
} }

View File

@ -2,6 +2,8 @@
<section class="media-files"> <section class="media-files">
<h1 class="title is-4">Media <h1 class="title is-4">Media
<span v-if="media.length > 0">({{ media.length }})</span> <span v-if="media.length > 0">({{ media.length }})</span>
<span class="has-text-grey-light"> / {{ $serverConfig.mediaProvider }}</span>
</h1> </h1>
<b-loading :active="isProcessing || loading.media"></b-loading> <b-loading :active="isProcessing || loading.media"></b-loading>
@ -45,16 +47,16 @@
<div class="thumbs"> <div class="thumbs">
<div v-for="m in group.items" :key="m.id" class="box thumb"> <div v-for="m in group.items" :key="m.id" class="box thumb">
<a @click="(e) => onMediaSelect(m, e)" :href="m.uri" target="_blank"> <a @click="(e) => onMediaSelect(m, e)" :href="m.url" target="_blank">
<img :src="m.thumbUri" :title="m.filename" /> <img :src="m.thumbUrl" :title="m.filename" />
</a> </a>
<span class="caption is-size-7" :title="m.filename">{{ m.filename }}</span> <span class="caption is-size-7" :title="m.filename">{{ m.filename }}</span>
<div class="actions has-text-right"> <div class="actions has-text-right">
<a :href="m.uri" target="_blank"> <a :href="m.url" target="_blank">
<b-icon icon="arrow-top-right" size="is-small" /> <b-icon icon="arrow-top-right" size="is-small" />
</a> </a>
<a href="#" @click.prevent="deleteMedia(m.id)"> <a href="#" @click.prevent="$utils.confirm(null, () => deleteMedia(m.id))">
<b-icon icon="trash-can-outline" size="is-small" /> <b-icon icon="trash-can-outline" size="is-small" />
</a> </a>
</div> </div>

View File

@ -146,6 +146,8 @@ type constants struct {
ViewTrackURL string ViewTrackURL string
OptinURL string OptinURL string
MessageURL string MessageURL string
MediaProvider string
} }
func initConstants() *constants { func initConstants() *constants {
@ -159,6 +161,7 @@ func initConstants() *constants {
} }
c.RootURL = strings.TrimRight(c.RootURL, "/") c.RootURL = strings.TrimRight(c.RootURL, "/")
c.Privacy.Exportable = maps.StringSliceToLookupMap(ko.Strings("privacy.exportable")) c.Privacy.Exportable = maps.StringSliceToLookupMap(ko.Strings("privacy.exportable"))
c.MediaProvider = ko.String("upload.provider")
// Static URLS. // Static URLS.
// url.com/subscription/{campaign_uuid}/{subscriber_uuid} // url.com/subscription/{campaign_uuid}/{subscriber_uuid}
@ -175,7 +178,6 @@ func initConstants() *constants {
// url.com/campaign/{campaign_uuid}/{subscriber_uuid}/px.png // url.com/campaign/{campaign_uuid}/{subscriber_uuid}/px.png
c.ViewTrackURL = fmt.Sprintf("%s/campaign/%%s/%%s/px.png", c.RootURL) c.ViewTrackURL = fmt.Sprintf("%s/campaign/%%s/%%s/px.png", c.RootURL)
return &c return &c
} }
@ -272,6 +274,7 @@ func initMediaStore() media.Store {
case "filesystem": case "filesystem":
var opts filesystem.Opts var opts filesystem.Opts
ko.Unmarshal("upload.filesystem", &opts) ko.Unmarshal("upload.filesystem", &opts)
opts.RootURL = ko.String("app.root")
opts.UploadPath = filepath.Clean(opts.UploadPath) opts.UploadPath = filepath.Clean(opts.UploadPath)
opts.UploadURI = filepath.Clean(opts.UploadURI) opts.UploadURI = filepath.Clean(opts.UploadURI)
uplder, err := filesystem.NewDiskStore(opts) uplder, err := filesystem.NewDiskStore(opts)

View File

@ -11,14 +11,14 @@ type Media struct {
ID int `db:"id" json:"id"` ID int `db:"id" json:"id"`
UUID string `db:"uuid" json:"uuid"` UUID string `db:"uuid" json:"uuid"`
Filename string `db:"filename" json:"filename"` Filename string `db:"filename" json:"filename"`
Width int `db:"width" json:"width"` Thumb string `db:"thumb" json:"thumb"`
Height int `db:"height" json:"height"`
CreatedAt null.Time `db:"created_at" json:"created_at"` CreatedAt null.Time `db:"created_at" json:"created_at"`
ThumbURI string `json:"thumb_uri"` ThumbURL string `json:"thumb_url"`
URI string `json:"uri"` Provider string `json:"provider"`
URL string `json:"url"`
} }
// Store represents set of methods to perform upload/delete operations. // Store represents functions to store and retrieve media (files).
type Store interface { type Store interface {
Put(string, string, io.ReadSeeker) (string, error) Put(string, string, io.ReadSeeker) (string, error)
Delete(string) error Delete(string) error

View File

@ -19,6 +19,7 @@ const tmpFilePrefix = "listmonk"
type Opts struct { type Opts struct {
UploadPath string `koanf:"upload_path"` UploadPath string `koanf:"upload_path"`
UploadURI string `koanf:"upload_uri"` UploadURI string `koanf:"upload_uri"`
RootURL string `koanf:"root_url"`
} }
// Client implements `media.Store` // Client implements `media.Store`
@ -26,6 +27,12 @@ type Client struct {
opts Opts opts Opts
} }
// This matches filenames, sans extensions, of the format
// filename_(number). The number is incremented in case
// new file uploads conflict with existing filenames
// on the filesystem.
var fnameRegexp = regexp.MustCompile(`(.+?)_([0-9]+)$`)
// NewDiskStore initialises store for Filesystem provider. // NewDiskStore initialises store for Filesystem provider.
func NewDiskStore(opts Opts) (media.Store, error) { func NewDiskStore(opts Opts) (media.Store, error) {
return &Client{ return &Client{
@ -34,7 +41,7 @@ func NewDiskStore(opts Opts) (media.Store, error) {
} }
// Put accepts the filename, the content type and file object itself and stores the file in disk. // Put accepts the filename, the content type and file object itself and stores the file in disk.
func (e *Client) Put(filename string, cType string, src io.ReadSeeker) (string, error) { func (c *Client) Put(filename string, cType string, src io.ReadSeeker) (string, error) {
var out *os.File var out *os.File
// There's no explicit name. Use the one posted in the HTTP request. // There's no explicit name. Use the one posted in the HTTP request.
if filename == "" { if filename == "" {
@ -44,7 +51,7 @@ func (e *Client) Put(filename string, cType string, src io.ReadSeeker) (string,
} }
} }
// Get the directory path // Get the directory path
dir := getDir(e.opts.UploadPath) dir := getDir(c.opts.UploadPath)
filename = assertUniqueFilename(dir, filename) filename = assertUniqueFilename(dir, filename)
o, err := os.OpenFile(filepath.Join(dir, filename), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0664) o, err := os.OpenFile(filepath.Join(dir, filename), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0664)
if err != nil { if err != nil {
@ -60,13 +67,13 @@ func (e *Client) Put(filename string, cType string, src io.ReadSeeker) (string,
} }
// Get accepts a filename and retrieves the full path from disk. // Get accepts a filename and retrieves the full path from disk.
func (e *Client) Get(name string) string { func (c *Client) Get(name string) string {
return fmt.Sprintf("%s/%s", e.opts.UploadURI, name) return fmt.Sprintf("%s%s/%s", c.opts.RootURL, c.opts.UploadURI, name)
} }
// Delete accepts a filename and removes it from disk. // Delete accepts a filename and removes it from disk.
func (e *Client) Delete(file string) error { func (c *Client) Delete(file string) error {
dir := getDir(e.opts.UploadPath) dir := getDir(c.opts.UploadPath)
err := os.Remove(filepath.Join(dir, file)) err := os.Remove(filepath.Join(dir, file))
if err != nil { if err != nil {
return err return err
@ -74,12 +81,6 @@ func (e *Client) Delete(file string) error {
return nil return nil
} }
// This matches filenames, sans extensions, of the format
// filename_(number). The number is incremented in case
// new file uploads conflict with existing filenames
// on the filesystem.
var fnameRegexp = regexp.MustCompile(`(.+?)_([0-9]+)$`)
// assertUniqueFilename takes a file path and check if it exists on the disk. If it doesn't, // assertUniqueFilename takes a file path and check if it exists on the disk. If it doesn't,
// it returns the same name and if it does, it adds a small random hash to the filename // it returns the same name and if it does, it adds a small random hash to the filename
// and returns that. // and returns that.

View File

@ -4,13 +4,14 @@ import (
"errors" "errors"
"fmt" "fmt"
"io" "io"
"strings"
"time" "time"
"github.com/knadh/listmonk/internal/media" "github.com/knadh/listmonk/internal/media"
"github.com/rhnvrm/simples3" "github.com/rhnvrm/simples3"
) )
const amznS3PublicURL = "https://%s.s3.%s.amazonaws.com/%s" const amznS3PublicURL = "https://%s.s3.%s.amazonaws.com%s"
// Opts represents AWS S3 specific params // Opts represents AWS S3 specific params
type Opts struct { type Opts struct {
@ -19,6 +20,7 @@ type Opts struct {
Region string `koanf:"aws_default_region"` Region string `koanf:"aws_default_region"`
Bucket string `koanf:"bucket"` Bucket string `koanf:"bucket"`
BucketPath string `koanf:"bucket_path"` BucketPath string `koanf:"bucket_path"`
BucketURL string `koanf:"bucket_url"`
BucketType string `koanf:"bucket_type"` BucketType string `koanf:"bucket_type"`
Expiry int `koanf:"expiry"` Expiry int `koanf:"expiry"`
} }
@ -54,55 +56,61 @@ func NewS3Store(opts Opts) (media.Store, error) {
} }
// Put takes in the filename, the content type and file object itself and uploads to S3. // Put takes in the filename, the content type and file object itself and uploads to S3.
func (e *Client) Put(name string, cType string, file io.ReadSeeker) (string, error) { func (c *Client) Put(name string, cType string, file io.ReadSeeker) (string, error) {
// Upload input parameters // Upload input parameters
upParams := simples3.UploadInput{ upParams := simples3.UploadInput{
Bucket: e.opts.Bucket, Bucket: c.opts.Bucket,
ObjectKey: getBucketPath(e.opts.BucketPath, name),
ContentType: cType, ContentType: cType,
FileName: name, FileName: name,
Body: file, Body: file,
// Paths inside the bucket should not start with /.
ObjectKey: strings.TrimPrefix(makeBucketPath(c.opts.BucketPath, name), "/"),
} }
// Perform an upload. // Perform an upload.
_, err := e.s3.FileUpload(upParams) if _, err := c.s3.FileUpload(upParams); err != nil {
if err != nil {
return "", err return "", err
} }
return name, nil return name, nil
} }
// Get accepts the filename of the object stored and retrieves from S3. // Get accepts the filename of the object stored and retrieves from S3.
func (e *Client) Get(name string) string { func (c *Client) Get(name string) string {
// Generate a private S3 pre-signed URL if it's a private bucket. // Generate a private S3 pre-signed URL if it's a private bucket.
if e.opts.BucketType == "private" { if c.opts.BucketType == "private" {
url := e.s3.GeneratePresignedURL(simples3.PresignedInput{ url := c.s3.GeneratePresignedURL(simples3.PresignedInput{
Bucket: e.opts.Bucket, Bucket: c.opts.Bucket,
ObjectKey: getBucketPath(e.opts.BucketPath, name), ObjectKey: makeBucketPath(c.opts.BucketPath, name),
Method: "GET", Method: "GET",
Timestamp: time.Now(), Timestamp: time.Now(),
ExpirySeconds: e.opts.Expiry, ExpirySeconds: c.opts.Expiry,
}) })
return url return url
} }
// Generate a public S3 URL if it's a public bucket. // Generate a public S3 URL if it's a public bucket.
url := fmt.Sprintf(amznS3PublicURL, e.opts.Bucket, e.opts.Region, getBucketPath(e.opts.BucketPath, name)) url := ""
if c.opts.BucketURL != "" {
url = c.opts.BucketURL + makeBucketPath(c.opts.BucketPath, name)
} else {
url = fmt.Sprintf(amznS3PublicURL, c.opts.Bucket, c.opts.Region,
makeBucketPath(c.opts.BucketPath, name))
}
return url return url
} }
// Delete accepts the filename of the object and deletes from S3. // Delete accepts the filename of the object and deletes from S3.
func (e *Client) Delete(name string) error { func (c *Client) Delete(name string) error {
err := e.s3.FileDelete(simples3.DeleteInput{ err := c.s3.FileDelete(simples3.DeleteInput{
Bucket: e.opts.Bucket, Bucket: c.opts.Bucket,
ObjectKey: getBucketPath(e.opts.BucketPath, name), ObjectKey: strings.TrimPrefix(makeBucketPath(c.opts.BucketPath, name), "/"),
}) })
return err return err
} }
// getBucketPath constructs the key for the object stored in S3. func makeBucketPath(bucketPath string, name string) string {
// If path is empty, the key is the combination of root of S3 bucket and filename. if bucketPath == "/" {
func getBucketPath(path string, name string) string { return "/" + name
if path == "" {
return fmt.Sprintf("%s", name)
} }
return fmt.Sprintf("%s/%s", path, name) return fmt.Sprintf("%s/%s", bucketPath, name)
} }

View File

@ -99,7 +99,7 @@ func handleUploadMedia(c echo.Context) error {
} }
// Write to the DB. // Write to the DB.
if _, err := app.queries.InsertMedia.Exec(uu, fName, thumbfName, 0, 0); err != nil { if _, err := app.queries.InsertMedia.Exec(uu, fName, thumbfName, app.constants.MediaProvider); err != nil {
cleanUp = true cleanUp = true
app.log.Printf("error inserting uploaded file to db: %v", err) app.log.Printf("error inserting uploaded file to db: %v", err)
return echo.NewHTTPError(http.StatusInternalServerError, return echo.NewHTTPError(http.StatusInternalServerError,
@ -115,14 +115,14 @@ func handleGetMedia(c echo.Context) error {
out = []media.Media{} out = []media.Media{}
) )
if err := app.queries.GetMedia.Select(&out); err != nil { if err := app.queries.GetMedia.Select(&out, app.constants.MediaProvider); err != nil {
return echo.NewHTTPError(http.StatusInternalServerError, return echo.NewHTTPError(http.StatusInternalServerError,
fmt.Sprintf("Error fetching media list: %s", pqErrMsg(err))) fmt.Sprintf("Error fetching media list: %s", pqErrMsg(err)))
} }
for i := 0; i < len(out); i++ { for i := 0; i < len(out); i++ {
out[i].URI = app.media.Get(out[i].Filename) out[i].URL = app.media.Get(out[i].Filename)
out[i].ThumbURI = app.media.Get(thumbPrefix + out[i].Filename) out[i].ThumbURL = app.media.Get(out[i].Thumb)
} }
return c.JSON(http.StatusOK, okResp{out}) return c.JSON(http.StatusOK, okResp{out})

View File

@ -654,10 +654,10 @@ UPDATE campaigns SET template_id = (SELECT id FROM def) WHERE (SELECT id FROM tp
-- media -- media
-- name: insert-media -- name: insert-media
INSERT INTO media (uuid, filename, thumb, width, height, created_at) VALUES($1, $2, $3, $4, $5, NOW()); INSERT INTO media (uuid, filename, thumb, provider, created_at) VALUES($1, $2, $3, $4, NOW());
-- name: get-media -- name: get-media
SELECT * FROM media ORDER BY created_at DESC; SELECT * FROM media WHERE provider=$1 ORDER BY created_at DESC;
-- name: delete-media -- name: delete-media
DELETE FROM media WHERE id=$1 RETURNING filename; DELETE FROM media WHERE id=$1 RETURNING filename;

View File

@ -128,10 +128,9 @@ DROP TABLE IF EXISTS media CASCADE;
CREATE TABLE media ( CREATE TABLE media (
id SERIAL PRIMARY KEY, id SERIAL PRIMARY KEY,
uuid uuid NOT NULL UNIQUE, uuid uuid NOT NULL UNIQUE,
provider TEXT NOT NULL,
filename TEXT NOT NULL, filename TEXT NOT NULL,
thumb TEXT NOT NULL, thumb TEXT NOT NULL,
width INT NOT NULL DEFAULT 0,
height INT NOT NULL DEFAULT 0,
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW() created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
); );