From 942eb7c3d86e925de5d9f7d0b6b40f7b8a9323fd Mon Sep 17 00:00:00 2001 From: Kailash Nadh Date: Wed, 8 Jul 2020 16:30:14 +0530 Subject: [PATCH] Add settings UI and "hot reload" support to the app. This is a major breaking change that moves away from having the entire app configuration in external TOML files to settings being in the database with a UI to update them dynamically. The app loads all config into memory (app settings, SMTP conf) on boot. "Hot" replacing them is complex and it's a fair tradeoff to instead just restart the application as it is practically instant. A new `settings` table stores arbitrary string keys with a JSONB value field which happens to support arbitrary types. After every settings update, the app gracefully releases all resources (HTTP server, DB pool, SMTP pool etc.) and restarts itself, occupying the same PID. If there are any running campaigns, the auto-restart doesn't happen and the user is prompted to invoke it manually with a one-click button once all running campaigns have been paused. --- Makefile | 2 +- admin.go | 22 +- config.toml.sample | 203 +------------- frontend/src/App.vue | 51 +++- frontend/src/api/index.js | 54 ++-- frontend/src/assets/style.scss | 34 +++ frontend/src/constants.js | 5 + frontend/src/main.js | 6 +- frontend/src/router/index.js | 6 + frontend/src/store/index.js | 2 + frontend/src/utils.js | 4 +- frontend/src/views/Forms.vue | 1 - frontend/src/views/Media.vue | 4 +- frontend/src/views/Settings.vue | 423 ++++++++++++++++++++++++++++++ go.mod | 6 +- go.sum | 45 ++++ handlers.go | 10 + init.go | 207 +++++++++++---- internal/manager/manager.go | 30 ++- internal/media/providers/s3/s3.go | 18 +- internal/messenger/emailer.go | 32 ++- internal/messenger/messenger.go | 1 + main.go | 129 ++++----- queries.go | 3 + queries.sql | 11 + schema.sql | 37 +++ settings.go | 179 +++++++++++++ 27 files changed, 1148 insertions(+), 377 deletions(-) create mode 100644 frontend/src/views/Settings.vue create mode 100644 settings.go diff --git a/Makefile b/Makefile index d8bfbf8..a8e6762 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,7 @@ deps: # Build steps. .PHONY: build build: - go build -o ${BIN} -ldflags="-s -w -X 'main.buildString=${BUILDSTR}'" + go build -o ${BIN} -ldflags="-s -w -X 'main.buildString=${BUILDSTR}'" .PHONY: build-frontend build-frontend: diff --git a/admin.go b/admin.go index 8fa6fcc..36607d7 100644 --- a/admin.go +++ b/admin.go @@ -5,6 +5,8 @@ import ( "encoding/json" "fmt" "net/http" + "syscall" + "time" "github.com/jmoiron/sqlx/types" "github.com/labstack/echo" @@ -14,7 +16,8 @@ type configScript struct { RootURL string `json:"rootURL"` FromEmail string `json:"fromEmail"` Messengers []string `json:"messengers"` - MediaProvider string `json:"media_provider"` + MediaProvider string `json:"mediaProvider"` + NeedsRestart bool `json:"needsRestart"` } // handleGetConfigScript returns general configuration as a Javascript @@ -28,11 +31,16 @@ func handleGetConfigScript(c echo.Context) error { Messengers: app.manager.GetMessengerNames(), MediaProvider: app.constants.MediaProvider, } + ) + app.Lock() + out.NeedsRestart = app.needsRestart + app.Unlock() + + var ( b = bytes.Buffer{} j = json.NewEncoder(&b) ) - b.Write([]byte(`var CONFIG = `)) _ = j.Encode(out) return c.Blob(http.StatusOK, "application/javascript", b.Bytes()) @@ -67,3 +75,13 @@ func handleGetDashboardCounts(c echo.Context) error { return c.JSON(http.StatusOK, okResp{out}) } + +// handleReloadApp restarts the app. +func handleReloadApp(c echo.Context) error { + app := c.Get("app").(*App) + go func() { + <-time.After(time.Millisecond * 500) + app.sigChan <- syscall.SIGHUP + }() + return c.JSON(http.StatusOK, okResp{true}) +} diff --git a/config.toml.sample b/config.toml.sample index 67eff67..514ca8d 100644 --- a/config.toml.sample +++ b/config.toml.sample @@ -1,199 +1,12 @@ [app] -# Interface and port where the app will run its webserver. -address = "0.0.0.0:9000" - -# Public root URL of the listmonk installation that'll be used -# in the messages for linking to images, unsubscribe page etc. -root = "https://listmonk.mysite.com" - -# (Optional) full URL to the static logo to be displayed on -# user facing view such as the unsubscription page. -# eg: https://mysite.com/images/logo.svg -logo_url = "https://listmonk.mysite.com/public/static/logo.png" - -# (Optional) full URL to the static favicon to be displayed on -# user facing view such as the unsubscription page. -# eg: https://mysite.com/images/favicon.png -favicon_url = "https://listmonk.mysite.com/public/static/favicon.png" - -# The default 'from' e-mail for outgoing e-mail campaigns. -from_email = "listmonk " - -# List of e-mail addresses to which admin notifications such as -# import updates, campaign completion, failure etc. should be sent. -# To disable notifications, set an empty list, eg: notify_emails = [] -notify_emails = ["admin1@mysite.com", "admin2@mysite.com"] - -# Maximum concurrent workers that will attempt to send messages -# simultaneously. This should ideally depend on the number of CPUs -# available, and should be based on the maximum number of messages -# a target SMTP server will accept. -concurrency = 5 - -# Maximum number of messages to be sent out per second per worker. -# If concurrency = 10 and message_rate = 10, then up to 10x10=100 messages -# may be pushed out every second. This, along with concurrency, should be -# tweaked to keep the net messages going out per second under the target -# SMTP's rate limits, if any. -message_rate = 5 - -# The number of errors (eg: SMTP timeouts while e-mailing) a running -# campaign should tolerate before it is paused for manual -# investigation or intervention. Set to 0 to never pause. -max_send_errors = 1000 - -# The number of subscribers to pull from the databse in a single iteration. -# Each iteration pulls subscribers from the database, sends messages to them, -# and then moves on to the next iteration to pull the next batch. -# This should ideally be higher than the maximum achievable throughput (concurrency * message_rate) -batch_size = 1000 - -[privacy] -# Allow subscribers to unsubscribe from all mailing lists and mark themselves -# as blacklisted? -allow_blacklist = false - -# Allow subscribers to export data recorded on them? -allow_export = false - -# Items to include in the data export. -# profile Subscriber's profile including custom attributes -# subscriptions Subscriber's subscription lists (private list names are masked) -# campaign_views Campaigns the subscriber has viewed and the view counts -# link_clicks Links that the subscriber has clicked and the click counts -exportable = ["profile", "subscriptions", "campaign_views", "link_clicks"] - -# Allow subscribers to delete themselves from the database? -# This deletes the subscriber and all their subscriptions. -# Their association to campaign views and link clicks are also -# removed while views and click counts remain (with no subscriber -# associated to them) so that stats and analytics aren't affected. -allow_wipe = false - + # Interface and port where the app will run its webserver. + address = "0.0.0.0:9000" # Database. [db] -host = "db" -port = 5432 -user = "listmonk" -password = "listmonk" -database = "listmonk" -ssl_mode = "disable" - -# Maximum active and idle connections to pool. -max_open = 50 -max_idle = 10 - -# SMTP servers. -[smtp] - [smtp.my0] - enabled = true - host = "my.smtp.server" - port = 25 - - # "cram", "plain", or "login". Empty string for no auth. - auth_protocol = "cram" - username = "xxxxx" - password = "" - - # Format to send e-mails in: html|plain|both. - email_format = "both" - - # Optional. Some SMTP servers require a FQDN in the hostname. - # By default, HELLOs go with "localhost". Set this if a custom - # hostname should be used. - hello_hostname = "" - - # Maximum concurrent connections to the SMTP server. - max_conns = 10 - - # Time to wait for new activity on a connection before closing - # it and removing it from the pool. - idle_timeout = "15s" - - # Message send / wait timeout. - wait_timeout = "5s" - - # The number of times a message should be retried if sending fails. - max_msg_retries = 2 - - # Enable STARTTLS. - tls_enabled = true - tls_skip_verify = false - - # One or more optional custom headers to be attached to all e-mails - # sent from this SMTP server. Uncomment the line to enable. - # email_headers = { "X-Sender" = "listmonk", "X-Custom-Header" = "listmonk" } - - [smtp.postal] - enabled = false - host = "my.smtp.server2" - port = 25 - - # cram or plain. - auth_protocol = "plain" - username = "xxxxx" - password = "" - - # Format to send e-mails in: html|plain|both. - email_format = "both" - - # Optional. Some SMTP servers require a FQDN in the hostname. - # By default, HELLOs go with "localhost". Set this if a custom - # hostname should be used. - hello_hostname = "" - - # Maximum concurrent connections to the SMTP server. - max_conns = 10 - - # Time to wait for new activity on a connection before closing - # it and removing it from the pool. - idle_timeout = "15s" - - # Message send / wait timeout. - wait_timeout = "5s" - - # The number of times a message should be retried if sending fails. - max_msg_retries = 2 - - # Enable STARTTLS. - tls_enabled = true - tls_skip_verify = false - -[upload] -# File storage backend. "filesystem" or "s3". -provider = "filesystem" - - [upload.s3] - # (Optional). AWS Access Key and Secret Key for the user to access the bucket. - # Leaving it empty would default to use instance IAM role. - aws_access_key_id = "" - aws_secret_access_key = "" - - # AWS Region where S3 bucket is hosted. - aws_default_region = "ap-south-1" - - # Bucket name. - bucket = "" - - # Path where the files will be stored inside bucket. Default is "/". - bucket_path = "/" - - # Optional full URL to the bucket. eg: https://files.mycustom.com - bucket_url = "" - - # "private" or "public". - bucket_type = "public" - - # (Optional) Specify TTL (in seconds) for the generated presigned URL. - # Expiry value is used only if the bucket is private. - expiry = 86400 - - [upload.filesystem] - # Path to the uploads directory where media will be uploaded. - upload_path="./uploads" - - # Upload URI that's visible to the outside world. - # The media uploaded to upload_path will be made available publicly - # under this URI, for instance, list.yoursite.com/uploads. - upload_uri = "/uploads" + host = "db" + port = 5432 + user = "listmonk" + password = "listmonk" + database = "listmonk" + ssl_mode = "disable" diff --git a/frontend/src/App.vue b/frontend/src/App.vue index 55f3be8..f61a9f4 100644 --- a/frontend/src/App.vue +++ b/frontend/src/App.vue @@ -63,9 +63,9 @@ icon="file-image-outline" label="Templates"> - + icon="cog-outline" label="Settings"> @@ -75,6 +75,18 @@
+
+
+ Settings have changed. Pause all running campaigns and restart the app + — + + Restart + +
+
+
@@ -82,8 +94,8 @@

Oops

- Can't connect to the listmonk backend.
- Make sure it is running and refresh this page. + Can't connect to the backend.
+ Make sure the server is running and refresh this page.

@@ -92,6 +104,7 @@ diff --git a/frontend/src/api/index.js b/frontend/src/api/index.js index 9c7e1d1..76b0367 100644 --- a/frontend/src/api/index.js +++ b/frontend/src/api/index.js @@ -9,22 +9,6 @@ const http = axios.create({ baseURL: process.env.BASE_URL, withCredentials: false, responseType: 'json', - // transformResponse: [ - // // Apply the defaut transformations as well. - // ...axios.defaults.transformResponse, - // (resp) => { - // if (!resp) { - // return resp; - // } - - // // There's an error message. - // if ('message' in resp && resp.message !== '') { - // return resp; - // } - - // return humps.camelizeKeys(resp.data); - // }, - // ], // Override the default serializer to switch params from becoming []id=a&[]id=b ... // in GET and DELETE requests to id=a&id=b. @@ -47,12 +31,13 @@ http.interceptors.response.use((resp) => { store.commit('setLoading', { model: resp.config.loading, status: false }); } - let data = {}; - if (resp.data && resp.data.data) { - if (typeof resp.data.data === 'object') { - data = humps.camelizeKeys(resp.data.data); - } else { - data = resp.data.data; + let data = { ...resp.data.data }; + if (!resp.config.preserveCase) { + if (resp.data && resp.data.data) { + if (typeof resp.data.data === 'object') { + // Transform field case. + data = humps.camelizeKeys(resp.data.data); + } } } @@ -75,11 +60,13 @@ http.interceptors.response.use((resp) => { msg = err.toString(); } - Toast.open({ - message: msg, - type: 'is-danger', - queue: false, - }); + if (!err.config.disableToast) { + Toast.open({ + message: msg, + type: 'is-danger', + queue: false, + }); + } return Promise.reject(err); }); @@ -88,6 +75,12 @@ http.interceptors.response.use((resp) => { // loading: modelName (set's the loading status in the global store: eg: store.loading.lists = true) // store: modelName (set's the API response in the global store. eg: store.lists: { ... } ) +// Health check endpoint that does not throw a toast. +export const getHealth = () => http.get('/api/health', + { disableToast: true }); + +export const reloadApp = () => http.post('/api/admin/reload'); + // Dashboard export const getDashboardCounts = () => http.get('/api/dashboard/counts', { loading: models.dashboard }); @@ -197,3 +190,10 @@ export const makeTemplateDefault = async (id) => http.put(`/api/templates/${id}/ export const deleteTemplate = async (id) => http.delete(`/api/templates/${id}`, { loading: models.templates }); + +// Settings. +export const getSettings = async () => http.get('/api/settings', + { loading: models.settings, preserveCase: true }); + +export const updateSettings = async (data) => http.put('/api/settings', data, + { loading: models.settings }); diff --git a/frontend/src/assets/style.scss b/frontend/src/assets/style.scss index 9ae032f..0e43896 100644 --- a/frontend/src/assets/style.scss +++ b/frontend/src/assets/style.scss @@ -77,6 +77,7 @@ section { } } + /* Two column sidebar+body layout */ #app { display: flex; @@ -126,6 +127,20 @@ section { } } +/* Global notices */ +.global-notices { + margin-bottom: 30px; +} +.notification { + padding: 10px 15px; + &.is-danger { + background: $white-ter; + color: $black; + border-left: 5px solid $red; + font-weight: bold; + } +} + /* HTML code editor */ .html-editor { position: relative; @@ -166,6 +181,11 @@ section { display: none; } +/* Toasts */ +.notices .toast { + animation: none; +} + /* Fix for button primary colour. */ .button.is-primary { background: $primary; @@ -453,6 +473,20 @@ section.campaign { } } +/* Settings */ +.settings { + .disabled { + opacity: 0.30; + } + .tab-content { + padding-top: 30px; + } + .box { + margin-bottom: 30px; + } +} + +/* C3 charting lib */ .c3 { .c3-chart-lines .c3-line { stroke-width: 2px; diff --git a/frontend/src/constants.js b/frontend/src/constants.js index 2072382..fd2cbea 100644 --- a/frontend/src/constants.js +++ b/frontend/src/constants.js @@ -1,10 +1,15 @@ export const models = Object.freeze({ + // This is the config loaded from /api/config.js directly onto the page + // via a diff --git a/go.mod b/go.mod index 1a47da3..6ef7bb4 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,5 @@ module github.com/knadh/listmonk + go 1.13 require ( @@ -7,12 +8,13 @@ require ( github.com/jaytaylor/html2text v0.0.0-20200220170450-61d9dc4d7195 github.com/jmoiron/sqlx v1.2.0 github.com/knadh/goyesql/v2 v2.1.1 - github.com/knadh/koanf v0.8.1 + github.com/knadh/koanf v0.12.0 github.com/knadh/smtppool v0.2.0 github.com/knadh/stuffbin v1.1.0 github.com/labstack/echo v3.3.10+incompatible github.com/labstack/gommon v0.3.0 // indirect github.com/lib/pq v1.3.0 + github.com/nats-io/nats-server/v2 v2.1.7 // indirect github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect github.com/olekukonko/tablewriter v0.0.4 // indirect github.com/rhnvrm/simples3 v0.5.0 @@ -21,4 +23,4 @@ require ( gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect gopkg.in/volatiletech/null.v6 v6.0.0-20170828023728-0bef4e07ae1b jaytaylor.com/html2text v0.0.0-20200220170450-61d9dc4d7195 -) \ No newline at end of file +) diff --git a/go.sum b/go.sum index e188b54..92dd450 100644 --- a/go.sum +++ b/go.sum @@ -8,6 +8,8 @@ github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44am github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/go-sql-driver/mysql v1.4.0 h1:7LxgVwFb2hIQtMm87NdgAVfXjnt4OePseqT1tKx+opk= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-yaml/yaml v2.1.0+incompatible h1:RYi2hDdss1u4YE7GwixGzWwVo47T8UQwnTLB6vQiq+o= @@ -15,6 +17,14 @@ github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaL github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE= github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/jaytaylor/html2text v0.0.0-20200220170450-61d9dc4d7195 h1:j0UEFmS7wSjAwKEIkgKBn8PRDfjcuggzr93R9wk53nQ= @@ -29,6 +39,8 @@ github.com/knadh/goyesql/v2 v2.1.1 h1:Orp5ldaxPM4ozKHfu1m7p6iolJFXDGOpF3/jyOgO6l github.com/knadh/goyesql/v2 v2.1.1/go.mod h1:pMzCA130/ZhEIoMmSmbEFXor3A2dxl5L+JllAc/l64s= github.com/knadh/koanf v0.8.1 h1:4VLACWqrkWRQIup3ooq6lOnaSbOJSNO+YVXnJn/NPZ8= github.com/knadh/koanf v0.8.1/go.mod h1:kVvmDbXnBtW49Czi4c1M+nnOWF0YSNZ8BaKvE/bCO1w= +github.com/knadh/koanf v0.12.0 h1:xQo0Y43CbzOix0tTeE+plIcfs1pTuaUI1/SsvDl2ROI= +github.com/knadh/koanf v0.12.0/go.mod h1:31bzRSM7vS5Vm9LNLo7B2Re1zhLOZT6EQKeodixBikE= github.com/knadh/smtppool v0.1.1 h1:pSi1Gc5TXOaN/Z/YiqfZbk/vd9dqzXzAfQiss0QSGQU= github.com/knadh/smtppool v0.1.1/go.mod h1:3DJHouXAgPDBz0kC50HukOsdapYSwIEfJGwuip46oCA= github.com/knadh/smtppool v0.2.0 h1:+llTWRljNIVg05MMu9TiefELTNwblexjsd1ALAPXZUs= @@ -59,12 +71,27 @@ github.com/mattn/go-sqlite3 v1.9.0 h1:pDRiWfl+++eC2FEFRy6jXmQlvp4Yh3z1MJKg4UeYM/ github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.2.2 h1:dxe5oCinTXiTIcfgmZecdCzPmAJKd46KsCWc35r0TV4= +github.com/mitchellh/mapstructure v1.2.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/nats-io/jwt v0.3.2 h1:+RB5hMpXUUA2dfxuhBTEkMOrYmM+gKIZYS1KjSostMI= +github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= +github.com/nats-io/nats-server v1.4.1 h1:Ul1oSOGNV/L8kjr4v6l2f9Yet6WY+LevH1/7cRZ/qyA= +github.com/nats-io/nats-server/v2 v2.1.7 h1:jCoQwDvRYJy3OpOTHeYfvIPLP46BMeDmH7XEJg/r42I= +github.com/nats-io/nats-server/v2 v2.1.7/go.mod h1:rbRrRE/Iv93O/rUvZ9dh4NfT0Cm9HWjW/BqOWLGgYiE= +github.com/nats-io/nats.go v1.10.0/go.mod h1:AjGArbfyR50+afOUotNX2Xs5SYHf+CoOa5HH1eEl2HE= +github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= +github.com/nats-io/nkeys v0.1.4 h1:aEsHIssIk6ETN5m2/MD8Y4B2X7FfXrBAUdkyRvbVYzA= +github.com/nats-io/nkeys v0.1.4/go.mod h1:XdZpAbhgyyODYqjTawOnIOI7VlbKSarI9Gfy1tqEu/s= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/olekukonko/tablewriter v0.0.4 h1:vHD/YYe1Wolo78koG299f7V/VAS08c6IpCLn+Ejf/w8= github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA= github.com/pelletier/go-toml v1.4.0 h1:u3Z1r+oOXJIkxqw34zVhyPgjBsm6X2wn21NWs/HfSeg= github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= +github.com/pelletier/go-toml v1.7.0 h1:7utD74fnzVc/cpcyy8sjrlFr5vYpypUixARcHIMIGuI= +github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rhnvrm/simples3 v0.5.0 h1:X+WX0hqoKScdoJAw/G3GArfZ6Ygsn8q+6MdocTMKXOw= @@ -86,22 +113,39 @@ github.com/valyala/fasttemplate v1.0.1 h1:tY9CJiPnMXf1ERmG2EyK7gNUd+c6RKGD0IfU8W github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59 h1:3zb4D3T4G8jdExgVU/95+vQXfpEPiMdCaZgmGVxjNHM= +golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8 h1:hVwzHzIUGRjiF7EcUjqNxk3NCfkPxbDKRdnNE1Rpg0U= golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65 h1:+rhAzEzT3f4JtomfC371qB+0Ola2caSKcY69NUBZrRQ= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24 h1:R8bzl0244nw47n1xKs1MUMAaTNgjavKcN/aX2Ss3+Fo= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d h1:nc5K6ox/4lTFbMVSL9WRR81ixkcwXThoiF6yf+R9scA= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= @@ -110,5 +154,6 @@ gopkg.in/volatiletech/null.v6 v6.0.0-20170828023728-0bef4e07ae1b h1:P+3+n9hUbqSD gopkg.in/volatiletech/null.v6 v6.0.0-20170828023728-0bef4e07ae1b/go.mod h1:0LRKfykySnChgQpG3Qpk+bkZFWazQ+MMfc5oldQCwnY= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= jaytaylor.com/html2text v0.0.0-20200220170450-61d9dc4d7195 h1:tj3Wzc08ekoAl8zEsLhT+5EmZ9TE/qpTTTi4oZjOPMw= jaytaylor.com/html2text v0.0.0-20200220170450-61d9dc4d7195/go.mod h1:OxvTsCwKosqQ1q7B+8FwXqg4rKZ/UG9dUW+g/VL2xH4= diff --git a/handlers.go b/handlers.go index c00a6b7..83a456d 100644 --- a/handlers.go +++ b/handlers.go @@ -37,10 +37,15 @@ var reUUID = regexp.MustCompile("^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[ // registerHandlers registers HTTP handlers. func registerHTTPHandlers(e *echo.Echo) { e.GET("/", handleIndexPage) + e.GET("/api/health", handleHealthCheck) e.GET("/api/config.js", handleGetConfigScript) e.GET("/api/dashboard/charts", handleGetDashboardCharts) e.GET("/api/dashboard/counts", handleGetDashboardCounts) + e.GET("/api/settings", handleGetSettings) + e.PUT("/api/settings", handleUpdateSettings) + e.POST("/api/admin/reload", handleReloadApp) + e.GET("/api/subscribers/:id", handleGetSubscriber) e.GET("/api/subscribers/:id/export", handleExportSubscriberData) e.POST("/api/subscribers", handleCreateSubscriber) @@ -140,6 +145,11 @@ func handleIndexPage(c echo.Context) error { return c.String(http.StatusOK, string(b)) } +// handleHealthCheck is a healthcheck endpoint that returns a 200 response. +func handleHealthCheck(c echo.Context) error { + return c.JSON(http.StatusOK, okResp{true}) +} + // validateUUID middleware validates the UUID string format for a given set of params. func validateUUID(next echo.HandlerFunc, params ...string) echo.HandlerFunc { return func(c echo.Context) error { diff --git a/init.go b/init.go index e3a3732..411a664 100644 --- a/init.go +++ b/init.go @@ -1,17 +1,25 @@ package main import ( + "encoding/json" "fmt" "html/template" "os" "path/filepath" "strings" + "syscall" + "time" "github.com/jmoiron/sqlx" + "github.com/jmoiron/sqlx/types" "github.com/knadh/goyesql/v2" goyesqlx "github.com/knadh/goyesql/v2/sqlx" "github.com/knadh/koanf" "github.com/knadh/koanf/maps" + "github.com/knadh/koanf/parsers/toml" + "github.com/knadh/koanf/providers/confmap" + "github.com/knadh/koanf/providers/file" + "github.com/knadh/koanf/providers/posflag" "github.com/knadh/listmonk/internal/manager" "github.com/knadh/listmonk/internal/media" "github.com/knadh/listmonk/internal/media/providers/filesystem" @@ -20,12 +28,74 @@ import ( "github.com/knadh/listmonk/internal/subimporter" "github.com/knadh/stuffbin" "github.com/labstack/echo" + flag "github.com/spf13/pflag" ) const ( queryFilePath = "queries.sql" ) +// constants contains static, constant config values required by the app. +type constants struct { + RootURL string `koanf:"root"` + LogoURL string `koanf:"logo_url"` + FaviconURL string `koanf:"favicon_url"` + FromEmail string `koanf:"from_email"` + NotifyEmails []string `koanf:"notify_emails"` + Privacy struct { + AllowBlacklist bool `koanf:"allow_blacklist"` + AllowExport bool `koanf:"allow_export"` + AllowWipe bool `koanf:"allow_wipe"` + Exportable map[string]bool `koanf:"-"` + } `koanf:"privacy"` + + UnsubURL string + LinkTrackURL string + ViewTrackURL string + OptinURL string + MessageURL string + + MediaProvider string +} + +func initFlags() { + f := flag.NewFlagSet("config", flag.ContinueOnError) + f.Usage = func() { + // Register --help handler. + fmt.Println(f.FlagUsages()) + os.Exit(0) + } + + // Register the commandline flags. + f.StringSlice("config", []string{"config.toml"}, + "path to one or more config files (will be merged in order)") + f.Bool("install", false, "run first time installation") + f.Bool("version", false, "current version of the build") + f.Bool("new-config", false, "generate sample config file") + f.String("static-dir", "", "(optional) path to directory with static files") + f.Bool("yes", false, "assume 'yes' to prompts, eg: during --install") + if err := f.Parse(os.Args[1:]); err != nil { + lo.Fatalf("error loading flags: %v", err) + } + + if err := ko.Load(posflag.Provider(f, ".", ko), nil); err != nil { + lo.Fatalf("error loading config: %v", err) + } +} + +// initConfigFiles loads the given config files into the koanf instance. +func initConfigFiles(files []string, ko *koanf.Koanf) { + for _, f := range files { + lo.Printf("reading config: %s", f) + if err := ko.Load(file.Provider(f), toml.Parser()); err != nil { + if os.IsNotExist(err) { + lo.Fatal("config file not found. If there isn't one yet, run --new-config to generate one.") + } + lo.Fatalf("error loadng config from file: %v.", err) + } + } +} + // initFileSystem initializes the stuffbin FileSystem to provide // access to bunded static assets to the app. func initFS(staticDir string) stuffbin.FileSystem { @@ -87,7 +157,6 @@ func initFS(staticDir string) stuffbin.FileSystem { // initDB initializes the main DB connection pool and parse and loads the app's // SQL queries into a prepared query map. func initDB() *sqlx.DB { - var dbCfg dbConf if err := ko.Unmarshal("db", &dbCfg); err != nil { lo.Fatalf("error loading db config: %v", err) @@ -98,7 +167,6 @@ func initDB() *sqlx.DB { if err != nil { lo.Fatalf("error connecting to DB: %v", err) } - return db } @@ -127,27 +195,22 @@ func initQueries(sqlFile string, db *sqlx.DB, fs stuffbin.FileSystem, prepareQue return qMap, &q } -// constants contains static, constant config values required by the app. -type constants struct { - RootURL string `koanf:"root"` - LogoURL string `koanf:"logo_url"` - FaviconURL string `koanf:"favicon_url"` - FromEmail string `koanf:"from_email"` - NotifyEmails []string `koanf:"notify_emails"` - Privacy struct { - AllowBlacklist bool `koanf:"allow_blacklist"` - AllowExport bool `koanf:"allow_export"` - AllowWipe bool `koanf:"allow_wipe"` - Exportable map[string]bool `koanf:"-"` - } `koanf:"privacy"` +// initSettings loads settings from the DB. +func initSettings(q *Queries) { + var s types.JSONText + if err := q.GetSettings.Get(&s); err != nil { + lo.Fatalf("error reading settings from DB: %s", pqErrMsg(err)) + } - UnsubURL string - LinkTrackURL string - ViewTrackURL string - OptinURL string - MessageURL string - - MediaProvider string + // Setting keys are dot separated, eg: app.favicon_url. Unflatten them into + // nested maps {app: {favicon_url}}. + var out map[string]interface{} + if err := json.Unmarshal(s, &out); err != nil { + lo.Fatalf("error unmarshalling settings from DB: %v", err) + } + if err := ko.Load(confmap.Provider(out, "."), nil); err != nil { + lo.Fatalf("error parsing settings from DB: %v", err) + } } func initConstants() *constants { @@ -159,6 +222,7 @@ func initConstants() *constants { if err := ko.Unmarshal("privacy", &c.Privacy); err != nil { lo.Fatalf("error loading app config: %v", err) } + c.RootURL = strings.TrimRight(c.RootURL, "/") c.Privacy.Exportable = maps.StringSliceToLookupMap(ko.Strings("privacy.exportable")) c.MediaProvider = ko.String("upload.provider") @@ -227,31 +291,35 @@ func initImporter(q *Queries, db *sqlx.DB, app *App) *subimporter.Importer { func initMessengers(m *manager.Manager) messenger.Messenger { var ( mapKeys = ko.MapKeys("smtp") - srv = make([]messenger.Server, 0, len(mapKeys)) + servers = make([]messenger.Server, 0, len(mapKeys)) ) + items := ko.Slices("smtp") + if len(items) == 0 { + lo.Fatalf("no SMTP servers found in config") + } + // Load the default SMTP messengers. - for _, name := range mapKeys { - if !ko.Bool(fmt.Sprintf("smtp.%s.enabled", name)) { - lo.Printf("skipped SMTP: %s", name) + for _, item := range items { + if !item.Bool("enabled") { continue } // Read the SMTP config. - s := messenger.Server{Name: name} - if err := ko.UnmarshalWithConf("smtp."+name, &s, koanf.UnmarshalConf{Tag: "json"}); err != nil { + var s messenger.Server + if err := item.UnmarshalWithConf("", &s, koanf.UnmarshalConf{Tag: "json"}); err != nil { lo.Fatalf("error loading SMTP: %v", err) } - srv = append(srv, s) - lo.Printf("loaded SMTP: %s (%s@%s)", s.Name, s.Username, s.Host) + servers = append(servers, s) + lo.Printf("loaded SMTP: %s@%s", item.String("username"), item.String("host")) } - if len(srv) == 0 { - lo.Fatalf("no SMTP servers found in config") + if len(servers) == 0 { + lo.Fatalf("no SMTP servers enabled in settings") } // Initialize the default e-mail messenger. - msgr, err := messenger.NewEmailer(srv...) + msgr, err := messenger.NewEmailer(servers...) if err != nil { lo.Fatalf("error loading e-mail messenger: %v", err) } @@ -266,28 +334,31 @@ func initMessengers(m *manager.Manager) messenger.Messenger { func initMediaStore() media.Store { switch provider := ko.String("upload.provider"); provider { case "s3": - var opts s3.Opts - ko.Unmarshal("upload.s3", &opts) - uplder, err := s3.NewS3Store(opts) + var o s3.Opts + ko.Unmarshal("upload.s3", &o) + up, err := s3.NewS3Store(o) if err != nil { lo.Fatalf("error initializing s3 upload provider %s", err) } - return uplder + lo.Println("media upload provider: s3") + return up case "filesystem": - var opts filesystem.Opts - ko.Unmarshal("upload.filesystem", &opts) - opts.RootURL = ko.String("app.root") - opts.UploadPath = filepath.Clean(opts.UploadPath) - opts.UploadURI = filepath.Clean(opts.UploadURI) - uplder, err := filesystem.NewDiskStore(opts) + var o filesystem.Opts + + ko.Unmarshal("upload.filesystem", &o) + o.RootURL = ko.String("app.root") + o.UploadPath = filepath.Clean(o.UploadPath) + o.UploadURI = filepath.Clean(o.UploadURI) + up, err := filesystem.NewDiskStore(o) if err != nil { lo.Fatalf("error initializing filesystem upload provider %s", err) } - return uplder + lo.Println("media upload provider: filesystem") + return up default: - lo.Fatalf("unknown provider. please select one of either filesystem or s3") + lo.Fatalf("unknown provider. select filesystem or s3") } return nil } @@ -312,7 +383,7 @@ func initNotifTemplates(path string, fs stuffbin.FileSystem, cs *constants) *tem } // initHTTPServer sets up and runs the app's main HTTP server and blocks forever. -func initHTTPServer(app *App) { +func initHTTPServer(app *App) *echo.Echo { // Initialize the HTTP server. var srv = echo.New() srv.HideBanner = true @@ -349,5 +420,47 @@ func initHTTPServer(app *App) { registerHTTPHandlers(srv) // Start the server. - srv.Logger.Fatal(srv.Start(ko.String("app.address"))) + go func() { + if err := srv.Start(ko.String("app.address")); err != nil { + if strings.Contains(err.Error(), "Server closed") { + lo.Println("HTTP server shut down") + } else { + lo.Fatalf("error starting HTTP server: %v", err) + } + } + }() + + return srv +} + +func awaitReload(sigChan chan os.Signal, closerWait chan bool, closer func()) chan bool { + // The blocking signal handler that main() waits on. + out := make(chan bool) + + // Respawn a new process and exit the running one. + respawn := func() { + if err := syscall.Exec(os.Args[0], os.Args, os.Environ()); err != nil { + lo.Fatalf("error spawning process: %v", err) + } + os.Exit(0) + } + + // Listen for reload signal. + go func() { + for range sigChan { + lo.Println("reloading on signal ...") + + go closer() + select { + case <-closerWait: + // Wait for the closer to finish. + respawn() + case <-time.After(time.Second * 3): + // Or timeout and force close. + respawn() + } + } + }() + + return out } diff --git a/internal/manager/manager.go b/internal/manager/manager.go index c2b08fa..eec880f 100644 --- a/internal/manager/manager.go +++ b/internal/manager/manager.go @@ -184,6 +184,13 @@ func (m *Manager) HasMessenger(id string) bool { return ok } +// HasRunningCampaigns checks if there are any active campaigns. +func (m *Manager) HasRunningCampaigns() bool { + m.campsMutex.Lock() + defer m.campsMutex.Unlock() + return len(m.camps) > 0 +} + // Run is a blocking function (that should be invoked as a goroutine) // that scans the data source at regular intervals for pending campaigns, // and queues them for processing. The process queue fetches batches of @@ -230,7 +237,11 @@ func (m *Manager) messageWorker() { for { select { // Campaign message. - case msg := <-m.campMsgQueue: + case msg, ok := <-m.campMsgQueue: + if !ok { + return + } + // Pause on hitting the message rate. if numMsg >= m.cfg.MessageRate { time.Sleep(time.Second) @@ -250,7 +261,10 @@ func (m *Manager) messageWorker() { } // Arbitrary message. - case msg := <-m.msgQueue: + case msg, ok := <-m.msgQueue: + if !ok { + return + } err := m.messengers[msg.Messenger].Push( msg.From, msg.To, msg.Subject, msg.Body, nil) if err != nil { @@ -291,6 +305,13 @@ func (m *Manager) TemplateFuncs(c *models.Campaign) template.FuncMap { } } +// Close closes and exits the campaign manager. +func (m *Manager) Close() { + close(m.subFetchQueue) + close(m.campMsgErrorQueue) + close(m.msgQueue) +} + // scanCampaigns is a blocking function that periodically scans the data source // for campaigns to process and dispatches them to the manager. func (m *Manager) scanCampaigns(tick time.Duration) { @@ -323,7 +344,10 @@ func (m *Manager) scanCampaigns(tick time.Duration) { // Aggregate errors from sending messages to check against the error threshold // after which a campaign is paused. - case e := <-m.campMsgErrorQueue: + case e, ok := <-m.campMsgErrorQueue: + if !ok { + return + } if m.cfg.MaxSendErrors < 1 { continue } diff --git a/internal/media/providers/s3/s3.go b/internal/media/providers/s3/s3.go index 9ccb0c2..eff7c7a 100644 --- a/internal/media/providers/s3/s3.go +++ b/internal/media/providers/s3/s3.go @@ -15,14 +15,14 @@ const amznS3PublicURL = "https://%s.s3.%s.amazonaws.com%s" // Opts represents AWS S3 specific params type Opts struct { - AccessKey string `koanf:"aws_access_key_id"` - SecretKey string `koanf:"aws_secret_access_key"` - Region string `koanf:"aws_default_region"` - Bucket string `koanf:"bucket"` - BucketPath string `koanf:"bucket_path"` - BucketURL string `koanf:"bucket_url"` - BucketType string `koanf:"bucket_type"` - Expiry int `koanf:"expiry"` + AccessKey string `koanf:"aws_access_key_id"` + SecretKey string `koanf:"aws_secret_access_key"` + Region string `koanf:"aws_default_region"` + Bucket string `koanf:"bucket"` + BucketPath string `koanf:"bucket_path"` + BucketURL string `koanf:"bucket_url"` + BucketType string `koanf:"bucket_type"` + Expiry time.Duration `koanf:"expiry"` } // Client implements `media.Store` for S3 provider @@ -83,7 +83,7 @@ func (c *Client) Get(name string) string { ObjectKey: makeBucketPath(c.opts.BucketPath, name), Method: "GET", Timestamp: time.Now(), - ExpirySeconds: c.opts.Expiry, + ExpirySeconds: int(c.opts.Expiry.Seconds()), }) return url } diff --git a/internal/messenger/emailer.go b/internal/messenger/emailer.go index bb40374..352bb8f 100644 --- a/internal/messenger/emailer.go +++ b/internal/messenger/emailer.go @@ -15,7 +15,6 @@ const emName = "email" // Server represents an SMTP server's credentials. type Server struct { - Name string Username string `json:"username"` Password string `json:"password"` AuthProtocol string `json:"auth_protocol"` @@ -33,16 +32,14 @@ type Server struct { // Emailer is the SMTP e-mail messenger. type Emailer struct { - servers map[string]*Server - serverNames []string - numServers int + servers []*Server } // NewEmailer creates and returns an e-mail Messenger backend. // It takes multiple SMTP configurations. func NewEmailer(servers ...Server) (*Emailer, error) { e := &Emailer{ - servers: make(map[string]*Server), + servers: make([]*Server, 0, len(servers)), } for _, srv := range servers { @@ -77,11 +74,9 @@ func NewEmailer(servers ...Server) (*Emailer, error) { } s.pool = pool - e.servers[s.Name] = &s - e.serverNames = append(e.serverNames, s.Name) + e.servers = append(e.servers, &s) } - e.numServers = len(e.serverNames) return e, nil } @@ -92,14 +87,16 @@ func (e *Emailer) Name() string { // Push pushes a message to the server. func (e *Emailer) Push(fromAddr string, toAddr []string, subject string, m []byte, atts []Attachment) error { - var key string - // If there are more than one SMTP servers, send to a random // one from the list. - if e.numServers > 1 { - key = e.serverNames[rand.Intn(e.numServers)] + var ( + ln = len(e.servers) + srv *Server + ) + if ln > 1 { + srv = e.servers[rand.Intn(ln)] } else { - key = e.serverNames[0] + srv = e.servers[0] } // Are there attachments? @@ -122,7 +119,6 @@ func (e *Emailer) Push(fromAddr string, toAddr []string, subject string, m []byt return err } - srv := e.servers[key] em := smtppool.Email{ From: fromAddr, To: toAddr, @@ -155,3 +151,11 @@ func (e *Emailer) Push(fromAddr string, toAddr []string, subject string, m []byt func (e *Emailer) Flush() error { return nil } + +// Close closes the SMTP pools. +func (e *Emailer) Close() error { + for _, s := range e.servers { + s.pool.Close() + } + return nil +} diff --git a/internal/messenger/messenger.go b/internal/messenger/messenger.go index 2b0441c..67b8e32 100644 --- a/internal/messenger/messenger.go +++ b/internal/messenger/messenger.go @@ -8,6 +8,7 @@ type Messenger interface { Name() string Push(fromAddr string, toAddr []string, subject string, message []byte, atts []Attachment) error Flush() error + Close() error } // Attachment represents a file or blob attachment that can be diff --git a/main.go b/main.go index 03ec0c8..dfa8d21 100644 --- a/main.go +++ b/main.go @@ -1,25 +1,25 @@ package main import ( + "context" "fmt" "html/template" "log" "os" + "os/signal" "strings" + "sync" + "syscall" "time" "github.com/jmoiron/sqlx" "github.com/knadh/koanf" - "github.com/knadh/koanf/parsers/toml" "github.com/knadh/koanf/providers/env" - "github.com/knadh/koanf/providers/file" - "github.com/knadh/koanf/providers/posflag" "github.com/knadh/listmonk/internal/manager" "github.com/knadh/listmonk/internal/media" "github.com/knadh/listmonk/internal/messenger" "github.com/knadh/listmonk/internal/subimporter" "github.com/knadh/stuffbin" - flag "github.com/spf13/pflag" ) // App contains the "global" components that are @@ -35,47 +35,38 @@ type App struct { media media.Store notifTpls *template.Template log *log.Logger + + // Channel for passing reload signals. + sigChan chan os.Signal + + // Global variable that stores the state indicating that a restart is required + // after a settings update. + needsRestart bool + sync.Mutex } var ( - // Global logger. lo = log.New(os.Stdout, "", log.Ldate|log.Ltime|log.Lshortfile) - - // Global configuration reader. ko = koanf.New(".") + fs stuffbin.FileSystem + db *sqlx.DB + queries *Queries + buildString string ) func init() { - f := flag.NewFlagSet("config", flag.ContinueOnError) - f.Usage = func() { - // Register --help handler. - fmt.Println(f.FlagUsages()) - os.Exit(0) - } - - // Register the commandline flags. - f.StringSlice("config", []string{"config.toml"}, - "path to one or more config files (will be merged in order)") - f.Bool("install", false, "run first time installation") - f.Bool("version", false, "current version of the build") - f.Bool("new-config", false, "generate sample config file") - f.String("static-dir", "", "(optional) path to directory with static files") - f.Bool("yes", false, "assume 'yes' to prompts, eg: during --install") - - if err := f.Parse(os.Args[1:]); err != nil { - lo.Fatalf("error loading flags: %v", err) - } + initFlags() // Display version. - if v, _ := f.GetBool("version"); v { + if ko.Bool("version") { fmt.Println(buildString) os.Exit(0) } // Generate new config. - if ok, _ := f.GetBool("new-config"); ok { + if ko.Bool("new-config") { if err := newConfigFile(); err != nil { lo.Println(err) os.Exit(1) @@ -84,38 +75,12 @@ func init() { os.Exit(0) } - // Load config files. - cFiles, _ := f.GetStringSlice("config") - for _, f := range cFiles { - lo.Printf("reading config: %s", f) - if err := ko.Load(file.Provider(f), toml.Parser()); err != nil { - if os.IsNotExist(err) { - lo.Fatal("config file not found. If there isn't one yet, run --new-config to generate one.") - } - lo.Fatalf("error loadng config from file: %v.", err) - } - } + // Load config files to pick up the database settings first. + initConfigFiles(ko.Strings("config"), ko) - // Load environment variables and merge into the loaded config. - if err := ko.Load(env.Provider("LISTMONK_", ".", func(s string) string { - return strings.Replace(strings.ToLower( - strings.TrimPrefix(s, "LISTMONK_")), "__", ".", -1) - }), nil); err != nil { - lo.Fatalf("error loading config from env: %v", err) - } - if err := ko.Load(posflag.Provider(f, ".", ko), nil); err != nil { - lo.Fatalf("error loading config: %v", err) - } -} - -func main() { - // Initialize the DB and the filesystem that are required by the installer - // and the app. - var ( - fs = initFS(ko.String("static-dir")) - db = initDB() - ) - defer db.Close() + // Connect to the database, load the filesystem to read SQL queries. + db = initDB() + fs = initFS(ko.String("static-dir")) // Installer mode? This runs before the SQL queries are loaded and prepared // as the installer needs to work on an empty DB. @@ -124,6 +89,22 @@ func main() { return } + // Load the SQL queries from the filesystem. + _, queries := initQueries(queryFilePath, db, fs, true) + + // Load settings from DB. + initSettings(queries) + + // Load environment variables and merge into the loaded config. + if err := ko.Load(env.Provider("LISTMONK_", ".", func(s string) string { + return strings.Replace(strings.ToLower( + strings.TrimPrefix(s, "LISTMONK_")), "__", ".", -1) + }), nil); err != nil { + lo.Fatalf("error loading config from env: %v", err) + } +} + +func main() { // Initialize the main app controller that wraps all of the app's // components. This is passed around HTTP handlers. app := &App{ @@ -143,6 +124,32 @@ func main() { // messages) get processed at the specified interval. go app.manager.Run(time.Second * 5) - // Start and run the app server. - initHTTPServer(app) + // Start the app server. + srv := initHTTPServer(app) + + // Wait for the reload signal with a callback to gracefully shut down resources. + // The `wait` channel is passed to awaitReload to wait for the callback to finish + // within N seconds, or do a force reload. + app.sigChan = make(chan os.Signal) + signal.Notify(app.sigChan, syscall.SIGHUP) + + closerWait := make(chan bool) + <-awaitReload(app.sigChan, closerWait, func() { + // Stop the HTTP server. + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + defer cancel() + srv.Shutdown(ctx) + + // Close the campaign manager. + app.manager.Close() + + // Close the DB pool. + app.db.DB.Close() + + // Close the messenger pool. + app.messenger.Close() + + // Signal the close. + closerWait <- true + }) } diff --git a/queries.go b/queries.go index 28cb505..dd880a2 100644 --- a/queries.go +++ b/queries.go @@ -76,6 +76,9 @@ type Queries struct { CreateLink *sqlx.Stmt `query:"create-link"` RegisterLinkClick *sqlx.Stmt `query:"register-link-click"` + GetSettings *sqlx.Stmt `query:"get-settings"` + UpdateSettings *sqlx.Stmt `query:"update-settings"` + // GetStats *sqlx.Stmt `query:"get-stats"` } diff --git a/queries.sql b/queries.sql index c1e2820..725ec10 100644 --- a/queries.sql +++ b/queries.sql @@ -724,3 +724,14 @@ SELECT JSON_BUILD_OBJECT('subscribers', JSON_BUILD_OBJECT( ) ), 'messages', (SELECT SUM(sent) AS messages FROM campaigns)); + +-- name: get-settings +SELECT JSON_OBJECT_AGG(key, value) AS settings + FROM ( + SELECT * FROM settings ORDER BY key + ) t; + +-- name: update-settings +UPDATE settings AS s SET value = c.value + -- For each key in the incoming JSON map, update the row with the key and it's value. + FROM(SELECT * FROM JSONB_EACH($1)) AS c(key, value) WHERE s.key = c.key; diff --git a/schema.sql b/schema.sql index 95a9f32..646772b 100644 --- a/schema.sql +++ b/schema.sql @@ -155,3 +155,40 @@ CREATE TABLE link_clicks ( DROP INDEX IF EXISTS idx_clicks_camp_id; CREATE INDEX idx_clicks_camp_id ON link_clicks(campaign_id); DROP INDEX IF EXISTS idx_clicks_link_id; CREATE INDEX idx_clicks_link_id ON link_clicks(link_id); DROP INDEX IF EXISTS idx_clicks_sub_id; CREATE INDEX idx_clicks_sub_id ON link_clicks(subscriber_id); + +-- settings +DROP TABLE IF EXISTS settings CASCADE; +CREATE TABLE settings ( + key TEXT NOT NULL UNIQUE, + value JSONB NOT NULL DEFAULT '{}', + updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW() +); +DROP INDEX IF EXISTS idx_settings_key; CREATE INDEX idx_settings_key ON settings(key); +INSERT INTO settings (key, value) VALUES + ('app.favicon_url', '""'), + ('app.from_email', '"listmonk "'), + ('app.logo_url', '"http://localhost:9000/public/static/logo.png"'), + ('app.concurrency', '10'), + ('app.message_rate', '10'), + ('app.batch_size', '1000'), + ('app.max_send_errors', '1000'), + ('app.notify_emails', '["admin1@mysite.com", "admin2@mysite.com"]'), + ('privacy.allow_blacklist', 'true'), + ('privacy.allow_export', 'true'), + ('privacy.allow_wipe', 'true'), + ('privacy.exportable', '["profile", "subscriptions", "campaign_views", "link_clicks"]'), + ('upload.provider', '"filesystem"'), + ('upload.filesystem.upload_path', '"uploads"'), + ('upload.filesystem.upload_uri', '"/uploads"'), + ('upload.s3.aws_access_key_id', '""'), + ('upload.s3.aws_secret_access_key', '""'), + ('upload.s3.aws_default_region', '"ap-south-b"'), + ('upload.s3.bucket', '""'), + ('upload.s3.bucket_domain', '""'), + ('upload.s3.bucket_path', '"/"'), + ('upload.s3.bucket_type', '"public"'), + ('upload.s3.expiry', '"14d"'), + ('smtp', + '[{"enabled":true, "host":"smtp.yoursite.com","port":25,"auth_protocol":"cram","username":"username","password":"password","hello_hostname":"","max_conns":10,"idle_timeout":"15s","wait_timeout":"5s","max_msg_retries":2,"tls_enabled":true,"tls_skip_verify":false,"email_headers":[]}, + {"enabled":false, "host":"smtp2.yoursite.com","port":587,"auth_protocol":"plain","username":"username","password":"password","hello_hostname":"","max_conns":10,"idle_timeout":"15s","wait_timeout":"5s","max_msg_retries":2,"tls_enabled":false,"tls_skip_verify":false,"email_headers":[]}]'), + ('messengers', '[]'); diff --git a/settings.go b/settings.go new file mode 100644 index 0000000..7369183 --- /dev/null +++ b/settings.go @@ -0,0 +1,179 @@ +package main + +import ( + "encoding/json" + "fmt" + "net/http" + "syscall" + "time" + + "github.com/jmoiron/sqlx/types" + "github.com/labstack/echo" +) + +type settings struct { + AppRootURL string `json:"app.root_url"` + AppLogoURL string `json:"app.logo_url"` + AppFaviconURL string `json:"app.favicon_url"` + AppFromEmail string `json:"app.from_email"` + AppNotifyEmails []string `json:"app.notify_emails"` + AppBatchSize int `json:"app.batch_size"` + AppConcurrency int `json:"app.concurrency"` + AppMaxSendErrors int `json:"app.max_send_errors"` + AppMessageRate int `json:"app.message_rate"` + + Messengers []interface{} `json:"messengers"` + + PrivacyAllowBlacklist bool `json:"privacy.allow_blacklist"` + PrivacyAllowExport bool `json:"privacy.allow_export"` + PrivacyAllowWipe bool `json:"privacy.allow_wipe"` + PrivacyExportable []string `json:"privacy.exportable"` + + SMTP []struct { + Enabled bool `json:"enabled"` + Host string `json:"host"` + HelloHostname string `json:"hello_hostname"` + Port int `json:"port"` + AuthProtocol string `json:"auth_protocol"` + Username string `json:"username"` + Password string `json:"password,omitempty"` + EmailHeaders []map[string]string `json:"email_headers"` + MaxConns int `json:"max_conns"` + MaxMsgRetries int `json:"max_msg_retries"` + IdleTimeout string `json:"idle_timeout"` + WaitTimeout string `json:"wait_timeout"` + TLSEnabled bool `json:"tls_enabled"` + TLSSkipVerify bool `json:"tls_skip_verify"` + } `json:"smtp"` + + UploadProvider string `json:"upload.provider"` + + UploadFilesystemUploadPath string `json:"upload.filesystem.upload_path"` + UploadFilesystemUploadURI string `json:"upload.filesystem.upload_uri"` + + UploadS3AwsAccessKeyID string `json:"upload.s3.aws_access_key_id"` + UploadS3AwsDefaultRegion string `json:"upload.s3.aws_default_region"` + UploadS3AwsSecretAccessKey string `json:"upload.s3.aws_secret_access_key,omitempty"` + UploadS3Bucket string `json:"upload.s3.bucket"` + UploadS3BucketDomain string `json:"upload.s3.bucket_domain"` + UploadS3BucketPath string `json:"upload.s3.bucket_path"` + UploadS3BucketType string `json:"upload.s3.bucket_type"` + UploadS3Expiry string `json:"upload.s3.expiry"` +} + +// handleGetSettings returns settings from the DB. +func handleGetSettings(c echo.Context) error { + app := c.Get("app").(*App) + + s, err := getSettings(app) + if err != nil { + return err + } + + // Empty out passwords. + for i := 0; i < len(s.SMTP); i++ { + s.SMTP[i].Password = "" + } + s.UploadS3AwsSecretAccessKey = "" + + return c.JSON(http.StatusOK, okResp{s}) +} + +// handleUpdateSettings returns settings from the DB. +func handleUpdateSettings(c echo.Context) error { + var ( + app = c.Get("app").(*App) + set settings + ) + + // Unmarshal and marshal the fields once to sanitize the settings blob. + if err := c.Bind(&set); err != nil { + return err + } + + // Get the existing settings. + cur, err := getSettings(app) + if err != nil { + return err + } + + // There should be at least one SMTP block that's enabled. + has := false + for i, s := range set.SMTP { + if s.Enabled { + has = true + } + + // If there's no password coming in from the frontend, attempt to get the + // last saved password for the SMTP block at the same position. + if set.SMTP[i].Password == "" { + if len(cur.SMTP) > i && + set.SMTP[i].Host == cur.SMTP[i].Host && + set.SMTP[i].Username == cur.SMTP[i].Username { + set.SMTP[i].Password = cur.SMTP[i].Password + } + } + } + if !has { + return echo.NewHTTPError(http.StatusBadRequest, + "At least one SMTP block should be enabled") + } + + // S3 password? + if set.UploadS3AwsSecretAccessKey == "" { + set.UploadS3AwsSecretAccessKey = cur.UploadS3AwsSecretAccessKey + } + + // Marshal settings. + b, err := json.Marshal(set) + if err != nil { + return echo.NewHTTPError(http.StatusInternalServerError, + fmt.Sprintf("Error encoding settings: %v", err)) + } + + // Update the settings in the DB. + if _, err := app.queries.UpdateSettings.Exec(b); err != nil { + return echo.NewHTTPError(http.StatusInternalServerError, + fmt.Sprintf("Error updating settings: %s", pqErrMsg(err))) + } + + // If there are any active campaigns, don't do an auto reload and + // warn the user on the frontend. + if app.manager.HasRunningCampaigns() { + app.Lock() + app.needsRestart = true + app.Unlock() + + return c.JSON(http.StatusOK, okResp{struct { + NeedsRestart bool `json:"needs_restart"` + }{true}}) + } + + // No running campaigns. Reload the app. + go func() { + <-time.After(time.Millisecond * 500) + app.sigChan <- syscall.SIGHUP + }() + + return c.JSON(http.StatusOK, okResp{true}) +} + +func getSettings(app *App) (settings, error) { + var ( + b types.JSONText + out settings + ) + + if err := app.queries.GetSettings.Get(&b); err != nil { + return out, echo.NewHTTPError(http.StatusInternalServerError, + fmt.Sprintf("Error fetching settings: %s", pqErrMsg(err))) + } + + // Unmarshall the settings and filter out sensitive fields. + if err := json.Unmarshal([]byte(b), &out); err != nil { + return out, echo.NewHTTPError(http.StatusInternalServerError, + fmt.Sprintf("Error parsing settings: %v", err)) + } + + return out, nil +}