diff --git a/Makefile b/Makefile index d8bfbf8..a8e6762 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,7 @@ deps: # Build steps. .PHONY: build build: - go build -o ${BIN} -ldflags="-s -w -X 'main.buildString=${BUILDSTR}'" + go build -o ${BIN} -ldflags="-s -w -X 'main.buildString=${BUILDSTR}'" .PHONY: build-frontend build-frontend: diff --git a/admin.go b/admin.go index 8fa6fcc..36607d7 100644 --- a/admin.go +++ b/admin.go @@ -5,6 +5,8 @@ import ( "encoding/json" "fmt" "net/http" + "syscall" + "time" "github.com/jmoiron/sqlx/types" "github.com/labstack/echo" @@ -14,7 +16,8 @@ type configScript struct { RootURL string `json:"rootURL"` FromEmail string `json:"fromEmail"` Messengers []string `json:"messengers"` - MediaProvider string `json:"media_provider"` + MediaProvider string `json:"mediaProvider"` + NeedsRestart bool `json:"needsRestart"` } // handleGetConfigScript returns general configuration as a Javascript @@ -28,11 +31,16 @@ func handleGetConfigScript(c echo.Context) error { Messengers: app.manager.GetMessengerNames(), MediaProvider: app.constants.MediaProvider, } + ) + app.Lock() + out.NeedsRestart = app.needsRestart + app.Unlock() + + var ( b = bytes.Buffer{} j = json.NewEncoder(&b) ) - b.Write([]byte(`var CONFIG = `)) _ = j.Encode(out) return c.Blob(http.StatusOK, "application/javascript", b.Bytes()) @@ -67,3 +75,13 @@ func handleGetDashboardCounts(c echo.Context) error { return c.JSON(http.StatusOK, okResp{out}) } + +// handleReloadApp restarts the app. +func handleReloadApp(c echo.Context) error { + app := c.Get("app").(*App) + go func() { + <-time.After(time.Millisecond * 500) + app.sigChan <- syscall.SIGHUP + }() + return c.JSON(http.StatusOK, okResp{true}) +} diff --git a/config.toml.sample b/config.toml.sample index 67eff67..514ca8d 100644 --- a/config.toml.sample +++ b/config.toml.sample @@ -1,199 +1,12 @@ [app] -# Interface and port where the app will run its webserver. -address = "0.0.0.0:9000" - -# Public root URL of the listmonk installation that'll be used -# in the messages for linking to images, unsubscribe page etc. -root = "https://listmonk.mysite.com" - -# (Optional) full URL to the static logo to be displayed on -# user facing view such as the unsubscription page. -# eg: https://mysite.com/images/logo.svg -logo_url = "https://listmonk.mysite.com/public/static/logo.png" - -# (Optional) full URL to the static favicon to be displayed on -# user facing view such as the unsubscription page. -# eg: https://mysite.com/images/favicon.png -favicon_url = "https://listmonk.mysite.com/public/static/favicon.png" - -# The default 'from' e-mail for outgoing e-mail campaigns. -from_email = "listmonk " - -# List of e-mail addresses to which admin notifications such as -# import updates, campaign completion, failure etc. should be sent. -# To disable notifications, set an empty list, eg: notify_emails = [] -notify_emails = ["admin1@mysite.com", "admin2@mysite.com"] - -# Maximum concurrent workers that will attempt to send messages -# simultaneously. This should ideally depend on the number of CPUs -# available, and should be based on the maximum number of messages -# a target SMTP server will accept. -concurrency = 5 - -# Maximum number of messages to be sent out per second per worker. -# If concurrency = 10 and message_rate = 10, then up to 10x10=100 messages -# may be pushed out every second. This, along with concurrency, should be -# tweaked to keep the net messages going out per second under the target -# SMTP's rate limits, if any. -message_rate = 5 - -# The number of errors (eg: SMTP timeouts while e-mailing) a running -# campaign should tolerate before it is paused for manual -# investigation or intervention. Set to 0 to never pause. -max_send_errors = 1000 - -# The number of subscribers to pull from the databse in a single iteration. -# Each iteration pulls subscribers from the database, sends messages to them, -# and then moves on to the next iteration to pull the next batch. -# This should ideally be higher than the maximum achievable throughput (concurrency * message_rate) -batch_size = 1000 - -[privacy] -# Allow subscribers to unsubscribe from all mailing lists and mark themselves -# as blacklisted? -allow_blacklist = false - -# Allow subscribers to export data recorded on them? -allow_export = false - -# Items to include in the data export. -# profile Subscriber's profile including custom attributes -# subscriptions Subscriber's subscription lists (private list names are masked) -# campaign_views Campaigns the subscriber has viewed and the view counts -# link_clicks Links that the subscriber has clicked and the click counts -exportable = ["profile", "subscriptions", "campaign_views", "link_clicks"] - -# Allow subscribers to delete themselves from the database? -# This deletes the subscriber and all their subscriptions. -# Their association to campaign views and link clicks are also -# removed while views and click counts remain (with no subscriber -# associated to them) so that stats and analytics aren't affected. -allow_wipe = false - + # Interface and port where the app will run its webserver. + address = "0.0.0.0:9000" # Database. [db] -host = "db" -port = 5432 -user = "listmonk" -password = "listmonk" -database = "listmonk" -ssl_mode = "disable" - -# Maximum active and idle connections to pool. -max_open = 50 -max_idle = 10 - -# SMTP servers. -[smtp] - [smtp.my0] - enabled = true - host = "my.smtp.server" - port = 25 - - # "cram", "plain", or "login". Empty string for no auth. - auth_protocol = "cram" - username = "xxxxx" - password = "" - - # Format to send e-mails in: html|plain|both. - email_format = "both" - - # Optional. Some SMTP servers require a FQDN in the hostname. - # By default, HELLOs go with "localhost". Set this if a custom - # hostname should be used. - hello_hostname = "" - - # Maximum concurrent connections to the SMTP server. - max_conns = 10 - - # Time to wait for new activity on a connection before closing - # it and removing it from the pool. - idle_timeout = "15s" - - # Message send / wait timeout. - wait_timeout = "5s" - - # The number of times a message should be retried if sending fails. - max_msg_retries = 2 - - # Enable STARTTLS. - tls_enabled = true - tls_skip_verify = false - - # One or more optional custom headers to be attached to all e-mails - # sent from this SMTP server. Uncomment the line to enable. - # email_headers = { "X-Sender" = "listmonk", "X-Custom-Header" = "listmonk" } - - [smtp.postal] - enabled = false - host = "my.smtp.server2" - port = 25 - - # cram or plain. - auth_protocol = "plain" - username = "xxxxx" - password = "" - - # Format to send e-mails in: html|plain|both. - email_format = "both" - - # Optional. Some SMTP servers require a FQDN in the hostname. - # By default, HELLOs go with "localhost". Set this if a custom - # hostname should be used. - hello_hostname = "" - - # Maximum concurrent connections to the SMTP server. - max_conns = 10 - - # Time to wait for new activity on a connection before closing - # it and removing it from the pool. - idle_timeout = "15s" - - # Message send / wait timeout. - wait_timeout = "5s" - - # The number of times a message should be retried if sending fails. - max_msg_retries = 2 - - # Enable STARTTLS. - tls_enabled = true - tls_skip_verify = false - -[upload] -# File storage backend. "filesystem" or "s3". -provider = "filesystem" - - [upload.s3] - # (Optional). AWS Access Key and Secret Key for the user to access the bucket. - # Leaving it empty would default to use instance IAM role. - aws_access_key_id = "" - aws_secret_access_key = "" - - # AWS Region where S3 bucket is hosted. - aws_default_region = "ap-south-1" - - # Bucket name. - bucket = "" - - # Path where the files will be stored inside bucket. Default is "/". - bucket_path = "/" - - # Optional full URL to the bucket. eg: https://files.mycustom.com - bucket_url = "" - - # "private" or "public". - bucket_type = "public" - - # (Optional) Specify TTL (in seconds) for the generated presigned URL. - # Expiry value is used only if the bucket is private. - expiry = 86400 - - [upload.filesystem] - # Path to the uploads directory where media will be uploaded. - upload_path="./uploads" - - # Upload URI that's visible to the outside world. - # The media uploaded to upload_path will be made available publicly - # under this URI, for instance, list.yoursite.com/uploads. - upload_uri = "/uploads" + host = "db" + port = 5432 + user = "listmonk" + password = "listmonk" + database = "listmonk" + ssl_mode = "disable" diff --git a/frontend/src/App.vue b/frontend/src/App.vue index 55f3be8..f61a9f4 100644 --- a/frontend/src/App.vue +++ b/frontend/src/App.vue @@ -63,9 +63,9 @@ icon="file-image-outline" label="Templates"> - + icon="cog-outline" label="Settings"> @@ -75,6 +75,18 @@
+
+
+ Settings have changed. Pause all running campaigns and restart the app + — + + Restart + +
+
+
@@ -82,8 +94,8 @@

Oops

- Can't connect to the listmonk backend.
- Make sure it is running and refresh this page. + Can't connect to the backend.
+ Make sure the server is running and refresh this page.

@@ -92,6 +104,7 @@ diff --git a/frontend/src/api/index.js b/frontend/src/api/index.js index 9c7e1d1..76b0367 100644 --- a/frontend/src/api/index.js +++ b/frontend/src/api/index.js @@ -9,22 +9,6 @@ const http = axios.create({ baseURL: process.env.BASE_URL, withCredentials: false, responseType: 'json', - // transformResponse: [ - // // Apply the defaut transformations as well. - // ...axios.defaults.transformResponse, - // (resp) => { - // if (!resp) { - // return resp; - // } - - // // There's an error message. - // if ('message' in resp && resp.message !== '') { - // return resp; - // } - - // return humps.camelizeKeys(resp.data); - // }, - // ], // Override the default serializer to switch params from becoming []id=a&[]id=b ... // in GET and DELETE requests to id=a&id=b. @@ -47,12 +31,13 @@ http.interceptors.response.use((resp) => { store.commit('setLoading', { model: resp.config.loading, status: false }); } - let data = {}; - if (resp.data && resp.data.data) { - if (typeof resp.data.data === 'object') { - data = humps.camelizeKeys(resp.data.data); - } else { - data = resp.data.data; + let data = { ...resp.data.data }; + if (!resp.config.preserveCase) { + if (resp.data && resp.data.data) { + if (typeof resp.data.data === 'object') { + // Transform field case. + data = humps.camelizeKeys(resp.data.data); + } } } @@ -75,11 +60,13 @@ http.interceptors.response.use((resp) => { msg = err.toString(); } - Toast.open({ - message: msg, - type: 'is-danger', - queue: false, - }); + if (!err.config.disableToast) { + Toast.open({ + message: msg, + type: 'is-danger', + queue: false, + }); + } return Promise.reject(err); }); @@ -88,6 +75,12 @@ http.interceptors.response.use((resp) => { // loading: modelName (set's the loading status in the global store: eg: store.loading.lists = true) // store: modelName (set's the API response in the global store. eg: store.lists: { ... } ) +// Health check endpoint that does not throw a toast. +export const getHealth = () => http.get('/api/health', + { disableToast: true }); + +export const reloadApp = () => http.post('/api/admin/reload'); + // Dashboard export const getDashboardCounts = () => http.get('/api/dashboard/counts', { loading: models.dashboard }); @@ -197,3 +190,10 @@ export const makeTemplateDefault = async (id) => http.put(`/api/templates/${id}/ export const deleteTemplate = async (id) => http.delete(`/api/templates/${id}`, { loading: models.templates }); + +// Settings. +export const getSettings = async () => http.get('/api/settings', + { loading: models.settings, preserveCase: true }); + +export const updateSettings = async (data) => http.put('/api/settings', data, + { loading: models.settings }); diff --git a/frontend/src/assets/style.scss b/frontend/src/assets/style.scss index 9ae032f..0e43896 100644 --- a/frontend/src/assets/style.scss +++ b/frontend/src/assets/style.scss @@ -77,6 +77,7 @@ section { } } + /* Two column sidebar+body layout */ #app { display: flex; @@ -126,6 +127,20 @@ section { } } +/* Global notices */ +.global-notices { + margin-bottom: 30px; +} +.notification { + padding: 10px 15px; + &.is-danger { + background: $white-ter; + color: $black; + border-left: 5px solid $red; + font-weight: bold; + } +} + /* HTML code editor */ .html-editor { position: relative; @@ -166,6 +181,11 @@ section { display: none; } +/* Toasts */ +.notices .toast { + animation: none; +} + /* Fix for button primary colour. */ .button.is-primary { background: $primary; @@ -453,6 +473,20 @@ section.campaign { } } +/* Settings */ +.settings { + .disabled { + opacity: 0.30; + } + .tab-content { + padding-top: 30px; + } + .box { + margin-bottom: 30px; + } +} + +/* C3 charting lib */ .c3 { .c3-chart-lines .c3-line { stroke-width: 2px; diff --git a/frontend/src/constants.js b/frontend/src/constants.js index 2072382..fd2cbea 100644 --- a/frontend/src/constants.js +++ b/frontend/src/constants.js @@ -1,10 +1,15 @@ export const models = Object.freeze({ + // This is the config loaded from /api/config.js directly onto the page + // via a diff --git a/go.mod b/go.mod index 1a47da3..6ef7bb4 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,5 @@ module github.com/knadh/listmonk + go 1.13 require ( @@ -7,12 +8,13 @@ require ( github.com/jaytaylor/html2text v0.0.0-20200220170450-61d9dc4d7195 github.com/jmoiron/sqlx v1.2.0 github.com/knadh/goyesql/v2 v2.1.1 - github.com/knadh/koanf v0.8.1 + github.com/knadh/koanf v0.12.0 github.com/knadh/smtppool v0.2.0 github.com/knadh/stuffbin v1.1.0 github.com/labstack/echo v3.3.10+incompatible github.com/labstack/gommon v0.3.0 // indirect github.com/lib/pq v1.3.0 + github.com/nats-io/nats-server/v2 v2.1.7 // indirect github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect github.com/olekukonko/tablewriter v0.0.4 // indirect github.com/rhnvrm/simples3 v0.5.0 @@ -21,4 +23,4 @@ require ( gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect gopkg.in/volatiletech/null.v6 v6.0.0-20170828023728-0bef4e07ae1b jaytaylor.com/html2text v0.0.0-20200220170450-61d9dc4d7195 -) \ No newline at end of file +) diff --git a/go.sum b/go.sum index e188b54..92dd450 100644 --- a/go.sum +++ b/go.sum @@ -8,6 +8,8 @@ github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44am github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/go-sql-driver/mysql v1.4.0 h1:7LxgVwFb2hIQtMm87NdgAVfXjnt4OePseqT1tKx+opk= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-yaml/yaml v2.1.0+incompatible h1:RYi2hDdss1u4YE7GwixGzWwVo47T8UQwnTLB6vQiq+o= @@ -15,6 +17,14 @@ github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaL github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE= github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/jaytaylor/html2text v0.0.0-20200220170450-61d9dc4d7195 h1:j0UEFmS7wSjAwKEIkgKBn8PRDfjcuggzr93R9wk53nQ= @@ -29,6 +39,8 @@ github.com/knadh/goyesql/v2 v2.1.1 h1:Orp5ldaxPM4ozKHfu1m7p6iolJFXDGOpF3/jyOgO6l github.com/knadh/goyesql/v2 v2.1.1/go.mod h1:pMzCA130/ZhEIoMmSmbEFXor3A2dxl5L+JllAc/l64s= github.com/knadh/koanf v0.8.1 h1:4VLACWqrkWRQIup3ooq6lOnaSbOJSNO+YVXnJn/NPZ8= github.com/knadh/koanf v0.8.1/go.mod h1:kVvmDbXnBtW49Czi4c1M+nnOWF0YSNZ8BaKvE/bCO1w= +github.com/knadh/koanf v0.12.0 h1:xQo0Y43CbzOix0tTeE+plIcfs1pTuaUI1/SsvDl2ROI= +github.com/knadh/koanf v0.12.0/go.mod h1:31bzRSM7vS5Vm9LNLo7B2Re1zhLOZT6EQKeodixBikE= github.com/knadh/smtppool v0.1.1 h1:pSi1Gc5TXOaN/Z/YiqfZbk/vd9dqzXzAfQiss0QSGQU= github.com/knadh/smtppool v0.1.1/go.mod h1:3DJHouXAgPDBz0kC50HukOsdapYSwIEfJGwuip46oCA= github.com/knadh/smtppool v0.2.0 h1:+llTWRljNIVg05MMu9TiefELTNwblexjsd1ALAPXZUs= @@ -59,12 +71,27 @@ github.com/mattn/go-sqlite3 v1.9.0 h1:pDRiWfl+++eC2FEFRy6jXmQlvp4Yh3z1MJKg4UeYM/ github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.2.2 h1:dxe5oCinTXiTIcfgmZecdCzPmAJKd46KsCWc35r0TV4= +github.com/mitchellh/mapstructure v1.2.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/nats-io/jwt v0.3.2 h1:+RB5hMpXUUA2dfxuhBTEkMOrYmM+gKIZYS1KjSostMI= +github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= +github.com/nats-io/nats-server v1.4.1 h1:Ul1oSOGNV/L8kjr4v6l2f9Yet6WY+LevH1/7cRZ/qyA= +github.com/nats-io/nats-server/v2 v2.1.7 h1:jCoQwDvRYJy3OpOTHeYfvIPLP46BMeDmH7XEJg/r42I= +github.com/nats-io/nats-server/v2 v2.1.7/go.mod h1:rbRrRE/Iv93O/rUvZ9dh4NfT0Cm9HWjW/BqOWLGgYiE= +github.com/nats-io/nats.go v1.10.0/go.mod h1:AjGArbfyR50+afOUotNX2Xs5SYHf+CoOa5HH1eEl2HE= +github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= +github.com/nats-io/nkeys v0.1.4 h1:aEsHIssIk6ETN5m2/MD8Y4B2X7FfXrBAUdkyRvbVYzA= +github.com/nats-io/nkeys v0.1.4/go.mod h1:XdZpAbhgyyODYqjTawOnIOI7VlbKSarI9Gfy1tqEu/s= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/olekukonko/tablewriter v0.0.4 h1:vHD/YYe1Wolo78koG299f7V/VAS08c6IpCLn+Ejf/w8= github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA= github.com/pelletier/go-toml v1.4.0 h1:u3Z1r+oOXJIkxqw34zVhyPgjBsm6X2wn21NWs/HfSeg= github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= +github.com/pelletier/go-toml v1.7.0 h1:7utD74fnzVc/cpcyy8sjrlFr5vYpypUixARcHIMIGuI= +github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rhnvrm/simples3 v0.5.0 h1:X+WX0hqoKScdoJAw/G3GArfZ6Ygsn8q+6MdocTMKXOw= @@ -86,22 +113,39 @@ github.com/valyala/fasttemplate v1.0.1 h1:tY9CJiPnMXf1ERmG2EyK7gNUd+c6RKGD0IfU8W github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59 h1:3zb4D3T4G8jdExgVU/95+vQXfpEPiMdCaZgmGVxjNHM= +golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8 h1:hVwzHzIUGRjiF7EcUjqNxk3NCfkPxbDKRdnNE1Rpg0U= golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65 h1:+rhAzEzT3f4JtomfC371qB+0Ola2caSKcY69NUBZrRQ= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24 h1:R8bzl0244nw47n1xKs1MUMAaTNgjavKcN/aX2Ss3+Fo= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d h1:nc5K6ox/4lTFbMVSL9WRR81ixkcwXThoiF6yf+R9scA= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= @@ -110,5 +154,6 @@ gopkg.in/volatiletech/null.v6 v6.0.0-20170828023728-0bef4e07ae1b h1:P+3+n9hUbqSD gopkg.in/volatiletech/null.v6 v6.0.0-20170828023728-0bef4e07ae1b/go.mod h1:0LRKfykySnChgQpG3Qpk+bkZFWazQ+MMfc5oldQCwnY= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= jaytaylor.com/html2text v0.0.0-20200220170450-61d9dc4d7195 h1:tj3Wzc08ekoAl8zEsLhT+5EmZ9TE/qpTTTi4oZjOPMw= jaytaylor.com/html2text v0.0.0-20200220170450-61d9dc4d7195/go.mod h1:OxvTsCwKosqQ1q7B+8FwXqg4rKZ/UG9dUW+g/VL2xH4= diff --git a/handlers.go b/handlers.go index c00a6b7..83a456d 100644 --- a/handlers.go +++ b/handlers.go @@ -37,10 +37,15 @@ var reUUID = regexp.MustCompile("^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[ // registerHandlers registers HTTP handlers. func registerHTTPHandlers(e *echo.Echo) { e.GET("/", handleIndexPage) + e.GET("/api/health", handleHealthCheck) e.GET("/api/config.js", handleGetConfigScript) e.GET("/api/dashboard/charts", handleGetDashboardCharts) e.GET("/api/dashboard/counts", handleGetDashboardCounts) + e.GET("/api/settings", handleGetSettings) + e.PUT("/api/settings", handleUpdateSettings) + e.POST("/api/admin/reload", handleReloadApp) + e.GET("/api/subscribers/:id", handleGetSubscriber) e.GET("/api/subscribers/:id/export", handleExportSubscriberData) e.POST("/api/subscribers", handleCreateSubscriber) @@ -140,6 +145,11 @@ func handleIndexPage(c echo.Context) error { return c.String(http.StatusOK, string(b)) } +// handleHealthCheck is a healthcheck endpoint that returns a 200 response. +func handleHealthCheck(c echo.Context) error { + return c.JSON(http.StatusOK, okResp{true}) +} + // validateUUID middleware validates the UUID string format for a given set of params. func validateUUID(next echo.HandlerFunc, params ...string) echo.HandlerFunc { return func(c echo.Context) error { diff --git a/init.go b/init.go index e3a3732..411a664 100644 --- a/init.go +++ b/init.go @@ -1,17 +1,25 @@ package main import ( + "encoding/json" "fmt" "html/template" "os" "path/filepath" "strings" + "syscall" + "time" "github.com/jmoiron/sqlx" + "github.com/jmoiron/sqlx/types" "github.com/knadh/goyesql/v2" goyesqlx "github.com/knadh/goyesql/v2/sqlx" "github.com/knadh/koanf" "github.com/knadh/koanf/maps" + "github.com/knadh/koanf/parsers/toml" + "github.com/knadh/koanf/providers/confmap" + "github.com/knadh/koanf/providers/file" + "github.com/knadh/koanf/providers/posflag" "github.com/knadh/listmonk/internal/manager" "github.com/knadh/listmonk/internal/media" "github.com/knadh/listmonk/internal/media/providers/filesystem" @@ -20,12 +28,74 @@ import ( "github.com/knadh/listmonk/internal/subimporter" "github.com/knadh/stuffbin" "github.com/labstack/echo" + flag "github.com/spf13/pflag" ) const ( queryFilePath = "queries.sql" ) +// constants contains static, constant config values required by the app. +type constants struct { + RootURL string `koanf:"root"` + LogoURL string `koanf:"logo_url"` + FaviconURL string `koanf:"favicon_url"` + FromEmail string `koanf:"from_email"` + NotifyEmails []string `koanf:"notify_emails"` + Privacy struct { + AllowBlacklist bool `koanf:"allow_blacklist"` + AllowExport bool `koanf:"allow_export"` + AllowWipe bool `koanf:"allow_wipe"` + Exportable map[string]bool `koanf:"-"` + } `koanf:"privacy"` + + UnsubURL string + LinkTrackURL string + ViewTrackURL string + OptinURL string + MessageURL string + + MediaProvider string +} + +func initFlags() { + f := flag.NewFlagSet("config", flag.ContinueOnError) + f.Usage = func() { + // Register --help handler. + fmt.Println(f.FlagUsages()) + os.Exit(0) + } + + // Register the commandline flags. + f.StringSlice("config", []string{"config.toml"}, + "path to one or more config files (will be merged in order)") + f.Bool("install", false, "run first time installation") + f.Bool("version", false, "current version of the build") + f.Bool("new-config", false, "generate sample config file") + f.String("static-dir", "", "(optional) path to directory with static files") + f.Bool("yes", false, "assume 'yes' to prompts, eg: during --install") + if err := f.Parse(os.Args[1:]); err != nil { + lo.Fatalf("error loading flags: %v", err) + } + + if err := ko.Load(posflag.Provider(f, ".", ko), nil); err != nil { + lo.Fatalf("error loading config: %v", err) + } +} + +// initConfigFiles loads the given config files into the koanf instance. +func initConfigFiles(files []string, ko *koanf.Koanf) { + for _, f := range files { + lo.Printf("reading config: %s", f) + if err := ko.Load(file.Provider(f), toml.Parser()); err != nil { + if os.IsNotExist(err) { + lo.Fatal("config file not found. If there isn't one yet, run --new-config to generate one.") + } + lo.Fatalf("error loadng config from file: %v.", err) + } + } +} + // initFileSystem initializes the stuffbin FileSystem to provide // access to bunded static assets to the app. func initFS(staticDir string) stuffbin.FileSystem { @@ -87,7 +157,6 @@ func initFS(staticDir string) stuffbin.FileSystem { // initDB initializes the main DB connection pool and parse and loads the app's // SQL queries into a prepared query map. func initDB() *sqlx.DB { - var dbCfg dbConf if err := ko.Unmarshal("db", &dbCfg); err != nil { lo.Fatalf("error loading db config: %v", err) @@ -98,7 +167,6 @@ func initDB() *sqlx.DB { if err != nil { lo.Fatalf("error connecting to DB: %v", err) } - return db } @@ -127,27 +195,22 @@ func initQueries(sqlFile string, db *sqlx.DB, fs stuffbin.FileSystem, prepareQue return qMap, &q } -// constants contains static, constant config values required by the app. -type constants struct { - RootURL string `koanf:"root"` - LogoURL string `koanf:"logo_url"` - FaviconURL string `koanf:"favicon_url"` - FromEmail string `koanf:"from_email"` - NotifyEmails []string `koanf:"notify_emails"` - Privacy struct { - AllowBlacklist bool `koanf:"allow_blacklist"` - AllowExport bool `koanf:"allow_export"` - AllowWipe bool `koanf:"allow_wipe"` - Exportable map[string]bool `koanf:"-"` - } `koanf:"privacy"` +// initSettings loads settings from the DB. +func initSettings(q *Queries) { + var s types.JSONText + if err := q.GetSettings.Get(&s); err != nil { + lo.Fatalf("error reading settings from DB: %s", pqErrMsg(err)) + } - UnsubURL string - LinkTrackURL string - ViewTrackURL string - OptinURL string - MessageURL string - - MediaProvider string + // Setting keys are dot separated, eg: app.favicon_url. Unflatten them into + // nested maps {app: {favicon_url}}. + var out map[string]interface{} + if err := json.Unmarshal(s, &out); err != nil { + lo.Fatalf("error unmarshalling settings from DB: %v", err) + } + if err := ko.Load(confmap.Provider(out, "."), nil); err != nil { + lo.Fatalf("error parsing settings from DB: %v", err) + } } func initConstants() *constants { @@ -159,6 +222,7 @@ func initConstants() *constants { if err := ko.Unmarshal("privacy", &c.Privacy); err != nil { lo.Fatalf("error loading app config: %v", err) } + c.RootURL = strings.TrimRight(c.RootURL, "/") c.Privacy.Exportable = maps.StringSliceToLookupMap(ko.Strings("privacy.exportable")) c.MediaProvider = ko.String("upload.provider") @@ -227,31 +291,35 @@ func initImporter(q *Queries, db *sqlx.DB, app *App) *subimporter.Importer { func initMessengers(m *manager.Manager) messenger.Messenger { var ( mapKeys = ko.MapKeys("smtp") - srv = make([]messenger.Server, 0, len(mapKeys)) + servers = make([]messenger.Server, 0, len(mapKeys)) ) + items := ko.Slices("smtp") + if len(items) == 0 { + lo.Fatalf("no SMTP servers found in config") + } + // Load the default SMTP messengers. - for _, name := range mapKeys { - if !ko.Bool(fmt.Sprintf("smtp.%s.enabled", name)) { - lo.Printf("skipped SMTP: %s", name) + for _, item := range items { + if !item.Bool("enabled") { continue } // Read the SMTP config. - s := messenger.Server{Name: name} - if err := ko.UnmarshalWithConf("smtp."+name, &s, koanf.UnmarshalConf{Tag: "json"}); err != nil { + var s messenger.Server + if err := item.UnmarshalWithConf("", &s, koanf.UnmarshalConf{Tag: "json"}); err != nil { lo.Fatalf("error loading SMTP: %v", err) } - srv = append(srv, s) - lo.Printf("loaded SMTP: %s (%s@%s)", s.Name, s.Username, s.Host) + servers = append(servers, s) + lo.Printf("loaded SMTP: %s@%s", item.String("username"), item.String("host")) } - if len(srv) == 0 { - lo.Fatalf("no SMTP servers found in config") + if len(servers) == 0 { + lo.Fatalf("no SMTP servers enabled in settings") } // Initialize the default e-mail messenger. - msgr, err := messenger.NewEmailer(srv...) + msgr, err := messenger.NewEmailer(servers...) if err != nil { lo.Fatalf("error loading e-mail messenger: %v", err) } @@ -266,28 +334,31 @@ func initMessengers(m *manager.Manager) messenger.Messenger { func initMediaStore() media.Store { switch provider := ko.String("upload.provider"); provider { case "s3": - var opts s3.Opts - ko.Unmarshal("upload.s3", &opts) - uplder, err := s3.NewS3Store(opts) + var o s3.Opts + ko.Unmarshal("upload.s3", &o) + up, err := s3.NewS3Store(o) if err != nil { lo.Fatalf("error initializing s3 upload provider %s", err) } - return uplder + lo.Println("media upload provider: s3") + return up case "filesystem": - var opts filesystem.Opts - ko.Unmarshal("upload.filesystem", &opts) - opts.RootURL = ko.String("app.root") - opts.UploadPath = filepath.Clean(opts.UploadPath) - opts.UploadURI = filepath.Clean(opts.UploadURI) - uplder, err := filesystem.NewDiskStore(opts) + var o filesystem.Opts + + ko.Unmarshal("upload.filesystem", &o) + o.RootURL = ko.String("app.root") + o.UploadPath = filepath.Clean(o.UploadPath) + o.UploadURI = filepath.Clean(o.UploadURI) + up, err := filesystem.NewDiskStore(o) if err != nil { lo.Fatalf("error initializing filesystem upload provider %s", err) } - return uplder + lo.Println("media upload provider: filesystem") + return up default: - lo.Fatalf("unknown provider. please select one of either filesystem or s3") + lo.Fatalf("unknown provider. select filesystem or s3") } return nil } @@ -312,7 +383,7 @@ func initNotifTemplates(path string, fs stuffbin.FileSystem, cs *constants) *tem } // initHTTPServer sets up and runs the app's main HTTP server and blocks forever. -func initHTTPServer(app *App) { +func initHTTPServer(app *App) *echo.Echo { // Initialize the HTTP server. var srv = echo.New() srv.HideBanner = true @@ -349,5 +420,47 @@ func initHTTPServer(app *App) { registerHTTPHandlers(srv) // Start the server. - srv.Logger.Fatal(srv.Start(ko.String("app.address"))) + go func() { + if err := srv.Start(ko.String("app.address")); err != nil { + if strings.Contains(err.Error(), "Server closed") { + lo.Println("HTTP server shut down") + } else { + lo.Fatalf("error starting HTTP server: %v", err) + } + } + }() + + return srv +} + +func awaitReload(sigChan chan os.Signal, closerWait chan bool, closer func()) chan bool { + // The blocking signal handler that main() waits on. + out := make(chan bool) + + // Respawn a new process and exit the running one. + respawn := func() { + if err := syscall.Exec(os.Args[0], os.Args, os.Environ()); err != nil { + lo.Fatalf("error spawning process: %v", err) + } + os.Exit(0) + } + + // Listen for reload signal. + go func() { + for range sigChan { + lo.Println("reloading on signal ...") + + go closer() + select { + case <-closerWait: + // Wait for the closer to finish. + respawn() + case <-time.After(time.Second * 3): + // Or timeout and force close. + respawn() + } + } + }() + + return out } diff --git a/internal/manager/manager.go b/internal/manager/manager.go index c2b08fa..eec880f 100644 --- a/internal/manager/manager.go +++ b/internal/manager/manager.go @@ -184,6 +184,13 @@ func (m *Manager) HasMessenger(id string) bool { return ok } +// HasRunningCampaigns checks if there are any active campaigns. +func (m *Manager) HasRunningCampaigns() bool { + m.campsMutex.Lock() + defer m.campsMutex.Unlock() + return len(m.camps) > 0 +} + // Run is a blocking function (that should be invoked as a goroutine) // that scans the data source at regular intervals for pending campaigns, // and queues them for processing. The process queue fetches batches of @@ -230,7 +237,11 @@ func (m *Manager) messageWorker() { for { select { // Campaign message. - case msg := <-m.campMsgQueue: + case msg, ok := <-m.campMsgQueue: + if !ok { + return + } + // Pause on hitting the message rate. if numMsg >= m.cfg.MessageRate { time.Sleep(time.Second) @@ -250,7 +261,10 @@ func (m *Manager) messageWorker() { } // Arbitrary message. - case msg := <-m.msgQueue: + case msg, ok := <-m.msgQueue: + if !ok { + return + } err := m.messengers[msg.Messenger].Push( msg.From, msg.To, msg.Subject, msg.Body, nil) if err != nil { @@ -291,6 +305,13 @@ func (m *Manager) TemplateFuncs(c *models.Campaign) template.FuncMap { } } +// Close closes and exits the campaign manager. +func (m *Manager) Close() { + close(m.subFetchQueue) + close(m.campMsgErrorQueue) + close(m.msgQueue) +} + // scanCampaigns is a blocking function that periodically scans the data source // for campaigns to process and dispatches them to the manager. func (m *Manager) scanCampaigns(tick time.Duration) { @@ -323,7 +344,10 @@ func (m *Manager) scanCampaigns(tick time.Duration) { // Aggregate errors from sending messages to check against the error threshold // after which a campaign is paused. - case e := <-m.campMsgErrorQueue: + case e, ok := <-m.campMsgErrorQueue: + if !ok { + return + } if m.cfg.MaxSendErrors < 1 { continue } diff --git a/internal/media/providers/s3/s3.go b/internal/media/providers/s3/s3.go index 9ccb0c2..eff7c7a 100644 --- a/internal/media/providers/s3/s3.go +++ b/internal/media/providers/s3/s3.go @@ -15,14 +15,14 @@ const amznS3PublicURL = "https://%s.s3.%s.amazonaws.com%s" // Opts represents AWS S3 specific params type Opts struct { - AccessKey string `koanf:"aws_access_key_id"` - SecretKey string `koanf:"aws_secret_access_key"` - Region string `koanf:"aws_default_region"` - Bucket string `koanf:"bucket"` - BucketPath string `koanf:"bucket_path"` - BucketURL string `koanf:"bucket_url"` - BucketType string `koanf:"bucket_type"` - Expiry int `koanf:"expiry"` + AccessKey string `koanf:"aws_access_key_id"` + SecretKey string `koanf:"aws_secret_access_key"` + Region string `koanf:"aws_default_region"` + Bucket string `koanf:"bucket"` + BucketPath string `koanf:"bucket_path"` + BucketURL string `koanf:"bucket_url"` + BucketType string `koanf:"bucket_type"` + Expiry time.Duration `koanf:"expiry"` } // Client implements `media.Store` for S3 provider @@ -83,7 +83,7 @@ func (c *Client) Get(name string) string { ObjectKey: makeBucketPath(c.opts.BucketPath, name), Method: "GET", Timestamp: time.Now(), - ExpirySeconds: c.opts.Expiry, + ExpirySeconds: int(c.opts.Expiry.Seconds()), }) return url } diff --git a/internal/messenger/emailer.go b/internal/messenger/emailer.go index bb40374..352bb8f 100644 --- a/internal/messenger/emailer.go +++ b/internal/messenger/emailer.go @@ -15,7 +15,6 @@ const emName = "email" // Server represents an SMTP server's credentials. type Server struct { - Name string Username string `json:"username"` Password string `json:"password"` AuthProtocol string `json:"auth_protocol"` @@ -33,16 +32,14 @@ type Server struct { // Emailer is the SMTP e-mail messenger. type Emailer struct { - servers map[string]*Server - serverNames []string - numServers int + servers []*Server } // NewEmailer creates and returns an e-mail Messenger backend. // It takes multiple SMTP configurations. func NewEmailer(servers ...Server) (*Emailer, error) { e := &Emailer{ - servers: make(map[string]*Server), + servers: make([]*Server, 0, len(servers)), } for _, srv := range servers { @@ -77,11 +74,9 @@ func NewEmailer(servers ...Server) (*Emailer, error) { } s.pool = pool - e.servers[s.Name] = &s - e.serverNames = append(e.serverNames, s.Name) + e.servers = append(e.servers, &s) } - e.numServers = len(e.serverNames) return e, nil } @@ -92,14 +87,16 @@ func (e *Emailer) Name() string { // Push pushes a message to the server. func (e *Emailer) Push(fromAddr string, toAddr []string, subject string, m []byte, atts []Attachment) error { - var key string - // If there are more than one SMTP servers, send to a random // one from the list. - if e.numServers > 1 { - key = e.serverNames[rand.Intn(e.numServers)] + var ( + ln = len(e.servers) + srv *Server + ) + if ln > 1 { + srv = e.servers[rand.Intn(ln)] } else { - key = e.serverNames[0] + srv = e.servers[0] } // Are there attachments? @@ -122,7 +119,6 @@ func (e *Emailer) Push(fromAddr string, toAddr []string, subject string, m []byt return err } - srv := e.servers[key] em := smtppool.Email{ From: fromAddr, To: toAddr, @@ -155,3 +151,11 @@ func (e *Emailer) Push(fromAddr string, toAddr []string, subject string, m []byt func (e *Emailer) Flush() error { return nil } + +// Close closes the SMTP pools. +func (e *Emailer) Close() error { + for _, s := range e.servers { + s.pool.Close() + } + return nil +} diff --git a/internal/messenger/messenger.go b/internal/messenger/messenger.go index 2b0441c..67b8e32 100644 --- a/internal/messenger/messenger.go +++ b/internal/messenger/messenger.go @@ -8,6 +8,7 @@ type Messenger interface { Name() string Push(fromAddr string, toAddr []string, subject string, message []byte, atts []Attachment) error Flush() error + Close() error } // Attachment represents a file or blob attachment that can be diff --git a/main.go b/main.go index 03ec0c8..dfa8d21 100644 --- a/main.go +++ b/main.go @@ -1,25 +1,25 @@ package main import ( + "context" "fmt" "html/template" "log" "os" + "os/signal" "strings" + "sync" + "syscall" "time" "github.com/jmoiron/sqlx" "github.com/knadh/koanf" - "github.com/knadh/koanf/parsers/toml" "github.com/knadh/koanf/providers/env" - "github.com/knadh/koanf/providers/file" - "github.com/knadh/koanf/providers/posflag" "github.com/knadh/listmonk/internal/manager" "github.com/knadh/listmonk/internal/media" "github.com/knadh/listmonk/internal/messenger" "github.com/knadh/listmonk/internal/subimporter" "github.com/knadh/stuffbin" - flag "github.com/spf13/pflag" ) // App contains the "global" components that are @@ -35,47 +35,38 @@ type App struct { media media.Store notifTpls *template.Template log *log.Logger + + // Channel for passing reload signals. + sigChan chan os.Signal + + // Global variable that stores the state indicating that a restart is required + // after a settings update. + needsRestart bool + sync.Mutex } var ( - // Global logger. lo = log.New(os.Stdout, "", log.Ldate|log.Ltime|log.Lshortfile) - - // Global configuration reader. ko = koanf.New(".") + fs stuffbin.FileSystem + db *sqlx.DB + queries *Queries + buildString string ) func init() { - f := flag.NewFlagSet("config", flag.ContinueOnError) - f.Usage = func() { - // Register --help handler. - fmt.Println(f.FlagUsages()) - os.Exit(0) - } - - // Register the commandline flags. - f.StringSlice("config", []string{"config.toml"}, - "path to one or more config files (will be merged in order)") - f.Bool("install", false, "run first time installation") - f.Bool("version", false, "current version of the build") - f.Bool("new-config", false, "generate sample config file") - f.String("static-dir", "", "(optional) path to directory with static files") - f.Bool("yes", false, "assume 'yes' to prompts, eg: during --install") - - if err := f.Parse(os.Args[1:]); err != nil { - lo.Fatalf("error loading flags: %v", err) - } + initFlags() // Display version. - if v, _ := f.GetBool("version"); v { + if ko.Bool("version") { fmt.Println(buildString) os.Exit(0) } // Generate new config. - if ok, _ := f.GetBool("new-config"); ok { + if ko.Bool("new-config") { if err := newConfigFile(); err != nil { lo.Println(err) os.Exit(1) @@ -84,38 +75,12 @@ func init() { os.Exit(0) } - // Load config files. - cFiles, _ := f.GetStringSlice("config") - for _, f := range cFiles { - lo.Printf("reading config: %s", f) - if err := ko.Load(file.Provider(f), toml.Parser()); err != nil { - if os.IsNotExist(err) { - lo.Fatal("config file not found. If there isn't one yet, run --new-config to generate one.") - } - lo.Fatalf("error loadng config from file: %v.", err) - } - } + // Load config files to pick up the database settings first. + initConfigFiles(ko.Strings("config"), ko) - // Load environment variables and merge into the loaded config. - if err := ko.Load(env.Provider("LISTMONK_", ".", func(s string) string { - return strings.Replace(strings.ToLower( - strings.TrimPrefix(s, "LISTMONK_")), "__", ".", -1) - }), nil); err != nil { - lo.Fatalf("error loading config from env: %v", err) - } - if err := ko.Load(posflag.Provider(f, ".", ko), nil); err != nil { - lo.Fatalf("error loading config: %v", err) - } -} - -func main() { - // Initialize the DB and the filesystem that are required by the installer - // and the app. - var ( - fs = initFS(ko.String("static-dir")) - db = initDB() - ) - defer db.Close() + // Connect to the database, load the filesystem to read SQL queries. + db = initDB() + fs = initFS(ko.String("static-dir")) // Installer mode? This runs before the SQL queries are loaded and prepared // as the installer needs to work on an empty DB. @@ -124,6 +89,22 @@ func main() { return } + // Load the SQL queries from the filesystem. + _, queries := initQueries(queryFilePath, db, fs, true) + + // Load settings from DB. + initSettings(queries) + + // Load environment variables and merge into the loaded config. + if err := ko.Load(env.Provider("LISTMONK_", ".", func(s string) string { + return strings.Replace(strings.ToLower( + strings.TrimPrefix(s, "LISTMONK_")), "__", ".", -1) + }), nil); err != nil { + lo.Fatalf("error loading config from env: %v", err) + } +} + +func main() { // Initialize the main app controller that wraps all of the app's // components. This is passed around HTTP handlers. app := &App{ @@ -143,6 +124,32 @@ func main() { // messages) get processed at the specified interval. go app.manager.Run(time.Second * 5) - // Start and run the app server. - initHTTPServer(app) + // Start the app server. + srv := initHTTPServer(app) + + // Wait for the reload signal with a callback to gracefully shut down resources. + // The `wait` channel is passed to awaitReload to wait for the callback to finish + // within N seconds, or do a force reload. + app.sigChan = make(chan os.Signal) + signal.Notify(app.sigChan, syscall.SIGHUP) + + closerWait := make(chan bool) + <-awaitReload(app.sigChan, closerWait, func() { + // Stop the HTTP server. + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + defer cancel() + srv.Shutdown(ctx) + + // Close the campaign manager. + app.manager.Close() + + // Close the DB pool. + app.db.DB.Close() + + // Close the messenger pool. + app.messenger.Close() + + // Signal the close. + closerWait <- true + }) } diff --git a/queries.go b/queries.go index 28cb505..dd880a2 100644 --- a/queries.go +++ b/queries.go @@ -76,6 +76,9 @@ type Queries struct { CreateLink *sqlx.Stmt `query:"create-link"` RegisterLinkClick *sqlx.Stmt `query:"register-link-click"` + GetSettings *sqlx.Stmt `query:"get-settings"` + UpdateSettings *sqlx.Stmt `query:"update-settings"` + // GetStats *sqlx.Stmt `query:"get-stats"` } diff --git a/queries.sql b/queries.sql index c1e2820..725ec10 100644 --- a/queries.sql +++ b/queries.sql @@ -724,3 +724,14 @@ SELECT JSON_BUILD_OBJECT('subscribers', JSON_BUILD_OBJECT( ) ), 'messages', (SELECT SUM(sent) AS messages FROM campaigns)); + +-- name: get-settings +SELECT JSON_OBJECT_AGG(key, value) AS settings + FROM ( + SELECT * FROM settings ORDER BY key + ) t; + +-- name: update-settings +UPDATE settings AS s SET value = c.value + -- For each key in the incoming JSON map, update the row with the key and it's value. + FROM(SELECT * FROM JSONB_EACH($1)) AS c(key, value) WHERE s.key = c.key; diff --git a/schema.sql b/schema.sql index 95a9f32..646772b 100644 --- a/schema.sql +++ b/schema.sql @@ -155,3 +155,40 @@ CREATE TABLE link_clicks ( DROP INDEX IF EXISTS idx_clicks_camp_id; CREATE INDEX idx_clicks_camp_id ON link_clicks(campaign_id); DROP INDEX IF EXISTS idx_clicks_link_id; CREATE INDEX idx_clicks_link_id ON link_clicks(link_id); DROP INDEX IF EXISTS idx_clicks_sub_id; CREATE INDEX idx_clicks_sub_id ON link_clicks(subscriber_id); + +-- settings +DROP TABLE IF EXISTS settings CASCADE; +CREATE TABLE settings ( + key TEXT NOT NULL UNIQUE, + value JSONB NOT NULL DEFAULT '{}', + updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW() +); +DROP INDEX IF EXISTS idx_settings_key; CREATE INDEX idx_settings_key ON settings(key); +INSERT INTO settings (key, value) VALUES + ('app.favicon_url', '""'), + ('app.from_email', '"listmonk "'), + ('app.logo_url', '"http://localhost:9000/public/static/logo.png"'), + ('app.concurrency', '10'), + ('app.message_rate', '10'), + ('app.batch_size', '1000'), + ('app.max_send_errors', '1000'), + ('app.notify_emails', '["admin1@mysite.com", "admin2@mysite.com"]'), + ('privacy.allow_blacklist', 'true'), + ('privacy.allow_export', 'true'), + ('privacy.allow_wipe', 'true'), + ('privacy.exportable', '["profile", "subscriptions", "campaign_views", "link_clicks"]'), + ('upload.provider', '"filesystem"'), + ('upload.filesystem.upload_path', '"uploads"'), + ('upload.filesystem.upload_uri', '"/uploads"'), + ('upload.s3.aws_access_key_id', '""'), + ('upload.s3.aws_secret_access_key', '""'), + ('upload.s3.aws_default_region', '"ap-south-b"'), + ('upload.s3.bucket', '""'), + ('upload.s3.bucket_domain', '""'), + ('upload.s3.bucket_path', '"/"'), + ('upload.s3.bucket_type', '"public"'), + ('upload.s3.expiry', '"14d"'), + ('smtp', + '[{"enabled":true, "host":"smtp.yoursite.com","port":25,"auth_protocol":"cram","username":"username","password":"password","hello_hostname":"","max_conns":10,"idle_timeout":"15s","wait_timeout":"5s","max_msg_retries":2,"tls_enabled":true,"tls_skip_verify":false,"email_headers":[]}, + {"enabled":false, "host":"smtp2.yoursite.com","port":587,"auth_protocol":"plain","username":"username","password":"password","hello_hostname":"","max_conns":10,"idle_timeout":"15s","wait_timeout":"5s","max_msg_retries":2,"tls_enabled":false,"tls_skip_verify":false,"email_headers":[]}]'), + ('messengers', '[]'); diff --git a/settings.go b/settings.go new file mode 100644 index 0000000..7369183 --- /dev/null +++ b/settings.go @@ -0,0 +1,179 @@ +package main + +import ( + "encoding/json" + "fmt" + "net/http" + "syscall" + "time" + + "github.com/jmoiron/sqlx/types" + "github.com/labstack/echo" +) + +type settings struct { + AppRootURL string `json:"app.root_url"` + AppLogoURL string `json:"app.logo_url"` + AppFaviconURL string `json:"app.favicon_url"` + AppFromEmail string `json:"app.from_email"` + AppNotifyEmails []string `json:"app.notify_emails"` + AppBatchSize int `json:"app.batch_size"` + AppConcurrency int `json:"app.concurrency"` + AppMaxSendErrors int `json:"app.max_send_errors"` + AppMessageRate int `json:"app.message_rate"` + + Messengers []interface{} `json:"messengers"` + + PrivacyAllowBlacklist bool `json:"privacy.allow_blacklist"` + PrivacyAllowExport bool `json:"privacy.allow_export"` + PrivacyAllowWipe bool `json:"privacy.allow_wipe"` + PrivacyExportable []string `json:"privacy.exportable"` + + SMTP []struct { + Enabled bool `json:"enabled"` + Host string `json:"host"` + HelloHostname string `json:"hello_hostname"` + Port int `json:"port"` + AuthProtocol string `json:"auth_protocol"` + Username string `json:"username"` + Password string `json:"password,omitempty"` + EmailHeaders []map[string]string `json:"email_headers"` + MaxConns int `json:"max_conns"` + MaxMsgRetries int `json:"max_msg_retries"` + IdleTimeout string `json:"idle_timeout"` + WaitTimeout string `json:"wait_timeout"` + TLSEnabled bool `json:"tls_enabled"` + TLSSkipVerify bool `json:"tls_skip_verify"` + } `json:"smtp"` + + UploadProvider string `json:"upload.provider"` + + UploadFilesystemUploadPath string `json:"upload.filesystem.upload_path"` + UploadFilesystemUploadURI string `json:"upload.filesystem.upload_uri"` + + UploadS3AwsAccessKeyID string `json:"upload.s3.aws_access_key_id"` + UploadS3AwsDefaultRegion string `json:"upload.s3.aws_default_region"` + UploadS3AwsSecretAccessKey string `json:"upload.s3.aws_secret_access_key,omitempty"` + UploadS3Bucket string `json:"upload.s3.bucket"` + UploadS3BucketDomain string `json:"upload.s3.bucket_domain"` + UploadS3BucketPath string `json:"upload.s3.bucket_path"` + UploadS3BucketType string `json:"upload.s3.bucket_type"` + UploadS3Expiry string `json:"upload.s3.expiry"` +} + +// handleGetSettings returns settings from the DB. +func handleGetSettings(c echo.Context) error { + app := c.Get("app").(*App) + + s, err := getSettings(app) + if err != nil { + return err + } + + // Empty out passwords. + for i := 0; i < len(s.SMTP); i++ { + s.SMTP[i].Password = "" + } + s.UploadS3AwsSecretAccessKey = "" + + return c.JSON(http.StatusOK, okResp{s}) +} + +// handleUpdateSettings returns settings from the DB. +func handleUpdateSettings(c echo.Context) error { + var ( + app = c.Get("app").(*App) + set settings + ) + + // Unmarshal and marshal the fields once to sanitize the settings blob. + if err := c.Bind(&set); err != nil { + return err + } + + // Get the existing settings. + cur, err := getSettings(app) + if err != nil { + return err + } + + // There should be at least one SMTP block that's enabled. + has := false + for i, s := range set.SMTP { + if s.Enabled { + has = true + } + + // If there's no password coming in from the frontend, attempt to get the + // last saved password for the SMTP block at the same position. + if set.SMTP[i].Password == "" { + if len(cur.SMTP) > i && + set.SMTP[i].Host == cur.SMTP[i].Host && + set.SMTP[i].Username == cur.SMTP[i].Username { + set.SMTP[i].Password = cur.SMTP[i].Password + } + } + } + if !has { + return echo.NewHTTPError(http.StatusBadRequest, + "At least one SMTP block should be enabled") + } + + // S3 password? + if set.UploadS3AwsSecretAccessKey == "" { + set.UploadS3AwsSecretAccessKey = cur.UploadS3AwsSecretAccessKey + } + + // Marshal settings. + b, err := json.Marshal(set) + if err != nil { + return echo.NewHTTPError(http.StatusInternalServerError, + fmt.Sprintf("Error encoding settings: %v", err)) + } + + // Update the settings in the DB. + if _, err := app.queries.UpdateSettings.Exec(b); err != nil { + return echo.NewHTTPError(http.StatusInternalServerError, + fmt.Sprintf("Error updating settings: %s", pqErrMsg(err))) + } + + // If there are any active campaigns, don't do an auto reload and + // warn the user on the frontend. + if app.manager.HasRunningCampaigns() { + app.Lock() + app.needsRestart = true + app.Unlock() + + return c.JSON(http.StatusOK, okResp{struct { + NeedsRestart bool `json:"needs_restart"` + }{true}}) + } + + // No running campaigns. Reload the app. + go func() { + <-time.After(time.Millisecond * 500) + app.sigChan <- syscall.SIGHUP + }() + + return c.JSON(http.StatusOK, okResp{true}) +} + +func getSettings(app *App) (settings, error) { + var ( + b types.JSONText + out settings + ) + + if err := app.queries.GetSettings.Get(&b); err != nil { + return out, echo.NewHTTPError(http.StatusInternalServerError, + fmt.Sprintf("Error fetching settings: %s", pqErrMsg(err))) + } + + // Unmarshall the settings and filter out sensitive fields. + if err := json.Unmarshal([]byte(b), &out); err != nil { + return out, echo.NewHTTPError(http.StatusInternalServerError, + fmt.Sprintf("Error parsing settings: %v", err)) + } + + return out, nil +}