hangar-tech
/
listmonk
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

- Refactor and move template compilation from runner to models.Campaign to support adhoc template funcs 

- Add support for {{ Track "https://url.com" }} in templates to register and track links
This commit is contained in:
Kailash Nadh 2018-10-31 18:24:21 +05:30
parent 1ae2905bf5
commit 81953d68d0
10 changed files with 191 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
campaigns.go
View File

@ -95,14 +95,14 @@ func handlePreviewCampaign(c echo.Context) error {
id, _ = strconv.Atoi(c.Param("id"))
body = c.FormValue("body")
camp models.Campaign
camp = &models.Campaign{}
)
if id < 1 {
return echo.NewHTTPError(http.StatusBadRequest, "Invalid ID.")
}
err := app.Queries.GetCampaignForPreview.Get(&camp, id)
err := app.Queries.GetCampaignForPreview.Get(camp, id)
if err != nil {
if err == sql.ErrNoRows {
return echo.NewHTTPError(http.StatusBadRequest, "Campaign not found.")
@ -130,25 +130,23 @@ func handlePreviewCampaign(c echo.Context) error {
}
// Compile the template.
if body == "" {
body = camp.Body
if body != "" {
camp.Body = body
}
tpl, err := runner.CompileMessageTemplate(camp.TemplateBody, body)
if err != nil {
if err := camp.CompileTemplate(app.Runner.TemplateFuncs(camp)); err != nil {
return echo.NewHTTPError(http.StatusBadRequest,
fmt.Sprintf("Error compiling template: %v", err))
}
// Render the message body.
var out = bytes.Buffer{}
if err := tpl.ExecuteTemplate(&out,
runner.BaseTPL,
runner.Message{Campaign: &camp, Subscriber: &sub, UnsubscribeURL: "#dummy"}); err != nil {
m := app.Runner.NewMessage(camp, &sub)
if err := m.Render(); err != nil {
return echo.NewHTTPError(http.StatusBadRequest,
fmt.Sprintf("Error executing template: %v", err))
fmt.Sprintf("Error rendering message: %v", err))
}
return c.HTML(http.StatusOK, out.String())
return c.HTML(http.StatusOK, string(m.Body))
}
// handleCreateCampaign handles campaign creation.
@ -479,14 +477,13 @@ func handleTestCampaign(c echo.Context) error {
// sendTestMessage takes a campaign and a subsriber and sends out a sample campain message.
func sendTestMessage(sub *models.Subscriber, camp *models.Campaign, app *App) error {
tpl, err := runner.CompileMessageTemplate(camp.TemplateBody, camp.Body)
if err != nil {
if err := camp.CompileTemplate(app.Runner.TemplateFuncs(camp)); err != nil {
return fmt.Errorf("Error compiling template: %v", err)
}
// Render the message body.
var out = bytes.Buffer{}
if err := tpl.ExecuteTemplate(&out,
if err := camp.Tpl.ExecuteTemplate(&out,
runner.BaseTPL,
runner.Message{Campaign: camp, Subscriber: sub, UnsubscribeURL: "#dummy"}); err != nil {
return fmt.Errorf("Error executing template: %v", err)

3
main.go
View File

@ -128,6 +128,7 @@ func registerHandlers(e *echo.Echo) {
// Subscriber facing views.
e.GET("/unsubscribe/:campUUID/:subUUID", handleUnsubscribePage)
e.POST("/unsubscribe/:campUUID/:subUUID", handleUnsubscribePage)
e.GET("/link/:linkUUID/:campUUID/:subUUID", handleLinkRedirect)
// Static views.
e.GET("/lists", handleIndexPage)
@ -223,6 +224,8 @@ func main() {
// url.com/unsubscribe/{campaign_uuid}/{subscriber_uuid}
UnsubscribeURL: fmt.Sprintf("%s/unsubscribe/%%s/%%s", app.Constants.RootURL),
// url.com/link/{campaign_uuid}/{subscriber_uuid}/{link_uuid}
LinkTrackURL: fmt.Sprintf("%s/link/%%s/%%s/%%s", app.Constants.RootURL),
}, newRunnerDB(q), logger)
app.Runner = r

41
models/models.go
View File

@ -5,6 +5,7 @@ import (
"encoding/json"
"fmt"
"html/template"
"regexp"
"github.com/jmoiron/sqlx"
"github.com/jmoiron/sqlx/types"
@ -37,6 +38,20 @@ const (
UserTypeUser = "user"
UserStatusEnabled = "enabled"
UserStatusDisabled = "disabled"
// BaseTpl is the name of the base template.
BaseTpl = "base"
// ContentTpl is the name of the compiled message.
ContentTpl = "content"
)
// Regular expression for matching {{ Track "http://link.com" }} in the template
// and substituting it with {{ Track "http://link.com" .Campaign.UUID .Subscriber.UUID }}
// before compilation. This string gimmick is to make linking easier for users.
var (
regexpLinkTag = regexp.MustCompile(`{{(\s+)?Track\s+?"(.+?)"(\s+)?}}`)
regexpLinkTagReplace = `{{ Track "$2" .Campaign.UUID .Subscriber.UUID }}`
)
// Base holds common fields shared across models.
@ -187,3 +202,29 @@ func (s SubscriberAttribs) Scan(src interface{}) error {
}
return fmt.Errorf("Could not not decode type %T -> %T", src, s)
}
// CompileTemplate compiles a campaign body template into its base
// template and sets the resultant template to Campaign.Tpl
func (c *Campaign) CompileTemplate(f template.FuncMap) error {
// Compile the base template.
t := regexpLinkTag.ReplaceAllString(c.TemplateBody, regexpLinkTagReplace)
baseTPL, err := template.New(BaseTpl).Funcs(f).Parse(t)
if err != nil {
return fmt.Errorf("error compiling base template: %v", err)
}
// Compile the campaign message.
t = regexpLinkTag.ReplaceAllString(c.Body, regexpLinkTagReplace)
msgTpl, err := template.New(ContentTpl).Funcs(f).Parse(t)
if err != nil {
return fmt.Errorf("error compiling message: %v", err)
}
out, err := baseTPL.AddParseTree(ContentTpl, msgTpl.Tree)
if err != nil {
return fmt.Errorf("error inserting child template: %v", err)
}
c.Tpl = out
return nil
}

3
queries.go
View File

@ -53,6 +53,9 @@ type Queries struct {
SetDefaultTemplate *sqlx.Stmt `query:"set-default-template"`
DeleteTemplate *sqlx.Stmt `query:"delete-template"`
CreateLink *sqlx.Stmt `query:"create-link"`
RegisterLinkClick *sqlx.Stmt `query:"register-link-click"`
// GetStats *sqlx.Stmt `query:"get-stats"`
}

18
queries.sql
View File

@ -90,7 +90,7 @@ sub AS (
WHERE uuid = $2 RETURNING id
)
UPDATE subscriber_lists SET status = 'unsubscribed' WHERE
subscriber_id = (SELECT id FROM sub) AND
subscriber_id = (SELECT id FROM sub) AND status != 'unsubscribed' AND
-- If $3 is false, unsubscribe from the campaign's lists, otherwise all lists.
CASE WHEN $3 IS FALSE THEN list_id = ANY(SELECT list_id FROM lists) ELSE list_id != 0 END;
@ -349,6 +349,22 @@ SELECT * FROM media ORDER BY created_at DESC;
-- name: delete-media
DELETE FROM media WHERE id=$1 RETURNING filename;
-- links
-- name: create-link
INSERT INTO links (uuid, url) VALUES($1, $2) ON CONFLICT (url) DO UPDATE SET url=EXCLUDED.url RETURNING uuid;
-- name: register-link-click
WITH link AS (
SELECT url, links.id AS link_id, campaigns.id as campaign_id, subscribers.id AS subscriber_id FROM links
LEFT JOIN campaigns ON (campaigns.uuid = $2)
LEFT JOIN subscribers ON (subscribers.uuid = $3)
WHERE links.uuid = $1
)
INSERT INTO link_clicks (campaign_id, subscriber_id, link_id)
VALUES((SELECT campaign_id FROM link), (SELECT subscriber_id FROM link), (SELECT link_id FROM link))
RETURNING (SELECT url FROM link);
-- -- name: get-stats
-- WITH lists AS (
-- SELECT type, COUNT(id) AS num FROM lists GROUP BY type

112
runner/runner.go
View File

@ -5,6 +5,7 @@ import (
"fmt"
"html/template"
"log"
"sync"
"time"
"github.com/knadh/listmonk/messenger"
@ -30,6 +31,7 @@ type DataSource interface {
PauseCampaign(campID int) error
CancelCampaign(campID int) error
FinishCampaign(campID int) error
CreateLink(url string) (string, error)
}
// Runner handles the scheduling, processing, and queuing of campaigns
@ -43,7 +45,13 @@ type Runner struct {
// Campaigns that are currently running.
camps map[int]*models.Campaign
msgQueue chan Message
// Links generated using Track() are cached here so as to not query
// the database for the link UUID for every message sent. This has to
// be locked as it may be used externally when previewing campaigns.
links map[string]string
linksMutex sync.RWMutex
msgQueue chan *Message
subFetchQueue chan *models.Campaign
}
@ -52,14 +60,14 @@ type Message struct {
Campaign *models.Campaign
Subscriber *models.Subscriber
UnsubscribeURL string
body []byte
to string
Body []byte
to string
}
// Config has parameters for configuring the runner.
type Config struct {
Concurrency int
LinkTrackURL string
UnsubscribeURL string
}
@ -70,14 +78,26 @@ func New(cfg Config, src DataSource, l *log.Logger) *Runner {
messengers: make(map[string]messenger.Messenger),
src: src,
camps: make(map[int]*models.Campaign, 0),
links: make(map[string]string, 0),
logger: l,
subFetchQueue: make(chan *models.Campaign, 100),
msgQueue: make(chan Message, cfg.Concurrency),
msgQueue: make(chan *Message, cfg.Concurrency),
}
return &r
}
// NewMessage creates and returns a Message that is made available
// to message templates while they're compiled.
func (r *Runner) NewMessage(c *models.Campaign, s *models.Subscriber) *Message {
return &Message{
to: s.Email,
Campaign: c,
Subscriber: s,
UnsubscribeURL: fmt.Sprintf(r.cfg.UnsubscribeURL, c.UUID, s.UUID),
}
}
// AddMessenger adds a Messenger messaging backend to the runner process.
func (r *Runner) AddMessenger(msg messenger.Messenger) error {
id := msg.Name()
@ -160,7 +180,7 @@ func (r *Runner) Run(tick time.Duration) {
// SpawnWorkers spawns workers goroutines that push out messages.
func (r *Runner) SpawnWorkers() {
for i := 0; i < r.cfg.Concurrency; i++ {
go func(ch chan Message) {
go func(ch chan *Message) {
for {
select {
case m := <-ch:
@ -168,21 +188,25 @@ func (r *Runner) SpawnWorkers() {
m.Campaign.FromEmail,
m.Subscriber.Email,
m.Campaign.Subject,
m.body)
m.Body)
}
}
}(r.msgQueue)
}
}
// TemplateFuncs returns the template functions to be applied into
// compiled campaign templates.
func (r *Runner) TemplateFuncs(c *models.Campaign) template.FuncMap {
return template.FuncMap{
"Track": func(url, campUUID, subUUID string) string {
return r.trackLink(url, campUUID, subUUID)
},
}
}
// addCampaign adds a campaign to the process queue.
func (r *Runner) addCampaign(c *models.Campaign) error {
var tplErr error
c.Tpl, tplErr = CompileMessageTemplate(c.TemplateBody, c.Body)
if tplErr != nil {
return tplErr
}
// Validate messenger.
if _, ok := r.messengers[c.MessengerID]; !ok {
@ -190,9 +214,13 @@ func (r *Runner) addCampaign(c *models.Campaign) error {
return fmt.Errorf("unknown messenger %s on campaign %s", c.MessengerID, c.Name)
}
// Load the template.
if err := c.CompileTemplate(r.TemplateFuncs(c)); err != nil {
return err
}
// Add the campaign to the active map.
r.camps[c.ID] = c
return nil
}
@ -225,17 +253,14 @@ func (r *Runner) nextSubscribers(c *models.Campaign, batchSize int) (bool, error
// Push messages.
for _, s := range subs {
to, body, err := r.makeMessage(c, s)
if err != nil {
r.logger.Printf("error preparing message (%s) (%s): %v", c.Name, s.Email, err)
m := r.NewMessage(c, s)
if err := m.Render(); err != nil {
r.logger.Printf("error rendering message (%s) (%s): %v", c.Name, s.Email, err)
continue
}
// Send the message.
r.msgQueue <- Message{Campaign: c,
Subscriber: s,
to: to,
body: body}
r.msgQueue <- m
}
return true, nil
@ -263,21 +288,40 @@ func (r *Runner) processExhaustedCampaign(c *models.Campaign) error {
return nil
}
// makeMessage prepares a campaign message for a subscriber and returns
// the 'to' address and the body.
func (r *Runner) makeMessage(c *models.Campaign, s *models.Subscriber) (string, []byte, error) {
// Render the message body.
var (
out = bytes.Buffer{}
tplMsg = Message{Campaign: c,
Subscriber: s,
UnsubscribeURL: fmt.Sprintf(r.cfg.UnsubscribeURL, c.UUID, s.UUID)}
)
if err := c.Tpl.ExecuteTemplate(&out, BaseTPL, tplMsg); err != nil {
return "", nil, err
// Render takes a Message, executes its pre-compiled Campaign.Tpl
// and applies the resultant bytes to Message.body to be used in messages.
func (m *Message) Render() error {
out := bytes.Buffer{}
if err := m.Campaign.Tpl.ExecuteTemplate(&out, models.BaseTpl, m); err != nil {
return err
}
m.Body = out.Bytes()
return nil
}
// trackLink register a URL and return its UUID to be used in message templates
// for tracking links.
func (r *Runner) trackLink(url, campUUID, subUUID string) string {
r.linksMutex.RLock()
if uu, ok := r.links[url]; ok {
return uu
}
r.linksMutex.RUnlock()
// Register link.
uu, err := r.src.CreateLink(url)
if err != nil {
r.logger.Printf("error registering tracking for link '%s': %v", url, err)
// If the registration fails, fail over to the original URL.
return url
}
return s.Email, out.Bytes(), nil
r.linksMutex.Lock()
r.links[url] = uu
r.linksMutex.Unlock()
return fmt.Sprintf(r.cfg.LinkTrackURL, uu, campUUID, subUUID)
}
// CompileMessageTemplate takes a base template body string and a child (message) template

13
runner_db.go
View File

@ -3,6 +3,7 @@ package main
import (
"github.com/knadh/listmonk/models"
"github.com/lib/pq"
uuid "github.com/satori/go.uuid"
)
// runnerDB implements runner.DataSource over the primary
@ -58,3 +59,15 @@ func (r *runnerDB) FinishCampaign(campID int) error {
_, err := r.queries.UpdateCampaignStatus.Exec(campID, models.CampaignStatusFinished)
return err
}
// CreateLink registers a URL with a UUID for tracking clicks and returns the UUID.
func (r *runnerDB) CreateLink(url string) (string, error) {
// Create a new UUID for the URL. If the URL already exists in the DB
// the UUID in the database is returned.
var uu string
if err := r.queries.CreateLink.Get(&uu, uuid.NewV4(), url); err != nil {
return "", err
}
return uu, nil
}

6
schema.sql
View File

@ -128,7 +128,6 @@ CREATE UNIQUE INDEX ON campaign_lists (campaign_id, list_id);
DROP TABLE IF EXISTS campaign_views CASCADE;
CREATE TABLE campaign_views (
id SERIAL PRIMARY KEY,
campaign_id INTEGER REFERENCES campaigns(id) ON DELETE CASCADE ON UPDATE CASCADE,
-- Subscribers may be deleted, but the link counts should remain.
@ -153,15 +152,14 @@ DROP TABLE IF EXISTS links CASCADE;
CREATE TABLE links (
id SERIAL PRIMARY KEY,
uuid uuid NOT NULL UNIQUE,
url TEXT NOT NULL,
url TEXT NOT NULL UNIQUE,
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
);
DROP TABLE IF EXISTS link_clicks CASCADE;
CREATE TABLE link_clicks (
id SERIAL PRIMARY KEY,
campaign_id INTEGER REFERENCES campaigns(id) ON DELETE CASCADE ON UPDATE CASCADE,
link_id INTEGER NULL REFERENCES links(id) ON DELETE CASCADE ON UPDATE CASCADE,
link_id INTEGER REFERENCES links(id) ON DELETE CASCADE ON UPDATE CASCADE,
-- Subscribers may be deleted, but the link counts should remain.
subscriber_id INTEGER NULL REFERENCES subscribers(id) ON DELETE SET NULL ON UPDATE CASCADE,

11
templates.go
View File

@ -76,7 +76,12 @@ func handlePreviewTemplate(c echo.Context) error {
tpls []models.Template
)
if body == "" {
if body != "" {
if strings.Count(body, tplTag) != 1 {
return echo.NewHTTPError(http.StatusBadRequest,
fmt.Sprintf("Template body should contain the %s placeholder exactly once", tplTag))
}
} else {
if id < 1 {
return echo.NewHTTPError(http.StatusBadRequest, "Invalid ID.")
}
@ -227,8 +232,8 @@ func validateTemplate(o models.Template) error {
return errors.New("invalid length for `name`")
}
if !strings.Contains(o.Body, tplTag) {
return fmt.Errorf("template body should contain the %s placeholder", tplTag)
if strings.Count(o.Body, tplTag) != 1 {
return fmt.Errorf("template body should contain the %s placeholder exactly once", tplTag)
}
return nil

14
utils.go
View File

@ -245,3 +245,17 @@ func normalizeTags(tags []string) []string {
return out
}
// makeErrorTpl takes error details and returns an errorTpl
// with the error details applied to be rendered in an HTML view.
func makeErrorTpl(pageTitle, heading, desc string) errorTpl {
if heading == "" {
heading = pageTitle
}
err := errorTpl{}
err.Title = pageTitle
err.ErrorTitle = heading
err.ErrorMessage = desc
return err
}