- Refactor and move template compilation from runner to models.Campaign to support adhoc template funcs

- Add support for {{ Track "https://url.com" }} in templates to register and track links

campaigns.go

@@ -95,14 +95,14 @@ func handlePreviewCampaign(c echo.Context) error {
 		id, _ = strconv.Atoi(c.Param("id"))
 		body  = c.FormValue("body")
 
-		camp models.Campaign
+		camp = &models.Campaign{}
 	)
 
 	if id < 1 {
 		return echo.NewHTTPError(http.StatusBadRequest, "Invalid ID.")
 	}
 
-	err := app.Queries.GetCampaignForPreview.Get(&camp, id)
+	err := app.Queries.GetCampaignForPreview.Get(camp, id)
 	if err != nil {
 		if err == sql.ErrNoRows {
 			return echo.NewHTTPError(http.StatusBadRequest, "Campaign not found.")
@@ -130,25 +130,23 @@ func handlePreviewCampaign(c echo.Context) error {
 	}
 
 	// Compile the template.
-	if body == "" {
+	if body != "" {
-		body = camp.Body
+		camp.Body = body
 	}
-	tpl, err := runner.CompileMessageTemplate(camp.TemplateBody, body)
+
-	if err != nil {
+	if err := camp.CompileTemplate(app.Runner.TemplateFuncs(camp)); err != nil {
 		return echo.NewHTTPError(http.StatusBadRequest,
 			fmt.Sprintf("Error compiling template: %v", err))
 	}
 
 	// Render the message body.
-	var out = bytes.Buffer{}
+	m := app.Runner.NewMessage(camp, &sub)
-	if err := tpl.ExecuteTemplate(&out,
+	if err := m.Render(); err != nil {
-		runner.BaseTPL,
-		runner.Message{Campaign: &camp, Subscriber: &sub, UnsubscribeURL: "#dummy"}); err != nil {
 		return echo.NewHTTPError(http.StatusBadRequest,
-			fmt.Sprintf("Error executing template: %v", err))
+			fmt.Sprintf("Error rendering message: %v", err))
 	}
 
-	return c.HTML(http.StatusOK, out.String())
+	return c.HTML(http.StatusOK, string(m.Body))
 }
 
 // handleCreateCampaign handles campaign creation.
@@ -479,14 +477,13 @@ func handleTestCampaign(c echo.Context) error {
 
 // sendTestMessage takes a campaign and a subsriber and sends out a sample campain message.
 func sendTestMessage(sub *models.Subscriber, camp *models.Campaign, app *App) error {
-	tpl, err := runner.CompileMessageTemplate(camp.TemplateBody, camp.Body)
+	if err := camp.CompileTemplate(app.Runner.TemplateFuncs(camp)); err != nil {
-	if err != nil {
 		return fmt.Errorf("Error compiling template: %v", err)
 	}
 
 	// Render the message body.
 	var out = bytes.Buffer{}
-	if err := tpl.ExecuteTemplate(&out,
+	if err := camp.Tpl.ExecuteTemplate(&out,
 		runner.BaseTPL,
 		runner.Message{Campaign: camp, Subscriber: sub, UnsubscribeURL: "#dummy"}); err != nil {
 		return fmt.Errorf("Error executing template: %v", err)

main.go

@@ -128,6 +128,7 @@ func registerHandlers(e *echo.Echo) {
 	// Subscriber facing views.
 	e.GET("/unsubscribe/:campUUID/:subUUID", handleUnsubscribePage)
 	e.POST("/unsubscribe/:campUUID/:subUUID", handleUnsubscribePage)
+	e.GET("/link/:linkUUID/:campUUID/:subUUID", handleLinkRedirect)
 
 	// Static views.
 	e.GET("/lists", handleIndexPage)
@@ -223,6 +224,8 @@ func main() {
 
 		// url.com/unsubscribe/{campaign_uuid}/{subscriber_uuid}
 		UnsubscribeURL: fmt.Sprintf("%s/unsubscribe/%%s/%%s", app.Constants.RootURL),
+		// url.com/link/{campaign_uuid}/{subscriber_uuid}/{link_uuid}
+		LinkTrackURL: fmt.Sprintf("%s/link/%%s/%%s/%%s", app.Constants.RootURL),
 	}, newRunnerDB(q), logger)
 	app.Runner = r
 

models/models.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"html/template"
+	"regexp"
 
 	"github.com/jmoiron/sqlx"
 	"github.com/jmoiron/sqlx/types"
@@ -37,6 +38,20 @@ const (
 	UserTypeUser       = "user"
 	UserStatusEnabled  = "enabled"
 	UserStatusDisabled = "disabled"
+
+	// BaseTpl is the name of the base template.
+	BaseTpl = "base"
+
+	// ContentTpl is the name of the compiled message.
+	ContentTpl = "content"
+)
+
+// Regular expression for matching {{ Track "http://link.com" }} in the template
+// and substituting it with {{ Track "http://link.com" .Campaign.UUID .Subscriber.UUID }}
+// before compilation. This string gimmick is to make linking easier for users.
+var (
+	regexpLinkTag        = regexp.MustCompile(`{{(\s+)?Track\s+?"(.+?)"(\s+)?}}`)
+	regexpLinkTagReplace = `{{ Track "$2" .Campaign.UUID .Subscriber.UUID }}`
 )
 
 // Base holds common fields shared across models.
@@ -187,3 +202,29 @@ func (s SubscriberAttribs) Scan(src interface{}) error {
 	}
 	return fmt.Errorf("Could not not decode type %T -> %T", src, s)
 }
+
+// CompileTemplate compiles a campaign body template into its base
+// template and sets the resultant template to Campaign.Tpl
+func (c *Campaign) CompileTemplate(f template.FuncMap) error {
+	// Compile the base template.
+	t := regexpLinkTag.ReplaceAllString(c.TemplateBody, regexpLinkTagReplace)
+	baseTPL, err := template.New(BaseTpl).Funcs(f).Parse(t)
+	if err != nil {
+		return fmt.Errorf("error compiling base template: %v", err)
+	}
+
+	// Compile the campaign message.
+	t = regexpLinkTag.ReplaceAllString(c.Body, regexpLinkTagReplace)
+	msgTpl, err := template.New(ContentTpl).Funcs(f).Parse(t)
+	if err != nil {
+		return fmt.Errorf("error compiling message: %v", err)
+	}
+
+	out, err := baseTPL.AddParseTree(ContentTpl, msgTpl.Tree)
+	if err != nil {
+		return fmt.Errorf("error inserting child template: %v", err)
+	}
+
+	c.Tpl = out
+	return nil
+}

queries.go

@@ -53,6 +53,9 @@ type Queries struct {
 	SetDefaultTemplate *sqlx.Stmt `query:"set-default-template"`
 	DeleteTemplate     *sqlx.Stmt `query:"delete-template"`
 
+	CreateLink        *sqlx.Stmt `query:"create-link"`
+	RegisterLinkClick *sqlx.Stmt `query:"register-link-click"`
+
 	// GetStats *sqlx.Stmt `query:"get-stats"`
 }
 

queries.sql

@@ -90,7 +90,7 @@ sub AS (
     WHERE uuid = $2 RETURNING id
 )
 UPDATE subscriber_lists SET status = 'unsubscribed' WHERE
-    subscriber_id = (SELECT id FROM sub) AND
+    subscriber_id = (SELECT id FROM sub) AND status != 'unsubscribed' AND
     -- If $3 is false, unsubscribe from the campaign's lists, otherwise all lists.
     CASE WHEN $3 IS FALSE THEN list_id = ANY(SELECT list_id FROM lists) ELSE list_id != 0 END;
 
@@ -349,6 +349,22 @@ SELECT * FROM media ORDER BY created_at DESC;
 -- name: delete-media
 DELETE FROM media WHERE id=$1 RETURNING filename;
 
+-- links
+-- name: create-link
+INSERT INTO links (uuid, url) VALUES($1, $2) ON CONFLICT (url) DO UPDATE SET url=EXCLUDED.url RETURNING uuid;
+
+-- name: register-link-click
+WITH link AS (
+    SELECT url, links.id AS link_id, campaigns.id as campaign_id, subscribers.id AS subscriber_id FROM links
+    LEFT JOIN campaigns ON (campaigns.uuid = $2)
+    LEFT JOIN subscribers ON (subscribers.uuid = $3)
+    WHERE links.uuid = $1
+)
+INSERT INTO link_clicks (campaign_id, subscriber_id, link_id)
+    VALUES((SELECT campaign_id FROM link), (SELECT subscriber_id FROM link), (SELECT link_id FROM link))
+    RETURNING (SELECT url FROM link);
+
+
 -- -- name: get-stats
 -- WITH lists AS (
 --     SELECT type, COUNT(id) AS num FROM lists GROUP BY type

runner/runner.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"html/template"
 	"log"
+	"sync"
 	"time"
 
 	"github.com/knadh/listmonk/messenger"
@@ -30,6 +31,7 @@ type DataSource interface {
 	PauseCampaign(campID int) error
 	CancelCampaign(campID int) error
 	FinishCampaign(campID int) error
+	CreateLink(url string) (string, error)
 }
 
 // Runner handles the scheduling, processing, and queuing of campaigns
@@ -43,7 +45,13 @@ type Runner struct {
 	// Campaigns that are currently running.
 	camps map[int]*models.Campaign
 
-	msgQueue      chan Message
+	// Links generated using Track() are cached here so as to not query
+	// the database for the link UUID for every message sent. This has to
+	// be locked as it may be used externally when previewing campaigns.
+	links      map[string]string
+	linksMutex sync.RWMutex
+
+	msgQueue      chan *Message
 	subFetchQueue chan *models.Campaign
 }
 
@@ -52,14 +60,14 @@ type Message struct {
 	Campaign       *models.Campaign
 	Subscriber     *models.Subscriber
 	UnsubscribeURL string
-
+	Body           []byte
-	body []byte
 	to             string
 }
 
 // Config has parameters for configuring the runner.
 type Config struct {
 	Concurrency    int
+	LinkTrackURL   string
 	UnsubscribeURL string
 }
 
@@ -70,14 +78,26 @@ func New(cfg Config, src DataSource, l *log.Logger) *Runner {
 		messengers:    make(map[string]messenger.Messenger),
 		src:           src,
 		camps:         make(map[int]*models.Campaign, 0),
+		links:         make(map[string]string, 0),
 		logger:        l,
 		subFetchQueue: make(chan *models.Campaign, 100),
-		msgQueue:      make(chan Message, cfg.Concurrency),
+		msgQueue:      make(chan *Message, cfg.Concurrency),
 	}
 
 	return &r
 }
 
+// NewMessage creates and returns a Message that is made available
+// to message templates while they're compiled.
+func (r *Runner) NewMessage(c *models.Campaign, s *models.Subscriber) *Message {
+	return &Message{
+		to:             s.Email,
+		Campaign:       c,
+		Subscriber:     s,
+		UnsubscribeURL: fmt.Sprintf(r.cfg.UnsubscribeURL, c.UUID, s.UUID),
+	}
+}
+
 // AddMessenger adds a Messenger messaging backend to the runner process.
 func (r *Runner) AddMessenger(msg messenger.Messenger) error {
 	id := msg.Name()
@@ -160,7 +180,7 @@ func (r *Runner) Run(tick time.Duration) {
 // SpawnWorkers spawns workers goroutines that push out messages.
 func (r *Runner) SpawnWorkers() {
 	for i := 0; i < r.cfg.Concurrency; i++ {
-		go func(ch chan Message) {
+		go func(ch chan *Message) {
 			for {
 				select {
 				case m := <-ch:
@@ -168,21 +188,25 @@ func (r *Runner) SpawnWorkers() {
 						m.Campaign.FromEmail,
 						m.Subscriber.Email,
 						m.Campaign.Subject,
-						m.body)
+						m.Body)
 				}
 			}
 		}(r.msgQueue)
 	}
 }
 
+// TemplateFuncs returns the template functions to be applied into
+// compiled campaign templates.
+func (r *Runner) TemplateFuncs(c *models.Campaign) template.FuncMap {
+	return template.FuncMap{
+		"Track": func(url, campUUID, subUUID string) string {
+			return r.trackLink(url, campUUID, subUUID)
+		},
+	}
+}
+
 // addCampaign adds a campaign to the process queue.
 func (r *Runner) addCampaign(c *models.Campaign) error {
-	var tplErr error
-
-	c.Tpl, tplErr = CompileMessageTemplate(c.TemplateBody, c.Body)
-	if tplErr != nil {
-		return tplErr
-	}
 
 	// Validate messenger.
 	if _, ok := r.messengers[c.MessengerID]; !ok {
@@ -190,9 +214,13 @@ func (r *Runner) addCampaign(c *models.Campaign) error {
 		return fmt.Errorf("unknown messenger %s on campaign %s", c.MessengerID, c.Name)
 	}
 
+	// Load the template.
+	if err := c.CompileTemplate(r.TemplateFuncs(c)); err != nil {
+		return err
+	}
+
 	// Add the campaign to the active map.
 	r.camps[c.ID] = c
-
 	return nil
 }
 
@@ -225,17 +253,14 @@ func (r *Runner) nextSubscribers(c *models.Campaign, batchSize int) (bool, error
 
 	// Push messages.
 	for _, s := range subs {
-		to, body, err := r.makeMessage(c, s)
+		m := r.NewMessage(c, s)
-		if err != nil {
+		if err := m.Render(); err != nil {
-			r.logger.Printf("error preparing message (%s) (%s): %v", c.Name, s.Email, err)
+			r.logger.Printf("error rendering message (%s) (%s): %v", c.Name, s.Email, err)
 			continue
 		}
 
 		// Send the message.
-		r.msgQueue <- Message{Campaign: c,
+		r.msgQueue <- m
-			Subscriber: s,
-			to:         to,
-			body:       body}
 	}
 
 	return true, nil
@@ -263,21 +288,40 @@ func (r *Runner) processExhaustedCampaign(c *models.Campaign) error {
 	return nil
 }
 
-// makeMessage prepares a campaign message for a subscriber and returns
+// Render takes a Message, executes its pre-compiled Campaign.Tpl
-// the 'to' address and the body.
+// and applies the resultant bytes to Message.body to be used in messages.
-func (r *Runner) makeMessage(c *models.Campaign, s *models.Subscriber) (string, []byte, error) {
+func (m *Message) Render() error {
-	// Render the message body.
+	out := bytes.Buffer{}
-	var (
+	if err := m.Campaign.Tpl.ExecuteTemplate(&out, models.BaseTpl, m); err != nil {
-		out    = bytes.Buffer{}
+		return err
-		tplMsg = Message{Campaign: c,
+	}
-			Subscriber:     s,
+	m.Body = out.Bytes()
-			UnsubscribeURL: fmt.Sprintf(r.cfg.UnsubscribeURL, c.UUID, s.UUID)}
+	return nil
-	)
+}
-	if err := c.Tpl.ExecuteTemplate(&out, BaseTPL, tplMsg); err != nil {
+
-		return "", nil, err
+// trackLink register a URL and return its UUID to be used in message templates
+// for tracking links.
+func (r *Runner) trackLink(url, campUUID, subUUID string) string {
+	r.linksMutex.RLock()
+	if uu, ok := r.links[url]; ok {
+		return uu
+	}
+	r.linksMutex.RUnlock()
+
+	// Register link.
+	uu, err := r.src.CreateLink(url)
+	if err != nil {
+		r.logger.Printf("error registering tracking for link '%s': %v", url, err)
+
+		// If the registration fails, fail over to the original URL.
+		return url
 	}
 
-	return s.Email, out.Bytes(), nil
+	r.linksMutex.Lock()
+	r.links[url] = uu
+	r.linksMutex.Unlock()
+
+	return fmt.Sprintf(r.cfg.LinkTrackURL, uu, campUUID, subUUID)
 }
 
 // CompileMessageTemplate takes a base template body string and a child (message) template

runner_db.go

@@ -3,6 +3,7 @@ package main
 import (
 	"github.com/knadh/listmonk/models"
 	"github.com/lib/pq"
+	uuid "github.com/satori/go.uuid"
 )
 
 // runnerDB implements runner.DataSource over the primary
@@ -58,3 +59,15 @@ func (r *runnerDB) FinishCampaign(campID int) error {
 	_, err := r.queries.UpdateCampaignStatus.Exec(campID, models.CampaignStatusFinished)
 	return err
 }
+
+// CreateLink registers a URL with a UUID for tracking clicks and returns the UUID.
+func (r *runnerDB) CreateLink(url string) (string, error) {
+	// Create a new UUID for the URL. If the URL already exists in the DB
+	// the UUID in the database is returned.
+	var uu string
+	if err := r.queries.CreateLink.Get(&uu, uuid.NewV4(), url); err != nil {
+		return "", err
+	}
+
+	return uu, nil
+}

schema.sql

@@ -128,7 +128,6 @@ CREATE UNIQUE INDEX ON campaign_lists (campaign_id, list_id);
 
 DROP TABLE IF EXISTS campaign_views CASCADE;
 CREATE TABLE campaign_views (
-    id               SERIAL PRIMARY KEY,
     campaign_id      INTEGER REFERENCES campaigns(id) ON DELETE CASCADE ON UPDATE CASCADE,
 
     -- Subscribers may be deleted, but the link counts should remain.
@@ -153,15 +152,14 @@ DROP TABLE IF EXISTS links CASCADE;
 CREATE TABLE links (
     id               SERIAL PRIMARY KEY,
     uuid uuid        NOT NULL UNIQUE,
-    url              TEXT NOT NULL,
+    url              TEXT NOT NULL UNIQUE,
     created_at       TIMESTAMP WITH TIME ZONE DEFAULT NOW()
 );
 
 DROP TABLE IF EXISTS link_clicks CASCADE;
 CREATE TABLE link_clicks (
-    id               SERIAL PRIMARY KEY,
     campaign_id      INTEGER REFERENCES campaigns(id) ON DELETE CASCADE ON UPDATE CASCADE,
-    link_id          INTEGER NULL REFERENCES links(id) ON DELETE CASCADE ON UPDATE CASCADE,
+    link_id          INTEGER REFERENCES links(id) ON DELETE CASCADE ON UPDATE CASCADE,
 
     -- Subscribers may be deleted, but the link counts should remain.
     subscriber_id    INTEGER NULL REFERENCES subscribers(id) ON DELETE SET NULL ON UPDATE CASCADE,

templates.go

@@ -76,7 +76,12 @@ func handlePreviewTemplate(c echo.Context) error {
 		tpls []models.Template
 	)
 
-	if body == "" {
+	if body != "" {
+		if strings.Count(body, tplTag) != 1 {
+			return echo.NewHTTPError(http.StatusBadRequest,
+				fmt.Sprintf("Template body should contain the %s placeholder exactly once", tplTag))
+		}
+	} else {
 		if id < 1 {
 			return echo.NewHTTPError(http.StatusBadRequest, "Invalid ID.")
 		}
@@ -227,8 +232,8 @@ func validateTemplate(o models.Template) error {
 		return errors.New("invalid length for `name`")
 	}
 
-	if !strings.Contains(o.Body, tplTag) {
+	if strings.Count(o.Body, tplTag) != 1 {
-		return fmt.Errorf("template body should contain the %s placeholder", tplTag)
+		return fmt.Errorf("template body should contain the %s placeholder exactly once", tplTag)
 	}
 
 	return nil

utils.go

@@ -245,3 +245,17 @@ func normalizeTags(tags []string) []string {
 
 	return out
 }
+
+// makeErrorTpl takes error details and returns an errorTpl
+// with the error details applied to be rendered in an HTML view.
+func makeErrorTpl(pageTitle, heading, desc string) errorTpl {
+	if heading == "" {
+		heading = pageTitle
+	}
+	err := errorTpl{}
+	err.Title = pageTitle
+	err.ErrorTitle = heading
+	err.ErrorMessage = desc
+
+	return err
+}