Improve campaign content format conversion.

Previously, converting between formats simply copied over raw content.
This update does actual conversion between different formats. While
lossy, this seems to a good enough approximation for even reasonbly
rich HTML content. Closes #348.

- richtext, html => plain
  Strips HTML and converts content to plain text.

- richtext, html => markdown
  Uses turndown (JS) lib to convert HTML to Markdown.

- plain => richtext, html
  Converts line breaks in plain text to HTML breaks.

- richtext => html
  "Beautifies" the HTML generated by the WYSIWYG editor unlike the
  earlier behaviour of dumping one long line of HTML.

- markdown => richtext, html
  Makes an API call to the backend to use the Goldmark lib to convert
  Markdown to HTML.

cmd/campaigns.go

@@ -14,7 +14,6 @@ import (
 	"time"
 
 	"github.com/gofrs/uuid"
-	"github.com/jaytaylor/html2text"
 	"github.com/knadh/listmonk/internal/messenger"
 	"github.com/knadh/listmonk/internal/subimporter"
 	"github.com/knadh/listmonk/models"
@@ -23,7 +22,8 @@ import (
 	null "gopkg.in/volatiletech/null.v6"
 )
 
-// campaignReq is a wrapper over the Campaign model.
+// campaignReq is a wrapper over the Campaign model for receiving
+// campaign creation and updation data from APIs.
 type campaignReq struct {
 	models.Campaign
 
@@ -42,6 +42,14 @@ type campaignReq struct {
 	Type string `json:"type"`
 }
 
+// campaignContentReq wraps params coming from API requests for converting
+// campaign content formats.
+type campaignContentReq struct {
+	models.Campaign
+	From string `json:"from"`
+	To   string `json:"to"`
+}
+
 type campaignStats struct {
 	ID        int       `db:"id" json:"id"`
 	Status    string    `db:"status" json:"status"`
@@ -201,15 +209,28 @@ func handlePreviewCampaign(c echo.Context) error {
 	return c.HTML(http.StatusOK, string(m.Body()))
 }
 
-// handleCampainBodyToText converts an HTML campaign body to plaintext.
+// handleCampaignContent handles campaign content (body) format conversions.
-func handleCampainBodyToText(c echo.Context) error {
+func handleCampaignContent(c echo.Context) error {
-	out, err := html2text.FromString(c.FormValue("body"),
+	var (
-		html2text.Options{PrettyTables: false})
+		app   = c.Get("app").(*App)
-	if err != nil {
+		id, _ = strconv.Atoi(c.Param("id"))
+	)
+
+	if id < 1 {
+		return echo.NewHTTPError(http.StatusBadRequest, app.i18n.T("globals.messages.invalidID"))
+	}
+
+	var camp campaignContentReq
+	if err := c.Bind(&camp); err != nil {
 		return err
 	}
 
-	return c.HTML(http.StatusOK, string(out))
+	out, err := camp.ConvertContent(camp.From, camp.To)
+	if err != nil {
+		return echo.NewHTTPError(http.StatusBadRequest, err.Error())
+	}
+
+	return c.JSON(http.StatusOK, okResp{out})
 }
 
 // handleCreateCampaign handles campaign creation.

cmd/handlers.go

@@ -97,6 +97,7 @@ func registerHTTPHandlers(e *echo.Echo, app *App) {
 	g.GET("/api/campaigns/:id", handleGetCampaigns)
 	g.GET("/api/campaigns/:id/preview", handlePreviewCampaign)
 	g.POST("/api/campaigns/:id/preview", handlePreviewCampaign)
+	g.POST("/api/campaigns/:id/content", handleCampaignContent)
 	g.POST("/api/campaigns/:id/text", handlePreviewCampaign)
 	g.POST("/api/campaigns/:id/test", handleTestCampaign)
 	g.POST("/api/campaigns", handleCreateCampaign)

frontend/package.json

@@ -25,6 +25,7 @@
     "quill-delta": "^4.2.2",
     "sass-loader": "^8.0.2",
     "textversionjs": "^1.1.3",
+    "turndown": "^7.0.0",
     "vue": "^2.6.11",
     "vue-c3": "^1.2.11",
     "vue-i18n": "^8.22.2",

frontend/src/api/index.js

@@ -156,6 +156,9 @@ export const getCampaignStats = async () => http.get('/api/campaigns/running/sta
 export const createCampaign = async (data) => http.post('/api/campaigns', data,
   { loading: models.campaigns });
 
+export const convertCampaignContent = async (data) => http.post(`/api/campaigns/${data.id}/content`, data,
+  { loading: models.campaigns });
+
 export const testCampaign = async (data) => http.post(`/api/campaigns/${data.id}/test`, data,
   { loading: models.campaigns });
 

frontend/src/components/Editor.vue

@@ -78,6 +78,7 @@ import 'quill/dist/quill.core.css';
 
 import { quillEditor, Quill } from 'vue-quill-editor';
 import CodeFlask from 'codeflask';
+import TurndownService from 'turndown';
 
 import CampaignPreview from './CampaignPreview.vue';
 import Media from '../views/Media.vue';
@@ -98,6 +99,8 @@ const regLink = new RegExp(/{{(\s+)?TrackLink(\s+)?"(.+?)"(\s+)?}}/);
 const Link = Quill.import('formats/link');
 Link.sanitize = (l) => l.replace(regLink, '{{ TrackLink `$3`}}');
 
+const turndown = new TurndownService();
+
 // Custom class to override the default indent behaviour to get inline CSS
 // style instead of classes.
 class IndentAttributor extends Quill.import('parchment').Attributor.Style {
@@ -191,6 +194,9 @@ export default {
           },
         },
       },
+
+      // HTML editor.
+      flask: null,
     };
   },
 
@@ -241,22 +247,25 @@ export default {
       `;
       this.$refs.htmlEditor.appendChild(el);
 
-      const flask = new CodeFlask(el.shadowRoot.getElementById('area'), {
+      this.flask = new CodeFlask(el.shadowRoot.getElementById('area'), {
         language: 'html',
         lineNumbers: false,
         styleParent: el.shadowRoot,
         readonly: this.disabled,
       });
-
+      this.flask.onUpdate((b) => {
-      flask.updateCode(this.form.body);
-      flask.onUpdate((b) => {
         this.form.body = b;
         this.$emit('input', { contentType: this.form.format, body: this.form.body });
       });
 
+      this.updateHTMLEditor();
       this.isReady = true;
     },
 
+    updateHTMLEditor() {
+      this.flask.updateCode(this.form.body);
+    },
+
     onTogglePreview() {
       this.isPreviewing = !this.isPreviewing;
     },
@@ -278,6 +287,46 @@ export default {
     onMediaSelect(m) {
       this.$refs.quill.quill.insertEmbed(this.lastSel.index || 0, 'image', m.url);
     },
+
+    beautifyHTML(str) {
+      const div = document.createElement('div');
+      div.innerHTML = str.trim();
+      return this.formatHTMLNode(div, 0).innerHTML;
+    },
+
+    formatHTMLNode(node, level) {
+      const lvl = level + 1;
+      const indentBefore = new Array(lvl + 1).join('  ');
+      const indentAfter = new Array(lvl - 1).join('  ');
+      let textNode = null;
+
+      for (let i = 0; i < node.children.length; i += 1) {
+        textNode = document.createTextNode(`\n${indentBefore}`);
+        node.insertBefore(textNode, node.children[i]);
+
+        this.formatHTMLNode(node.children[i], lvl);
+        if (node.lastElementChild === node.children[i]) {
+          textNode = document.createTextNode(`\n${indentAfter}`);
+          node.appendChild(textNode);
+        }
+      }
+
+      return node;
+    },
+
+    trimLines(str, removeEmptyLines) {
+      const out = str.split('\n');
+      for (let i = 0; i < out.length; i += 1) {
+        const line = out[i].trim();
+        if (removeEmptyLines) {
+          out[i] = line;
+        } else if (line === '') {
+          out[i] = '';
+        }
+      }
+
+      return out.join('\n').replace(/\n\s*\n\s*\n/g, '\n\n');
+    },
   },
 
   computed: {
@@ -306,14 +355,45 @@ export default {
       this.onEditorChange();
     },
 
-    htmlFormat(f) {
+    htmlFormat(to, from) {
-      if (f !== 'html') {
+      // On switch to HTML, initialize the HTML editor.
-        return;
+      if (to === 'html') {
+        this.$nextTick(() => {
+          this.initHTMLEditor();
+        });
       }
 
-      this.$nextTick(() => {
+      if ((from === 'richtext' || from === 'html') && to === 'plain') {
-        this.initHTMLEditor();
+        // richtext, html => plain
-      });
+
+        // Preserve line breaks when converting HTML to plaintext. Quill produces
+        // HTML without any linebreaks.
+        const d = document.createElement('div');
+        d.innerHTML = this.beautifyHTML(this.form.body);
+        this.form.body = this.trimLines(d.innerText.trim(), true);
+      } else if ((from === 'richtext' || from === 'html') && to === 'markdown') {
+        // richtext, html => markdown
+        this.form.body = turndown.turndown(this.form.body).replace(/\n\n+/ig, '\n\n');
+      } else if (from === 'plain' && (to === 'richtext' || to === 'html')) {
+        // plain => richtext, html
+        this.form.body = this.form.body.replace(/\n/ig, '<br>\n');
+      } else if (from === 'richtext' && to === 'html') {
+        // richtext => html
+        this.form.body = this.trimLines(this.beautifyHTML(this.form.body), false);
+      } else if (from === 'markdown' && (to === 'richtext' || to === 'html')) {
+        // markdown => richtext, html.
+        this.$api.convertCampaignContent({
+          id: 1, body: this.form.body, from, to,
+        }).then((data) => {
+          this.form.body = this.beautifyHTML(data.trim());
+          // Update the HTML editor.
+          if (to === 'html') {
+            this.updateHTMLEditor();
+          }
+        });
+      }
+
+      this.onEditorChange();
     },
   },
 

frontend/yarn.lock

@@ -3641,6 +3641,11 @@ domhandler@^2.3.0:
   dependencies:
     domelementtype "1"
 
+domino@^2.1.6:
+  version "2.1.6"
+  resolved "https://registry.yarnpkg.com/domino/-/domino-2.1.6.tgz#fe4ace4310526e5e7b9d12c7de01b7f485a57ffe"
+  integrity sha512-3VdM/SXBZX2omc9JF9nOPCtDaYQ67BGp5CoLpIQlO2KCAPETs8TcDHacF26jXadGbvUteZzRTeos2fhID5+ucQ==
+
 domutils@1.5.1:
   version "1.5.1"
   resolved "https://registry.yarnpkg.com/domutils/-/domutils-1.5.1.tgz#dcd8488a26f563d61079e48c9f7b7e32373682cf"
@@ -9253,6 +9258,13 @@ tunnel-agent@^0.6.0:
   dependencies:
     safe-buffer "^5.0.1"
 
+turndown@^7.0.0:
+  version "7.0.0"
+  resolved "https://registry.yarnpkg.com/turndown/-/turndown-7.0.0.tgz#19b2a6a2d1d700387a1e07665414e4af4fec5225"
+  integrity sha512-G1FfxfR0mUNMeGjszLYl3kxtopC4O9DRRiMlMDDVHvU1jaBkGFg4qxIyjIk2aiKLHyDyZvZyu4qBO2guuYBy3Q==
+  dependencies:
+    domino "^2.1.6"
+
 tweetnacl@^0.14.3, tweetnacl@~0.14.0:
   version "0.14.5"
   resolved "https://registry.yarnpkg.com/tweetnacl/-/tweetnacl-0.14.5.tgz#5ae68177f192d4456269d108afa93ff8743f4f64"

models/models.go

@@ -383,6 +383,30 @@ func (c *Campaign) CompileTemplate(f template.FuncMap) error {
 	return nil
 }
 
+// ConvertContent converts a campaign's body from one format to another,
+// for example, Markdown to HTML.
+func (c *Campaign) ConvertContent(from, to string) (string, error) {
+	body := c.Body
+	for _, r := range regTplFuncs {
+		body = r.regExp.ReplaceAllString(body, r.replace)
+	}
+
+	// If the format is markdown, convert Markdown to HTML.
+	var out string
+	if from == CampaignContentTypeMarkdown &&
+		(to == CampaignContentTypeHTML || to == CampaignContentTypeRichtext) {
+		var b bytes.Buffer
+		if err := markdown.Convert([]byte(c.Body), &b); err != nil {
+			return out, err
+		}
+		out = b.String()
+	} else {
+		return out, errors.New("unknown formats to convert")
+	}
+
+	return out, nil
+}
+
 // FirstName splits the name by spaces and returns the first chunk
 // of the name that's greater than 2 characters in length, assuming
 // that it is the subscriber's first name.