From 65d25fc3f9da94016e4013460de718269509e213 Mon Sep 17 00:00:00 2001 From: Kailash Nadh Date: Sun, 9 May 2021 15:36:31 +0530 Subject: [PATCH] Improve campaign content format conversion. Previously, converting between formats simply copied over raw content. This update does actual conversion between different formats. While lossy, this seems to a good enough approximation for even reasonbly rich HTML content. Closes #348. - richtext, html => plain Strips HTML and converts content to plain text. - richtext, html => markdown Uses turndown (JS) lib to convert HTML to Markdown. - plain => richtext, html Converts line breaks in plain text to HTML breaks. - richtext => html "Beautifies" the HTML generated by the WYSIWYG editor unlike the earlier behaviour of dumping one long line of HTML. - markdown => richtext, html Makes an API call to the backend to use the Goldmark lib to convert Markdown to HTML. --- cmd/campaigns.go | 37 ++++++++--- cmd/handlers.go | 1 + frontend/package.json | 1 + frontend/src/api/index.js | 3 + frontend/src/components/Editor.vue | 100 ++++++++++++++++++++++++++--- frontend/yarn.lock | 12 ++++ models/models.go | 24 +++++++ 7 files changed, 160 insertions(+), 18 deletions(-) diff --git a/cmd/campaigns.go b/cmd/campaigns.go index 0e606bb..aa6cc61 100644 --- a/cmd/campaigns.go +++ b/cmd/campaigns.go @@ -14,7 +14,6 @@ import ( "time" "github.com/gofrs/uuid" - "github.com/jaytaylor/html2text" "github.com/knadh/listmonk/internal/messenger" "github.com/knadh/listmonk/internal/subimporter" "github.com/knadh/listmonk/models" @@ -23,7 +22,8 @@ import ( null "gopkg.in/volatiletech/null.v6" ) -// campaignReq is a wrapper over the Campaign model. +// campaignReq is a wrapper over the Campaign model for receiving +// campaign creation and updation data from APIs. type campaignReq struct { models.Campaign @@ -42,6 +42,14 @@ type campaignReq struct { Type string `json:"type"` } +// campaignContentReq wraps params coming from API requests for converting +// campaign content formats. +type campaignContentReq struct { + models.Campaign + From string `json:"from"` + To string `json:"to"` +} + type campaignStats struct { ID int `db:"id" json:"id"` Status string `db:"status" json:"status"` @@ -201,15 +209,28 @@ func handlePreviewCampaign(c echo.Context) error { return c.HTML(http.StatusOK, string(m.Body())) } -// handleCampainBodyToText converts an HTML campaign body to plaintext. -func handleCampainBodyToText(c echo.Context) error { - out, err := html2text.FromString(c.FormValue("body"), - html2text.Options{PrettyTables: false}) - if err != nil { +// handleCampaignContent handles campaign content (body) format conversions. +func handleCampaignContent(c echo.Context) error { + var ( + app = c.Get("app").(*App) + id, _ = strconv.Atoi(c.Param("id")) + ) + + if id < 1 { + return echo.NewHTTPError(http.StatusBadRequest, app.i18n.T("globals.messages.invalidID")) + } + + var camp campaignContentReq + if err := c.Bind(&camp); err != nil { return err } - return c.HTML(http.StatusOK, string(out)) + out, err := camp.ConvertContent(camp.From, camp.To) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, err.Error()) + } + + return c.JSON(http.StatusOK, okResp{out}) } // handleCreateCampaign handles campaign creation. diff --git a/cmd/handlers.go b/cmd/handlers.go index 0a0d9b7..1825084 100644 --- a/cmd/handlers.go +++ b/cmd/handlers.go @@ -97,6 +97,7 @@ func registerHTTPHandlers(e *echo.Echo, app *App) { g.GET("/api/campaigns/:id", handleGetCampaigns) g.GET("/api/campaigns/:id/preview", handlePreviewCampaign) g.POST("/api/campaigns/:id/preview", handlePreviewCampaign) + g.POST("/api/campaigns/:id/content", handleCampaignContent) g.POST("/api/campaigns/:id/text", handlePreviewCampaign) g.POST("/api/campaigns/:id/test", handleTestCampaign) g.POST("/api/campaigns", handleCreateCampaign) diff --git a/frontend/package.json b/frontend/package.json index 6b6f3a6..d1ff7ca 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -25,6 +25,7 @@ "quill-delta": "^4.2.2", "sass-loader": "^8.0.2", "textversionjs": "^1.1.3", + "turndown": "^7.0.0", "vue": "^2.6.11", "vue-c3": "^1.2.11", "vue-i18n": "^8.22.2", diff --git a/frontend/src/api/index.js b/frontend/src/api/index.js index 92b0898..baf0032 100644 --- a/frontend/src/api/index.js +++ b/frontend/src/api/index.js @@ -156,6 +156,9 @@ export const getCampaignStats = async () => http.get('/api/campaigns/running/sta export const createCampaign = async (data) => http.post('/api/campaigns', data, { loading: models.campaigns }); +export const convertCampaignContent = async (data) => http.post(`/api/campaigns/${data.id}/content`, data, + { loading: models.campaigns }); + export const testCampaign = async (data) => http.post(`/api/campaigns/${data.id}/test`, data, { loading: models.campaigns }); diff --git a/frontend/src/components/Editor.vue b/frontend/src/components/Editor.vue index e6e0bc8..1202e62 100644 --- a/frontend/src/components/Editor.vue +++ b/frontend/src/components/Editor.vue @@ -78,6 +78,7 @@ import 'quill/dist/quill.core.css'; import { quillEditor, Quill } from 'vue-quill-editor'; import CodeFlask from 'codeflask'; +import TurndownService from 'turndown'; import CampaignPreview from './CampaignPreview.vue'; import Media from '../views/Media.vue'; @@ -98,6 +99,8 @@ const regLink = new RegExp(/{{(\s+)?TrackLink(\s+)?"(.+?)"(\s+)?}}/); const Link = Quill.import('formats/link'); Link.sanitize = (l) => l.replace(regLink, '{{ TrackLink `$3`}}'); +const turndown = new TurndownService(); + // Custom class to override the default indent behaviour to get inline CSS // style instead of classes. class IndentAttributor extends Quill.import('parchment').Attributor.Style { @@ -191,6 +194,9 @@ export default { }, }, }, + + // HTML editor. + flask: null, }; }, @@ -241,22 +247,25 @@ export default { `; this.$refs.htmlEditor.appendChild(el); - const flask = new CodeFlask(el.shadowRoot.getElementById('area'), { + this.flask = new CodeFlask(el.shadowRoot.getElementById('area'), { language: 'html', lineNumbers: false, styleParent: el.shadowRoot, readonly: this.disabled, }); - - flask.updateCode(this.form.body); - flask.onUpdate((b) => { + this.flask.onUpdate((b) => { this.form.body = b; this.$emit('input', { contentType: this.form.format, body: this.form.body }); }); + this.updateHTMLEditor(); this.isReady = true; }, + updateHTMLEditor() { + this.flask.updateCode(this.form.body); + }, + onTogglePreview() { this.isPreviewing = !this.isPreviewing; }, @@ -278,6 +287,46 @@ export default { onMediaSelect(m) { this.$refs.quill.quill.insertEmbed(this.lastSel.index || 0, 'image', m.url); }, + + beautifyHTML(str) { + const div = document.createElement('div'); + div.innerHTML = str.trim(); + return this.formatHTMLNode(div, 0).innerHTML; + }, + + formatHTMLNode(node, level) { + const lvl = level + 1; + const indentBefore = new Array(lvl + 1).join(' '); + const indentAfter = new Array(lvl - 1).join(' '); + let textNode = null; + + for (let i = 0; i < node.children.length; i += 1) { + textNode = document.createTextNode(`\n${indentBefore}`); + node.insertBefore(textNode, node.children[i]); + + this.formatHTMLNode(node.children[i], lvl); + if (node.lastElementChild === node.children[i]) { + textNode = document.createTextNode(`\n${indentAfter}`); + node.appendChild(textNode); + } + } + + return node; + }, + + trimLines(str, removeEmptyLines) { + const out = str.split('\n'); + for (let i = 0; i < out.length; i += 1) { + const line = out[i].trim(); + if (removeEmptyLines) { + out[i] = line; + } else if (line === '') { + out[i] = ''; + } + } + + return out.join('\n').replace(/\n\s*\n\s*\n/g, '\n\n'); + }, }, computed: { @@ -306,14 +355,45 @@ export default { this.onEditorChange(); }, - htmlFormat(f) { - if (f !== 'html') { - return; + htmlFormat(to, from) { + // On switch to HTML, initialize the HTML editor. + if (to === 'html') { + this.$nextTick(() => { + this.initHTMLEditor(); + }); } - this.$nextTick(() => { - this.initHTMLEditor(); - }); + if ((from === 'richtext' || from === 'html') && to === 'plain') { + // richtext, html => plain + + // Preserve line breaks when converting HTML to plaintext. Quill produces + // HTML without any linebreaks. + const d = document.createElement('div'); + d.innerHTML = this.beautifyHTML(this.form.body); + this.form.body = this.trimLines(d.innerText.trim(), true); + } else if ((from === 'richtext' || from === 'html') && to === 'markdown') { + // richtext, html => markdown + this.form.body = turndown.turndown(this.form.body).replace(/\n\n+/ig, '\n\n'); + } else if (from === 'plain' && (to === 'richtext' || to === 'html')) { + // plain => richtext, html + this.form.body = this.form.body.replace(/\n/ig, '
\n'); + } else if (from === 'richtext' && to === 'html') { + // richtext => html + this.form.body = this.trimLines(this.beautifyHTML(this.form.body), false); + } else if (from === 'markdown' && (to === 'richtext' || to === 'html')) { + // markdown => richtext, html. + this.$api.convertCampaignContent({ + id: 1, body: this.form.body, from, to, + }).then((data) => { + this.form.body = this.beautifyHTML(data.trim()); + // Update the HTML editor. + if (to === 'html') { + this.updateHTMLEditor(); + } + }); + } + + this.onEditorChange(); }, }, diff --git a/frontend/yarn.lock b/frontend/yarn.lock index 090c6de..b93d80e 100644 --- a/frontend/yarn.lock +++ b/frontend/yarn.lock @@ -3641,6 +3641,11 @@ domhandler@^2.3.0: dependencies: domelementtype "1" +domino@^2.1.6: + version "2.1.6" + resolved "https://registry.yarnpkg.com/domino/-/domino-2.1.6.tgz#fe4ace4310526e5e7b9d12c7de01b7f485a57ffe" + integrity sha512-3VdM/SXBZX2omc9JF9nOPCtDaYQ67BGp5CoLpIQlO2KCAPETs8TcDHacF26jXadGbvUteZzRTeos2fhID5+ucQ== + domutils@1.5.1: version "1.5.1" resolved "https://registry.yarnpkg.com/domutils/-/domutils-1.5.1.tgz#dcd8488a26f563d61079e48c9f7b7e32373682cf" @@ -9253,6 +9258,13 @@ tunnel-agent@^0.6.0: dependencies: safe-buffer "^5.0.1" +turndown@^7.0.0: + version "7.0.0" + resolved "https://registry.yarnpkg.com/turndown/-/turndown-7.0.0.tgz#19b2a6a2d1d700387a1e07665414e4af4fec5225" + integrity sha512-G1FfxfR0mUNMeGjszLYl3kxtopC4O9DRRiMlMDDVHvU1jaBkGFg4qxIyjIk2aiKLHyDyZvZyu4qBO2guuYBy3Q== + dependencies: + domino "^2.1.6" + tweetnacl@^0.14.3, tweetnacl@~0.14.0: version "0.14.5" resolved "https://registry.yarnpkg.com/tweetnacl/-/tweetnacl-0.14.5.tgz#5ae68177f192d4456269d108afa93ff8743f4f64" diff --git a/models/models.go b/models/models.go index 72419e8..c16dc0c 100644 --- a/models/models.go +++ b/models/models.go @@ -383,6 +383,30 @@ func (c *Campaign) CompileTemplate(f template.FuncMap) error { return nil } +// ConvertContent converts a campaign's body from one format to another, +// for example, Markdown to HTML. +func (c *Campaign) ConvertContent(from, to string) (string, error) { + body := c.Body + for _, r := range regTplFuncs { + body = r.regExp.ReplaceAllString(body, r.replace) + } + + // If the format is markdown, convert Markdown to HTML. + var out string + if from == CampaignContentTypeMarkdown && + (to == CampaignContentTypeHTML || to == CampaignContentTypeRichtext) { + var b bytes.Buffer + if err := markdown.Convert([]byte(c.Body), &b); err != nil { + return out, err + } + out = b.String() + } else { + return out, errors.New("unknown formats to convert") + } + + return out, nil +} + // FirstName splits the name by spaces and returns the first chunk // of the name that's greater than 2 characters in length, assuming // that it is the subscriber's first name.