Add subscriber export feature

cmd/handlers.go

@@ -72,6 +72,8 @@ func registerHTTPHandlers(e *echo.Echo) {
 	g.PUT("/api/subscribers/query/blocklist", handleBlocklistSubscribersByQuery)
 	g.PUT("/api/subscribers/query/lists", handleManageSubscriberListsByQuery)
 	g.GET("/api/subscribers", handleQuerySubscribers)
+	g.GET("/api/subscribers/export",
+		middleware.GzipWithConfig(middleware.GzipConfig{Level: 9})(handleExportSubscribers))
 
 	g.GET("/api/import/subscribers", handleGetImportSubscribers)
 	g.GET("/api/import/subscribers/logs", handleGetImportSubscriberStats)

cmd/init.go

@@ -46,6 +46,7 @@ type constants struct {
 	FromEmail    string   `koanf:"from_email"`
 	NotifyEmails []string `koanf:"notify_emails"`
 	Lang         string   `koanf:"lang"`
+	DBBatchSize  int      `koanf:"batch_size"`
 	Privacy      struct {
 		IndividualTracking bool            `koanf:"individual_tracking"`
 		AllowBlocklist     bool            `koanf:"allow_blocklist"`

cmd/queries.go

@@ -35,6 +35,7 @@ type Queries struct {
 
 	// Non-prepared arbitrary subscriber queries.
 	QuerySubscribers                       string `query:"query-subscribers"`
+	QuerySubscribersForExport              string `query:"query-subscribers-for-export"`
 	QuerySubscribersTpl                    string `query:"query-subscribers-template"`
 	DeleteSubscribersByQuery               string `query:"delete-subscribers-by-query"`
 	AddSubscribersToListsByQuery           string `query:"add-subscribers-to-lists-by-query"`

cmd/subscribers.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"database/sql"
+	"encoding/csv"
 	"encoding/json"
 	"fmt"
 	"net/http"
@@ -160,6 +161,98 @@ func handleQuerySubscribers(c echo.Context) error {
 	return c.JSON(http.StatusOK, okResp{out})
 }
 
+// handleExportSubscribers handles querying subscribers based on an arbitrary SQL expression.
+func handleExportSubscribers(c echo.Context) error {
+	var (
+		app = c.Get("app").(*App)
+
+		// Limit the subscribers to a particular list?
+		listID, _ = strconv.Atoi(c.FormValue("list_id"))
+
+		// The "WHERE ?" bit.
+		query = sanitizeSQLExp(c.FormValue("query"))
+	)
+
+	listIDs := pq.Int64Array{}
+	if listID < 0 {
+		return echo.NewHTTPError(http.StatusBadRequest, app.i18n.T("globals.messages.errorID"))
+	} else if listID > 0 {
+		listIDs = append(listIDs, int64(listID))
+	}
+
+	// There's an arbitrary query condition.
+	cond := ""
+	if query != "" {
+		cond = " AND " + query
+	}
+
+	stmt := fmt.Sprintf(app.queries.QuerySubscribersForExport, cond)
+
+	// Verify that the arbitrary SQL search expression is read only.
+	if cond != "" {
+		tx, err := app.db.Unsafe().BeginTxx(context.Background(), &sql.TxOptions{ReadOnly: true})
+		if err != nil {
+			app.log.Printf("error preparing subscriber query: %v", err)
+			return echo.NewHTTPError(http.StatusBadRequest,
+				app.i18n.Ts2("subscribers.errorPreparingQuery", "error", pqErrMsg(err)))
+		}
+		defer tx.Rollback()
+
+		if _, err := tx.Query(stmt, nil, 0, 1); err != nil {
+			return echo.NewHTTPError(http.StatusBadRequest,
+				app.i18n.Ts2("subscribers.errorPreparingQuery", "error", pqErrMsg(err)))
+		}
+	}
+
+	// Prepare the actual query statement.
+	tx, err := db.Preparex(stmt)
+	if err != nil {
+		return echo.NewHTTPError(http.StatusBadRequest,
+			app.i18n.Ts2("subscribers.errorPreparingQuery", "error", pqErrMsg(err)))
+	}
+
+	// Run the query until all rows are exhausted.
+	var (
+		id = 0
+
+		h  = c.Response().Header()
+		wr = csv.NewWriter(c.Response())
+	)
+
+	h.Set(echo.HeaderContentType, echo.MIMEOctetStream)
+	h.Set("Content-type", "text/csv")
+	h.Set(echo.HeaderContentDisposition, "attachment; filename="+"subscribers.csv")
+	h.Set("Content-Transfer-Encoding", "binary")
+	h.Set("Cache-Control", "no-cache")
+	wr.Write([]string{"uuid", "email", "name", "attributes", "status", "created_at", "updated_at"})
+
+loop:
+	for {
+		var out []models.SubscriberExport
+		if err := tx.Select(&out, listIDs, id, app.constants.DBBatchSize); err != nil {
+			return echo.NewHTTPError(http.StatusInternalServerError,
+				app.i18n.Ts2("globals.messages.errorFetching",
+					"name", "globals.terms.subscribers", "error", pqErrMsg(err)))
+		}
+		if len(out) == 0 {
+			break loop
+		}
+
+		for _, r := range out {
+			if err = wr.Write([]string{r.UUID, r.Email, r.Name, r.Attribs, r.Status,
+				r.CreatedAt.Time.String(), r.UpdatedAt.Time.String()}); err != nil {
+				app.log.Printf("error streaming CSV export: %v", err)
+				break loop
+			}
+		}
+		wr.Flush()
+
+		id = out[len(out)-1].ID
+	}
+
+	return nil
+}
+
 // handleCreateSubscriber handles the creation of a new subscriber.
 func handleCreateSubscriber(c echo.Context) error {
 	var (

frontend/src/constants.js

@@ -18,6 +18,7 @@ export const uris = Object.freeze({
   previewCampaign: '/api/campaigns/:id/preview',
   previewTemplate: '/api/templates/:id/preview',
   previewRawTemplate: '/api/templates/preview',
+  exportSubscribers: '/api/subscribers/export',
 });
 
 // Keys used in Vuex store.

frontend/src/views/Subscribers.vue

@@ -104,6 +104,11 @@
       paginated backend-pagination pagination-position="both" @page-change="onPageChange"
       :current-page="queryParams.page" :per-page="subscribers.perPage" :total="subscribers.total"
       hoverable checkable backend-sorting @sort="onSort">
+        <template slot="top-left">
+          <a href='' @click.prevent="exportSubscribers">
+            <b-icon icon="cloud-download-outline" size="is-small" /> Export
+          </a>
+        </template>
         <template slot-scope="props">
             <b-table-column field="status" label="Status" sortable>
               <a :href="`/subscribers/${props.row.id}`"
@@ -195,6 +200,7 @@ import { mapState } from 'vuex';
 import SubscriberForm from './SubscriberForm.vue';
 import SubscriberBulkList from './SubscriberBulkList.vue';
 import EmptyPlaceholder from '../components/EmptyPlaceholder.vue';
+import { uris } from '../constants';
 
 export default Vue.extend({
   components: {
@@ -369,6 +375,15 @@ export default Vue.extend({
       this.$utils.confirm(this.$t('subscribers.confirmBlocklist', { num: this.numSelectedSubscribers }), fn);
     },
 
+    exportSubscribers() {
+      this.$utils.confirm(this.$t('subscribers.confirmExport', { num: this.subscribers.total }), () => {
+        const q = new URLSearchParams();
+        q.append('query', this.queryParams.queryExp);
+        q.append('list_id', this.queryParams.listID);
+        document.location.href = `${uris.exportSubscribers}?${q.toString()}`;
+      });
+    },
+
     deleteSubscribers() {
       let fn = null;
       if (!this.bulk.all && this.bulk.checked.length > 0) {

i18n/en.json

@@ -315,6 +315,7 @@
     "subscribers.attribsHelp": "Attributes are defined as a JSON map, for example:",
     "subscribers.blocklistedHelp": "Blocklisted subscribers will never receive any e-mails.",
     "subscribers.confirmBlocklist": "Blocklist {num} subscriber(s)?",
+    "subscribers.confirmExport": "Export {num} subscriber(s)?",
     "subscribers.confirmDelete": "Delete {num} subscriber(s)?",
     "subscribers.downloadData": "Download data",
     "subscribers.email": "E-mail",

models/models.go

@@ -128,6 +128,17 @@ type SubscriberAttribs map[string]interface{}
 // Subscribers represents a slice of Subscriber.
 type Subscribers []Subscriber
 
+// SubscriberExport represents a subscriber record that is exported to raw data.
+type SubscriberExport struct {
+	Base
+
+	UUID    string `db:"uuid" json:"uuid"`
+	Email   string `db:"email" json:"email"`
+	Name    string `db:"name" json:"name"`
+	Attribs string `db:"attribs" json:"attribs"`
+	Status  string `db:"status" json:"status"`
+}
+
 // List represents a mailing list.
 type List struct {
 	Base

queries.sql

@@ -238,6 +238,22 @@ SELECT COUNT(*) OVER () AS total, subscribers.* FROM subscribers
     %s
     ORDER BY %s %s OFFSET $2 LIMIT $3;
 
+-- name: query-subscribers-for-export
+-- raw: true
+-- Unprepared statement for issuring arbitrary WHERE conditions for
+-- searching subscribers to do bulk CSV export.
+-- %s = arbitrary expression
+SELECT s.id, s.uuid, s.email, s.name, s.status, s.attribs, s.created_at, s.updated_at FROM subscribers s
+    LEFT JOIN subscriber_lists sl
+    ON (
+        -- Optional list filtering.
+        (CASE WHEN CARDINALITY($1::INT[]) > 0 THEN true ELSE false END)
+        AND sl.subscriber_id = s.id
+    )
+    WHERE sl.list_id = ALL($1::INT[]) AND id > $2
+    %s
+    ORDER BY s.id ASC LIMIT $3;
+
 -- name: query-subscribers-template
 -- raw: true
 -- This raw query is reused in multiple queries (blocklist, add to list, delete)