Add subscriber export feature

This commit is contained in:
Kailash Nadh 2021-01-23 18:23:29 +05:30
parent 3498a727f5
commit ec1c4f30ed
9 changed files with 141 additions and 0 deletions

View File

@ -72,6 +72,8 @@ func registerHTTPHandlers(e *echo.Echo) {
g.PUT("/api/subscribers/query/blocklist", handleBlocklistSubscribersByQuery) g.PUT("/api/subscribers/query/blocklist", handleBlocklistSubscribersByQuery)
g.PUT("/api/subscribers/query/lists", handleManageSubscriberListsByQuery) g.PUT("/api/subscribers/query/lists", handleManageSubscriberListsByQuery)
g.GET("/api/subscribers", handleQuerySubscribers) g.GET("/api/subscribers", handleQuerySubscribers)
g.GET("/api/subscribers/export",
middleware.GzipWithConfig(middleware.GzipConfig{Level: 9})(handleExportSubscribers))
g.GET("/api/import/subscribers", handleGetImportSubscribers) g.GET("/api/import/subscribers", handleGetImportSubscribers)
g.GET("/api/import/subscribers/logs", handleGetImportSubscriberStats) g.GET("/api/import/subscribers/logs", handleGetImportSubscriberStats)

View File

@ -46,6 +46,7 @@ type constants struct {
FromEmail string `koanf:"from_email"` FromEmail string `koanf:"from_email"`
NotifyEmails []string `koanf:"notify_emails"` NotifyEmails []string `koanf:"notify_emails"`
Lang string `koanf:"lang"` Lang string `koanf:"lang"`
DBBatchSize int `koanf:"batch_size"`
Privacy struct { Privacy struct {
IndividualTracking bool `koanf:"individual_tracking"` IndividualTracking bool `koanf:"individual_tracking"`
AllowBlocklist bool `koanf:"allow_blocklist"` AllowBlocklist bool `koanf:"allow_blocklist"`

View File

@ -35,6 +35,7 @@ type Queries struct {
// Non-prepared arbitrary subscriber queries. // Non-prepared arbitrary subscriber queries.
QuerySubscribers string `query:"query-subscribers"` QuerySubscribers string `query:"query-subscribers"`
QuerySubscribersForExport string `query:"query-subscribers-for-export"`
QuerySubscribersTpl string `query:"query-subscribers-template"` QuerySubscribersTpl string `query:"query-subscribers-template"`
DeleteSubscribersByQuery string `query:"delete-subscribers-by-query"` DeleteSubscribersByQuery string `query:"delete-subscribers-by-query"`
AddSubscribersToListsByQuery string `query:"add-subscribers-to-lists-by-query"` AddSubscribersToListsByQuery string `query:"add-subscribers-to-lists-by-query"`

View File

@ -3,6 +3,7 @@ package main
import ( import (
"context" "context"
"database/sql" "database/sql"
"encoding/csv"
"encoding/json" "encoding/json"
"fmt" "fmt"
"net/http" "net/http"
@ -160,6 +161,98 @@ func handleQuerySubscribers(c echo.Context) error {
return c.JSON(http.StatusOK, okResp{out}) return c.JSON(http.StatusOK, okResp{out})
} }
// handleExportSubscribers handles querying subscribers based on an arbitrary SQL expression.
func handleExportSubscribers(c echo.Context) error {
var (
app = c.Get("app").(*App)
// Limit the subscribers to a particular list?
listID, _ = strconv.Atoi(c.FormValue("list_id"))
// The "WHERE ?" bit.
query = sanitizeSQLExp(c.FormValue("query"))
)
listIDs := pq.Int64Array{}
if listID < 0 {
return echo.NewHTTPError(http.StatusBadRequest, app.i18n.T("globals.messages.errorID"))
} else if listID > 0 {
listIDs = append(listIDs, int64(listID))
}
// There's an arbitrary query condition.
cond := ""
if query != "" {
cond = " AND " + query
}
stmt := fmt.Sprintf(app.queries.QuerySubscribersForExport, cond)
// Verify that the arbitrary SQL search expression is read only.
if cond != "" {
tx, err := app.db.Unsafe().BeginTxx(context.Background(), &sql.TxOptions{ReadOnly: true})
if err != nil {
app.log.Printf("error preparing subscriber query: %v", err)
return echo.NewHTTPError(http.StatusBadRequest,
app.i18n.Ts2("subscribers.errorPreparingQuery", "error", pqErrMsg(err)))
}
defer tx.Rollback()
if _, err := tx.Query(stmt, nil, 0, 1); err != nil {
return echo.NewHTTPError(http.StatusBadRequest,
app.i18n.Ts2("subscribers.errorPreparingQuery", "error", pqErrMsg(err)))
}
}
// Prepare the actual query statement.
tx, err := db.Preparex(stmt)
if err != nil {
return echo.NewHTTPError(http.StatusBadRequest,
app.i18n.Ts2("subscribers.errorPreparingQuery", "error", pqErrMsg(err)))
}
// Run the query until all rows are exhausted.
var (
id = 0
h = c.Response().Header()
wr = csv.NewWriter(c.Response())
)
h.Set(echo.HeaderContentType, echo.MIMEOctetStream)
h.Set("Content-type", "text/csv")
h.Set(echo.HeaderContentDisposition, "attachment; filename="+"subscribers.csv")
h.Set("Content-Transfer-Encoding", "binary")
h.Set("Cache-Control", "no-cache")
wr.Write([]string{"uuid", "email", "name", "attributes", "status", "created_at", "updated_at"})
loop:
for {
var out []models.SubscriberExport
if err := tx.Select(&out, listIDs, id, app.constants.DBBatchSize); err != nil {
return echo.NewHTTPError(http.StatusInternalServerError,
app.i18n.Ts2("globals.messages.errorFetching",
"name", "globals.terms.subscribers", "error", pqErrMsg(err)))
}
if len(out) == 0 {
break loop
}
for _, r := range out {
if err = wr.Write([]string{r.UUID, r.Email, r.Name, r.Attribs, r.Status,
r.CreatedAt.Time.String(), r.UpdatedAt.Time.String()}); err != nil {
app.log.Printf("error streaming CSV export: %v", err)
break loop
}
}
wr.Flush()
id = out[len(out)-1].ID
}
return nil
}
// handleCreateSubscriber handles the creation of a new subscriber. // handleCreateSubscriber handles the creation of a new subscriber.
func handleCreateSubscriber(c echo.Context) error { func handleCreateSubscriber(c echo.Context) error {
var ( var (

View File

@ -18,6 +18,7 @@ export const uris = Object.freeze({
previewCampaign: '/api/campaigns/:id/preview', previewCampaign: '/api/campaigns/:id/preview',
previewTemplate: '/api/templates/:id/preview', previewTemplate: '/api/templates/:id/preview',
previewRawTemplate: '/api/templates/preview', previewRawTemplate: '/api/templates/preview',
exportSubscribers: '/api/subscribers/export',
}); });
// Keys used in Vuex store. // Keys used in Vuex store.

View File

@ -104,6 +104,11 @@
paginated backend-pagination pagination-position="both" @page-change="onPageChange" paginated backend-pagination pagination-position="both" @page-change="onPageChange"
:current-page="queryParams.page" :per-page="subscribers.perPage" :total="subscribers.total" :current-page="queryParams.page" :per-page="subscribers.perPage" :total="subscribers.total"
hoverable checkable backend-sorting @sort="onSort"> hoverable checkable backend-sorting @sort="onSort">
<template slot="top-left">
<a href='' @click.prevent="exportSubscribers">
<b-icon icon="cloud-download-outline" size="is-small" /> Export
</a>
</template>
<template slot-scope="props"> <template slot-scope="props">
<b-table-column field="status" label="Status" sortable> <b-table-column field="status" label="Status" sortable>
<a :href="`/subscribers/${props.row.id}`" <a :href="`/subscribers/${props.row.id}`"
@ -195,6 +200,7 @@ import { mapState } from 'vuex';
import SubscriberForm from './SubscriberForm.vue'; import SubscriberForm from './SubscriberForm.vue';
import SubscriberBulkList from './SubscriberBulkList.vue'; import SubscriberBulkList from './SubscriberBulkList.vue';
import EmptyPlaceholder from '../components/EmptyPlaceholder.vue'; import EmptyPlaceholder from '../components/EmptyPlaceholder.vue';
import { uris } from '../constants';
export default Vue.extend({ export default Vue.extend({
components: { components: {
@ -369,6 +375,15 @@ export default Vue.extend({
this.$utils.confirm(this.$t('subscribers.confirmBlocklist', { num: this.numSelectedSubscribers }), fn); this.$utils.confirm(this.$t('subscribers.confirmBlocklist', { num: this.numSelectedSubscribers }), fn);
}, },
exportSubscribers() {
this.$utils.confirm(this.$t('subscribers.confirmExport', { num: this.subscribers.total }), () => {
const q = new URLSearchParams();
q.append('query', this.queryParams.queryExp);
q.append('list_id', this.queryParams.listID);
document.location.href = `${uris.exportSubscribers}?${q.toString()}`;
});
},
deleteSubscribers() { deleteSubscribers() {
let fn = null; let fn = null;
if (!this.bulk.all && this.bulk.checked.length > 0) { if (!this.bulk.all && this.bulk.checked.length > 0) {

View File

@ -315,6 +315,7 @@
"subscribers.attribsHelp": "Attributes are defined as a JSON map, for example:", "subscribers.attribsHelp": "Attributes are defined as a JSON map, for example:",
"subscribers.blocklistedHelp": "Blocklisted subscribers will never receive any e-mails.", "subscribers.blocklistedHelp": "Blocklisted subscribers will never receive any e-mails.",
"subscribers.confirmBlocklist": "Blocklist {num} subscriber(s)?", "subscribers.confirmBlocklist": "Blocklist {num} subscriber(s)?",
"subscribers.confirmExport": "Export {num} subscriber(s)?",
"subscribers.confirmDelete": "Delete {num} subscriber(s)?", "subscribers.confirmDelete": "Delete {num} subscriber(s)?",
"subscribers.downloadData": "Download data", "subscribers.downloadData": "Download data",
"subscribers.email": "E-mail", "subscribers.email": "E-mail",

View File

@ -128,6 +128,17 @@ type SubscriberAttribs map[string]interface{}
// Subscribers represents a slice of Subscriber. // Subscribers represents a slice of Subscriber.
type Subscribers []Subscriber type Subscribers []Subscriber
// SubscriberExport represents a subscriber record that is exported to raw data.
type SubscriberExport struct {
Base
UUID string `db:"uuid" json:"uuid"`
Email string `db:"email" json:"email"`
Name string `db:"name" json:"name"`
Attribs string `db:"attribs" json:"attribs"`
Status string `db:"status" json:"status"`
}
// List represents a mailing list. // List represents a mailing list.
type List struct { type List struct {
Base Base

View File

@ -238,6 +238,22 @@ SELECT COUNT(*) OVER () AS total, subscribers.* FROM subscribers
%s %s
ORDER BY %s %s OFFSET $2 LIMIT $3; ORDER BY %s %s OFFSET $2 LIMIT $3;
-- name: query-subscribers-for-export
-- raw: true
-- Unprepared statement for issuring arbitrary WHERE conditions for
-- searching subscribers to do bulk CSV export.
-- %s = arbitrary expression
SELECT s.id, s.uuid, s.email, s.name, s.status, s.attribs, s.created_at, s.updated_at FROM subscribers s
LEFT JOIN subscriber_lists sl
ON (
-- Optional list filtering.
(CASE WHEN CARDINALITY($1::INT[]) > 0 THEN true ELSE false END)
AND sl.subscriber_id = s.id
)
WHERE sl.list_id = ALL($1::INT[]) AND id > $2
%s
ORDER BY s.id ASC LIMIT $3;
-- name: query-subscribers-template -- name: query-subscribers-template
-- raw: true -- raw: true
-- This raw query is reused in multiple queries (blocklist, add to list, delete) -- This raw query is reused in multiple queries (blocklist, add to list, delete)