diff --git a/cmd/handlers.go b/cmd/handlers.go index 295a321..502b386 100644 --- a/cmd/handlers.go +++ b/cmd/handlers.go @@ -72,6 +72,8 @@ func registerHTTPHandlers(e *echo.Echo) { g.PUT("/api/subscribers/query/blocklist", handleBlocklistSubscribersByQuery) g.PUT("/api/subscribers/query/lists", handleManageSubscriberListsByQuery) g.GET("/api/subscribers", handleQuerySubscribers) + g.GET("/api/subscribers/export", + middleware.GzipWithConfig(middleware.GzipConfig{Level: 9})(handleExportSubscribers)) g.GET("/api/import/subscribers", handleGetImportSubscribers) g.GET("/api/import/subscribers/logs", handleGetImportSubscriberStats) diff --git a/cmd/init.go b/cmd/init.go index 3a18cb7..0dac9bc 100644 --- a/cmd/init.go +++ b/cmd/init.go @@ -46,6 +46,7 @@ type constants struct { FromEmail string `koanf:"from_email"` NotifyEmails []string `koanf:"notify_emails"` Lang string `koanf:"lang"` + DBBatchSize int `koanf:"batch_size"` Privacy struct { IndividualTracking bool `koanf:"individual_tracking"` AllowBlocklist bool `koanf:"allow_blocklist"` diff --git a/cmd/queries.go b/cmd/queries.go index bf2edc4..cfbe478 100644 --- a/cmd/queries.go +++ b/cmd/queries.go @@ -35,6 +35,7 @@ type Queries struct { // Non-prepared arbitrary subscriber queries. QuerySubscribers string `query:"query-subscribers"` + QuerySubscribersForExport string `query:"query-subscribers-for-export"` QuerySubscribersTpl string `query:"query-subscribers-template"` DeleteSubscribersByQuery string `query:"delete-subscribers-by-query"` AddSubscribersToListsByQuery string `query:"add-subscribers-to-lists-by-query"` diff --git a/cmd/subscribers.go b/cmd/subscribers.go index e0c0cf9..7f91462 100644 --- a/cmd/subscribers.go +++ b/cmd/subscribers.go @@ -3,6 +3,7 @@ package main import ( "context" "database/sql" + "encoding/csv" "encoding/json" "fmt" "net/http" @@ -160,6 +161,98 @@ func handleQuerySubscribers(c echo.Context) error { return c.JSON(http.StatusOK, okResp{out}) } +// handleExportSubscribers handles querying subscribers based on an arbitrary SQL expression. +func handleExportSubscribers(c echo.Context) error { + var ( + app = c.Get("app").(*App) + + // Limit the subscribers to a particular list? + listID, _ = strconv.Atoi(c.FormValue("list_id")) + + // The "WHERE ?" bit. + query = sanitizeSQLExp(c.FormValue("query")) + ) + + listIDs := pq.Int64Array{} + if listID < 0 { + return echo.NewHTTPError(http.StatusBadRequest, app.i18n.T("globals.messages.errorID")) + } else if listID > 0 { + listIDs = append(listIDs, int64(listID)) + } + + // There's an arbitrary query condition. + cond := "" + if query != "" { + cond = " AND " + query + } + + stmt := fmt.Sprintf(app.queries.QuerySubscribersForExport, cond) + + // Verify that the arbitrary SQL search expression is read only. + if cond != "" { + tx, err := app.db.Unsafe().BeginTxx(context.Background(), &sql.TxOptions{ReadOnly: true}) + if err != nil { + app.log.Printf("error preparing subscriber query: %v", err) + return echo.NewHTTPError(http.StatusBadRequest, + app.i18n.Ts2("subscribers.errorPreparingQuery", "error", pqErrMsg(err))) + } + defer tx.Rollback() + + if _, err := tx.Query(stmt, nil, 0, 1); err != nil { + return echo.NewHTTPError(http.StatusBadRequest, + app.i18n.Ts2("subscribers.errorPreparingQuery", "error", pqErrMsg(err))) + } + } + + // Prepare the actual query statement. + tx, err := db.Preparex(stmt) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, + app.i18n.Ts2("subscribers.errorPreparingQuery", "error", pqErrMsg(err))) + } + + // Run the query until all rows are exhausted. + var ( + id = 0 + + h = c.Response().Header() + wr = csv.NewWriter(c.Response()) + ) + + h.Set(echo.HeaderContentType, echo.MIMEOctetStream) + h.Set("Content-type", "text/csv") + h.Set(echo.HeaderContentDisposition, "attachment; filename="+"subscribers.csv") + h.Set("Content-Transfer-Encoding", "binary") + h.Set("Cache-Control", "no-cache") + wr.Write([]string{"uuid", "email", "name", "attributes", "status", "created_at", "updated_at"}) + +loop: + for { + var out []models.SubscriberExport + if err := tx.Select(&out, listIDs, id, app.constants.DBBatchSize); err != nil { + return echo.NewHTTPError(http.StatusInternalServerError, + app.i18n.Ts2("globals.messages.errorFetching", + "name", "globals.terms.subscribers", "error", pqErrMsg(err))) + } + if len(out) == 0 { + break loop + } + + for _, r := range out { + if err = wr.Write([]string{r.UUID, r.Email, r.Name, r.Attribs, r.Status, + r.CreatedAt.Time.String(), r.UpdatedAt.Time.String()}); err != nil { + app.log.Printf("error streaming CSV export: %v", err) + break loop + } + } + wr.Flush() + + id = out[len(out)-1].ID + } + + return nil +} + // handleCreateSubscriber handles the creation of a new subscriber. func handleCreateSubscriber(c echo.Context) error { var ( diff --git a/frontend/src/constants.js b/frontend/src/constants.js index ffed0bb..0654ac1 100644 --- a/frontend/src/constants.js +++ b/frontend/src/constants.js @@ -18,6 +18,7 @@ export const uris = Object.freeze({ previewCampaign: '/api/campaigns/:id/preview', previewTemplate: '/api/templates/:id/preview', previewRawTemplate: '/api/templates/preview', + exportSubscribers: '/api/subscribers/export', }); // Keys used in Vuex store. diff --git a/frontend/src/views/Subscribers.vue b/frontend/src/views/Subscribers.vue index b4bee69..9784ef6 100644 --- a/frontend/src/views/Subscribers.vue +++ b/frontend/src/views/Subscribers.vue @@ -104,6 +104,11 @@ paginated backend-pagination pagination-position="both" @page-change="onPageChange" :current-page="queryParams.page" :per-page="subscribers.perPage" :total="subscribers.total" hoverable checkable backend-sorting @sort="onSort"> +