feat: wip, better digest handling (+ eventual digest resend logic) (#7995)

* feat: wip, better digest handling (+ eventual digest resend logic)

- await emailer.send call in digest.send method
- save send success to a new sorted set digest:{interval}:byUid

* feat: continuing work on digest tools

- Added ACP page to view digest settings and delivery times per user

* feat: added paginator and stub buttons for resending digest

* feat: wrapping up digest revamp

- New language strings in ACP digest page
- Client-side ACP script for digest ACP page
- Websocket call for ACP page to execute digests
- Broke out logic to retrieve user digest settings to getUsersInterval

* fix: minor cleanup

* fix: #8010 and some style suggestions from baris

* fix: resolve confusing comment

src/controllers/admin.js

@@ -9,6 +9,7 @@ var adminController = {
 	postQueue: require('./admin/postqueue'),
 	blacklist: require('./admin/blacklist'),
 	groups: require('./admin/groups'),
+	digest: require('./admin/digest'),
 	appearance: require('./admin/appearance'),
 	extend: {
 		widgets: require('./admin/widgets'),

src/controllers/admin/digest.js

@@ -0,0 +1,23 @@
+'use strict';
+
+const meta = require('../../meta');
+const digest = require('../../user/digest');
+const pagination = require('../../pagination');
+
+const digestController = module.exports;
+
+digestController.get = async function (req, res) {
+	const page = parseInt(req.query.page, 10) || 1;
+	const resultsPerPage = 50;
+	const start = Math.max(0, page - 1) * resultsPerPage;
+	const stop = start + resultsPerPage - 1;
+	const delivery = await digest.getDeliveryTimes(start, stop);
+
+	const pageCount = Math.ceil(delivery.count / resultsPerPage);
+	res.render('admin/manage/digest', {
+		title: '[[admin/menu:manage/digest]]',
+		delivery: delivery.users,
+		default: meta.config.dailyDigestFreq,
+		pagination: pagination.create(page, pageCount),
+	});
+};

src/routes/admin.js

@@ -80,6 +80,7 @@ function addRoutes(router, middleware, controllers) {
 	router.get('/manage/groups/:name', middlewares, controllers.admin.groups.get);
 
 	router.get('/manage/uploads', middlewares, controllers.admin.uploads.get);
+	router.get('/manage/digest', middlewares, controllers.admin.digest.get);
 
 	router.get('/settings/:term?', middlewares, controllers.admin.settings.get);
 

src/socket.io/admin.js

@@ -42,6 +42,7 @@ SocketAdmin.analytics = {};
 SocketAdmin.logs = {};
 SocketAdmin.errors = {};
 SocketAdmin.uploads = {};
+SocketAdmin.digest = {};
 
 SocketAdmin.before = async function (socket, method) {
 	const isAdmin = await user.isAdministrator(socket.uid);
@@ -352,4 +353,22 @@ SocketAdmin.uploads.delete = function (socket, pathToFile, callback) {
 	fs.unlink(pathToFile, callback);
 };
 
+SocketAdmin.digest.resend = async (socket, data) => {
+	const uid = data.uid;
+	const interval = data.action.startsWith('resend-') ? data.action.slice(7) : await userDigest.getUsersInterval(uid);
+
+	if (!interval && meta.config.dailyDigestFreq === 'off') {
+		throw new Error('[[error:digest-not-enabled]]');
+	}
+
+	if (uid) {
+		await userDigest.execute({
+			interval: interval || meta.config.dailyDigestFreq,
+			subscribers: [uid],
+		});
+	} else {
+		await userDigest.execute({ interval: interval });
+	}
+};
+
 require('../promisify')(SocketAdmin);

src/user/digest.js

@@ -4,6 +4,7 @@ const async = require('async');
 const winston = require('winston');
 const nconf = require('nconf');
 
+const db = require('../database');
 const batch = require('../batch');
 const meta = require('../meta');
 const user = require('../user');
@@ -28,17 +29,49 @@ Digest.execute = async function (payload) {
 		return;
 	}
 	try {
-		const count = await Digest.send({
+		await Digest.send({
 			interval: payload.interval,
 			subscribers: subscribers,
 		});
-		winston.info('[user/jobs] Digest (' + payload.interval + ') scheduling completed. ' + count + ' email(s) sent.');
+		winston.info('[user/jobs] Digest (' + payload.interval + ') scheduling completed. Sending emails; this may take some time...');
 	} catch (err) {
 		winston.error('[user/jobs] Could not send digests (' + payload.interval + ')', err);
 		throw err;
 	}
 };
 
+Digest.getUsersInterval = async (uids) => {
+	// Checks whether user specifies digest setting, or null/false for system default setting
+	let single = false;
+	if (!Array.isArray(uids) && !isNaN(parseInt(uids, 10))) {
+		uids = [uids];
+		single = true;
+	}
+
+	let settings = await Promise.all([
+		db.isSortedSetMembers('digest:day:uids', uids),
+		db.isSortedSetMembers('digest:week:uids', uids),
+		db.isSortedSetMembers('digest:month:uids', uids),
+	]);
+	settings = settings.reduce((memo, cur, idx) => {
+		switch (idx) {
+		case 0:
+			memo = cur.map(bool => (bool === true ? 'day' : bool));
+			break;
+		case 1:
+			memo = cur.map(bool => (bool === true ? 'week' : bool));
+			break;
+		case 2:
+			memo = cur.map(bool => (bool === true ? 'month' : bool));
+			break;
+		}
+
+		return memo;
+	});
+
+	return single ? settings[0] : settings;
+};
+
 Digest.getSubscribers = async function (interval) {
 	var subscribers = [];
 
@@ -99,21 +132,53 @@ Digest.send = async function (data) {
 			return topicObj;
 		});
 		emailsSent += 1;
-		emailer.send('digest', userObj.uid, {
-			subject: '[[email:digest.subject, ' + (now.getFullYear() + '/' + (now.getMonth() + 1) + '/' + now.getDate()) + ']]',
-			username: userObj.username,
-			userslug: userObj.userslug,
-			notifications: notifications,
-			recent: topicsData,
-			interval: data.interval,
-			showUnsubscribe: true,
-		}, function (err) {
-			if (err) {
-				winston.error('[user/jobs] Could not send digest email', err);
-			}
-		});
+		try {
+			await emailer.send('digest', userObj.uid, {
+				subject: '[[email:digest.subject, ' + (now.getFullYear() + '/' + (now.getMonth() + 1) + '/' + now.getDate()) + ']]',
+				username: userObj.username,
+				userslug: userObj.userslug,
+				notifications: notifications,
+				recent: topicsData,
+				interval: data.interval,
+				showUnsubscribe: true,
+			});
+		} catch (err) {
+			winston.error('[user/jobs] Could not send digest email', err);
+		}
+
+		if (data.interval !== 'alltime') {
+			await db.sortedSetAdd('digest:delivery', now.getTime(), userObj.uid);
+		}
+	}, function () {
+		winston.info('[user/jobs] Digest (' + data.interval + ') sending completed. ' + emailsSent + ' emails sent.');
 	});
-	return emailsSent;
+};
+
+Digest.getDeliveryTimes = async (start, stop) => {
+	const count = await db.sortedSetCard('users:joindate');
+	const uids = await user.getUidsFromSet('users:joindate', start, stop);
+	if (!uids) {
+		return [];
+	}
+
+	// Grab the last time a digest was successfully delivered to these uids
+	const scores = await db.sortedSetScores('digest:delivery', uids);
+
+	// Get users' digest settings
+	const settings = await Digest.getUsersInterval(uids);
+
+	// Populate user data
+	let userData = await user.getUsersFields(uids, ['username', 'picture']);
+	userData = userData.map((user, idx) => {
+		user.lastDelivery = scores[idx] ? new Date(scores[idx]).toISOString() : '[[admin/manage/digest:null]]';
+		user.setting = settings[idx];
+		return user;
+	});
+
+	return {
+		users: userData,
+		count: count,
+	};
 };
 
 async function getTermTopics(term, uid, start, stop) {

src/views/admin/manage/digest.tpl

@@ -0,0 +1,51 @@
+<p class="lead">[[admin/manage/digest:lead]]</p>
+<p>[[admin/manage/digest:disclaimer]]</p>
+<p>[[admin/manage/digest:disclaimer-continued]]</p>
+
+<hr />
+
+<table class="table table-striped">
+	<thead>
+		<th>[[admin/manage/digest:user]]</th>
+		<th>[[admin/manage/digest:subscription]]</th>
+		<th>[[admin/manage/digest:last-delivery]]</th>
+		<th></th>
+	</thead>
+	<tbody>
+		<!-- BEGIN delivery -->
+		<tr>
+			<td><a href="{config.relative_path}/uid/{../uid}">{buildAvatar(delivery, "sm", true)} {../username}</a></td>
+			<td>{{{if ../setting}}}{../setting}{{{else}}}<em>[[admin/manage/digest:default]]</em>{{{end}}}</td>
+			<td>{../lastDelivery}</td>
+			<td><button class="btn btn-xs btn-default" data-action="resend" data-uid="{../uid}">[[admin/manage/digest:resend]]</button></td>
+		</tr>
+		<!-- END delivery -->
+		<!-- IF !delivery.length -->
+		<tr>
+			<td colspan="4">
+				<div class="alert alert-success">
+					[[admin/manage/digest:no-delivery-data]]
+				</div>
+			</td>
+		</tr>
+		<!-- ENDIF !delivery.length -->
+	</tbody>
+	<tfoot>
+		<tr>
+			<td colspan="4"><!-- IMPORT partials/paginator.tpl --></td>
+		</tr>
+		<tr>
+			<td colspan="4">
+				<em>[[admin/manage/digest:default-help, {default}]]</em>
+			</td>
+		</tr>
+		<tr>
+			<td colspan="4">
+				[[admin/manage/digest:manual-run]]
+				<button class="btn btn-xs btn-default" data-action="resend-day">[[admin/settings/user:digest-freq.daily]]</button>
+				<button class="btn btn-xs btn-default" data-action="resend-week">[[admin/settings/user:digest-freq.weekly]]</button>
+				<button class="btn btn-xs btn-default" data-action="resend-month">[[admin/settings/user:digest-freq.monthly]]</button>
+			</td>
+		</tr>
+	</tfoot>
+</table>

src/views/admin/partials/menu.tpl

@@ -34,6 +34,7 @@
 			<li><a href="{relative_path}/admin/manage/post-queue">[[admin/menu:manage/post-queue]]</a></li>
 			<li><a href="{relative_path}/admin/manage/ip-blacklist">[[admin/menu:manage/ip-blacklist]]</a></li>
 			<li><a href="{relative_path}/admin/manage/uploads">[[admin/menu:manage/uploads]]</a></li>
+			<li><a href="{relative_path}/admin/manage/digest">[[admin/menu:manage/digest]]</a></li>
 		</ul>
 	</section>
 
@@ -195,6 +196,7 @@
 					<li><a href="{relative_path}/admin/manage/post-queue">[[admin/menu:manage/post-queue]]</a></li>
 					<li><a href="{relative_path}/admin/manage/ip-blacklist">[[admin/menu:manage/ip-blacklist]]</a></li>
 					<li><a href="{relative_path}/admin/manage/uploads">[[admin/menu:manage/uploads]]</a></li>
+					<li><a href="{relative_path}/admin/manage/digest">[[admin/menu:manage/digest]]</a></li>
 				</ul>
 			</li>
 			<li class="dropdown menu-item">