From 0185ea1b4f5e6420473e7af10302d2ed655189ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bar=C4=B1=C5=9F=20Soner=20U=C5=9Fakl=C4=B1?= Date: Tue, 9 Feb 2021 15:27:08 -0500 Subject: [PATCH] perf: make digests a little bit faster and use batch.processArray dont load data for users who have no email or have not confirmed their emails --- src/emailer.js | 16 ++++----- src/user/digest.js | 84 +++++++++++++++++++++++++--------------------- 2 files changed, 54 insertions(+), 46 deletions(-) diff --git a/src/emailer.js b/src/emailer.js index 84ace4403d..baa6bfe644 100644 --- a/src/emailer.js +++ b/src/emailer.js @@ -216,22 +216,22 @@ Emailer.send = async (template, uid, params) => { throw Error('[emailer] App not ready!'); } - const [userData, userSettings] = await Promise.all([ - User.getUserFields(uid, ['email', 'username', 'email:confirmed']), - User.getSettings(uid), - ]); - + const userData = await User.getUserFields(uid, ['email', 'username', 'email:confirmed']); if (!userData || !userData.email) { - winston.warn(`uid : ${uid} has no email, not sending "${template}" email.`); + if (process.env.NODE_ENV === 'development') { + winston.warn(`uid : ${uid} has no email, not sending "${template}" email.`); + } return; } const allowedTpls = ['verify_email', 'welcome', 'registration_accepted']; if (meta.config.requireEmailConfirmation && !userData['email:confirmed'] && !allowedTpls.includes(template)) { - winston.warn(`uid : ${uid} (${userData.email}) has not confirmed email, not sending "${template}" email.`); + if (process.env.NODE_ENV === 'development') { + winston.warn(`uid : ${uid} (${userData.email}) has not confirmed email, not sending "${template}" email.`); + } return; } - + const userSettings = await User.getSettings(uid); // Combined passed-in payload with default values params = { ...Emailer._defaultPayload, ...params }; params.uid = uid; diff --git a/src/user/digest.js b/src/user/digest.js index 8bd247bc0b..56f204f721 100644 --- a/src/user/digest.js +++ b/src/user/digest.js @@ -1,6 +1,5 @@ 'use strict'; -const async = require('async'); const winston = require('winston'); const nconf = require('nconf'); @@ -29,7 +28,7 @@ Digest.execute = async function (payload) { return; } try { - winston.info(`[user/jobs] Digest (${payload.interval}) scheduling completed. Sending emails; this may take some time...`); + winston.info(`[user/jobs] Digest (${payload.interval}) scheduling completed (${subscribers.length} subscribers). Sending emails; this may take some time...`); await Digest.send({ interval: payload.interval, subscribers: subscribers, @@ -100,46 +99,55 @@ Digest.send = async function (data) { return emailsSent; } - await async.eachLimit(data.subscribers, 100, async (uid) => { - const userObj = await user.getUserFields(uid, ['uid', 'username', 'userslug', 'lastonline']); - const [notifications, topTopics, popularTopics, recentTopics] = await Promise.all([ - user.notifications.getUnreadInterval(userObj.uid, data.interval), - getTermTopics(data.interval, userObj.uid, 0, 9, 'votes'), - getTermTopics(data.interval, userObj.uid, 0, 9, 'posts'), - getTermTopics(data.interval, userObj.uid, 0, 9, 'recent'), - ]); - const unreadNotifs = notifications.filter(Boolean); - // If there are no notifications and no new topics, don't bother sending a digest - if (!unreadNotifs.length && !topTopics.length && !popularTopics.length && !recentTopics.length) { + await batch.processArray(data.subscribers, async (uids) => { + let userData = await user.getUsersFields(uids, ['uid', 'email', 'email:confirmed', 'username', 'userslug', 'lastonline']); + userData = userData.filter(u => u && u.email && (!meta.config.requireEmailConfirmation || userData['email:confirmed'])); + if (!userData.length) { return; } - - unreadNotifs.forEach((n) => { - if (n.image && !n.image.startsWith('http')) { - n.image = nconf.get('base_url') + n.image; + await Promise.all(userData.map(async (userObj) => { + const [notifications, topTopics, popularTopics, recentTopics] = await Promise.all([ + user.notifications.getUnreadInterval(userObj.uid, data.interval), + getTermTopics(data.interval, userObj.uid, 0, 9, 'votes'), + getTermTopics(data.interval, userObj.uid, 0, 9, 'posts'), + getTermTopics(data.interval, userObj.uid, 0, 9, 'recent'), + ]); + const unreadNotifs = notifications.filter(Boolean); + // If there are no notifications and no new topics, don't bother sending a digest + if (!unreadNotifs.length && !topTopics.length && !popularTopics.length && !recentTopics.length) { + return; } - if (n.path) { - n.notification_url = n.path.startsWith('http') ? n.path : nconf.get('base_url') + n.path; + + unreadNotifs.forEach((n) => { + if (n.image && !n.image.startsWith('http')) { + n.image = nconf.get('base_url') + n.image; + } + if (n.path) { + n.notification_url = n.path.startsWith('http') ? n.path : nconf.get('base_url') + n.path; + } + }); + + emailsSent += 1; + const now = new Date(); + await emailer.send('digest', userObj.uid, { + subject: `[[email:digest.subject, ${now.getFullYear()}/${now.getMonth() + 1}/${now.getDate()}]]`, + username: userObj.username, + userslug: userObj.userslug, + notifications: unreadNotifs, + recent: recentTopics, + topTopics: topTopics, + popularTopics: popularTopics, + interval: data.interval, + showUnsubscribe: true, + }).catch(err => winston.error(`[user/jobs] Could not send digest email\n[emailer.send] ${err.stack}`)); + + if (data.interval !== 'alltime') { + await db.sortedSetAdd('digest:delivery', now.getTime(), userObj.uid); } - }); - - emailsSent += 1; - const now = new Date(); - await emailer.send('digest', userObj.uid, { - subject: `[[email:digest.subject, ${now.getFullYear()}/${now.getMonth() + 1}/${now.getDate()}]]`, - username: userObj.username, - userslug: userObj.userslug, - notifications: unreadNotifs, - recent: recentTopics, - topTopics: topTopics, - popularTopics: popularTopics, - interval: data.interval, - showUnsubscribe: true, - }).catch(err => winston.error(`[user/jobs] Could not send digest email\n[emailer.send] ${err.stack}`)); - - if (data.interval !== 'alltime') { - await db.sortedSetAdd('digest:delivery', now.getTime(), userObj.uid); - } + })); + }, { + interval: 1000, + batch: 100, }); winston.info(`[user/jobs] Digest (${data.interval}) sending completed. ${emailsSent} emails sent.`); };