Merge branch 'master' into develop

install/package.json

@@ -110,7 +110,7 @@
         "nodebb-theme-harmony": "2.1.36",
         "nodebb-theme-lavender": "7.1.19",
         "nodebb-theme-peace": "2.2.49",
-        "nodebb-theme-persona": "14.1.25",
+        "nodebb-theme-persona": "14.1.26",
         "nodebb-widget-essentials": "7.0.41",
         "nodemailer": "7.0.12",
         "nprogress": "0.2.0",
@@ -124,6 +124,7 @@
         "pretty": "^2.0.0",
         "progress-webpack-plugin": "1.0.16",
         "prompt": "1.3.0",
+        "qs": "6.14.1",
         "redis": "5.10.0",
         "rimraf": "6.1.2",
         "rss": "1.2.2",

public/src/admin/settings.js

@@ -25,10 +25,13 @@ define('admin/settings', [
 			});
 			const offset = mainHader.outerHeight(true);
 			// https://stackoverflow.com/a/11814275/583363
-			tocList.find('a').on('click', function (event) {
-				event.preventDefault();
+			tocList.find('a').on('click', function () {
 				const href = $(this).attr('href');
-				$(href)[0].scrollIntoView();
+				const $target = $(href);
+				if (!$target.length) {
+					return;
+				}
+				$target.get(0).scrollIntoView(true);
 				window.location.hash = href;
 				scrollBy(0, -offset);
 				setTimeout(() => {

public/src/client/chats/events.js

@@ -17,9 +17,11 @@ define('forum/chats/events', [
 		'event:chats.typing': onChatTyping,
 	};
 	let chatNavWrapper = null;
+
 	let Chats = null;
+
 	Events.init = async function () {
-		Chats = await require('forum/chats');
+		Chats = await app.require('forum/chats');
 		chatNavWrapper = $('[component="chat/nav-wrapper"]');
 		Events.removeListeners();
 		for (const [eventName, handler] of Object.entries(events)) {

src/activitypub/helpers.js

@@ -65,10 +65,11 @@ Helpers.isUri = (value) => {
 	});
 };
 
-Helpers.assertAccept = accept => (accept && accept.split(',').some((value) => {
-	const parts = value.split(';').map(v => v.trim());
-	return activitypub._constants.acceptableTypes.includes(value || parts[0]);
-}));
+Helpers.assertAccept = (accept) => {
+	if (!accept) return false;
+	const normalized = accept.split(',').map(s => s.trim().replace(/\s*;\s*/g, ';')).join(',');
+	return activitypub._constants.acceptableTypes.some(type => normalized.includes(type));
+};
 
 Helpers.isWebfinger = (value) => {
 	// N.B. returns normalized handle, so truthy check!

src/activitypub/index.js

@@ -41,7 +41,7 @@ ActivityPub._constants = Object.freeze({
 	acceptablePublicAddresses: ['https://www.w3.org/ns/activitystreams#Public', 'as:Public', 'Public'],
 	acceptableTypes: [
 		'application/activity+json',
-		'application/ld+json; profile="https://www.w3.org/ns/activitystreams"',
+		'application/ld+json;profile="https://www.w3.org/ns/activitystreams"',
 	],
 	acceptedPostTypes: [
 		'Note', 'Page', 'Article', 'Question', 'Video',

src/routes/feeds.js

@@ -61,6 +61,11 @@ async function validateTokenIfRequiresLogin(requiresLogin, cid, req, res) {
 	return true;
 }
 
+function stripUnicodeControlChars(str) {
+	// eslint-disable-next-line no-control-regex
+	return str.replace(/[\u0000-\u0008\u000B\u000C\u000E-\u001F\u007F-\u009F]/g, '');
+}
+
 async function generateForTopic(req, res, next) {
 	if (meta.config['feeds:disableRSS']) {
 		return next();
@@ -80,20 +85,21 @@ async function generateForTopic(req, res, next) {
 	if (await validateTokenIfRequiresLogin(!userPrivileges['topics:read'], topic.cid, req, res)) {
 		const topicData = await topics.getTopicWithPosts(topic, `tid:${tid}:posts`, req.uid || req.query.uid || 0, 0, 24, true);
 
+		const mainPost = topicData.posts[0];
 		topics.modifyPostsByPrivilege(topicData, userPrivileges);
-
+		const title = stripUnicodeControlChars(topicData.title);
 		const feed = new rss({
-			title: utils.stripHTMLTags(topicData.title, utils.tags),
-			description: topicData.posts.length ? topicData.posts[0].content : '',
+			title: utils.stripHTMLTags(title, utils.tags),
+			description: topicData.posts.length ? stripUnicodeControlChars(mainPost.content) : '',
 			feed_url: `${nconf.get('url')}/topic/${tid}.rss`,
 			site_url: `${nconf.get('url')}/topic/${topicData.slug}`,
-			image_url: topicData.posts.length ? topicData.posts[0].picture : '',
-			author: topicData.posts.length ? topicData.posts[0].username : '',
+			image_url: topicData.posts.length ? mainPost.picture : '',
+			author: topicData.posts.length ? mainPost.username : '',
 			ttl: 60,
 		});
 
 		if (topicData.posts.length > 0) {
-			feed.pubDate = new Date(parseInt(topicData.posts[0].timestamp, 10)).toUTCString();
+			feed.pubDate = new Date(parseInt(mainPost.timestamp, 10)).toUTCString();
 		}
 		const replies = topicData.posts.slice(1);
 		replies.forEach((postData) => {
@@ -103,8 +109,8 @@ async function generateForTopic(req, res, next) {
 				).toUTCString();
 
 				feed.item({
-					title: `Reply to ${utils.stripHTMLTags(topicData.title, utils.tags)} on ${dateStamp}`,
-					description: postData.content,
+					title: `Reply to ${utils.stripHTMLTags(title, utils.tags)} on ${dateStamp}`,
+					description: stripUnicodeControlChars(postData.content),
 					url: `${nconf.get('url')}/post/${postData.pid}`,
 					author: postData.user ? postData.user.username : '',
 					date: dateStamp,

src/topics/unread.js

@@ -141,8 +141,8 @@ module.exports = function (Topics) {
 		});
 
 		tids = await privileges.topics.filterTids('topics:read', tids, params.uid);
-		const topicData = (await Topics.getTopicsFields(tids, ['tid', 'cid', 'uid', 'postcount', 'deleted', 'scheduled', 'tags']))
-			.filter(t => t.scheduled || !t.deleted);
+		const topicData = (await Topics.getTopicsFields(tids, ['tid', 'cid', 'uid', 'postcount', 'deleted', 'tags']))
+			.filter(t => !t.deleted);
 		const topicCids = _.uniq(topicData.map(topic => topic.cid)).filter(Boolean);
 
 		const categoryWatchState = await categories.getWatchState(topicCids, params.uid);

src/user/admin.js

@@ -55,7 +55,8 @@ module.exports = function (User) {
 			fields: fieldsToExport,
 			showIps: fieldsToExport.includes('ip'),
 		});
-
+		const customUserFields = await db.getSortedSetRange('user-custom-fields', 0, -1);
+		const fieldsToWrapInQuotes = ['fullname', 'signature', 'aboutme', ...customUserFields];
 		if (!showIps && fields.includes('ip')) {
 			fields.splice(fields.indexOf('ip'), 1);
 		}
@@ -63,7 +64,7 @@ module.exports = function (User) {
 			path.join(baseDir, 'build/export', 'users.csv'),
 			'w'
 		);
-		fs.promises.appendFile(fd, `${fields.map(f => `"${f}"`).join(',')}\n`);
+		await fs.promises.appendFile(fd, `${fields.map(f => `"${f}"`).join(',')}\n`);
 		await batch.processSortedSet('users:joindate', async (uids) => {
 			const userFieldsToLoad = fields.filter(field => field !== 'ip' && field !== 'password');
 			const usersData = await User.getUsersFields(uids, userFieldsToLoad);
@@ -76,6 +77,11 @@ module.exports = function (User) {
 				if (Array.isArray(userIps[index])) {
 					user.ip = userIps[index].join(',');
 				}
+				fieldsToWrapInQuotes.forEach((field) => {
+					if (user[field]) {
+						user[field] = `"${String(user[field])}"`;
+					}
+				});
 			});
 
 			const opts = { fields, header: false };

src/user/digest.js

@@ -84,6 +84,7 @@ Digest.getSubscribers = async function (interval) {
 
 Digest.send = async function (data) {
 	let emailsSent = 0;
+	let emailsFailed = 0;
 	if (!data || !data.subscribers || !data.subscribers.length) {
 		return emailsSent;
 	}
@@ -100,6 +101,7 @@ Digest.send = async function (data) {
 			return;
 		}
 		const userSettings = await user.getMultipleUserSettings(userData.map(u => u.uid));
+		const successfullUids = [];
 		await Promise.all(userData.map(async (userObj, index) => {
 			const userSetting = userSettings[index];
 			const [publicRooms, notifications, topics] = await Promise.all([
@@ -124,57 +126,62 @@ Digest.send = async function (data) {
 				}
 			});
 
-			emailsSent += 1;
-			await emailer.send('digest', userObj.uid, {
-				subject: `[[email:digest.subject, ${date.toLocaleDateString(userSetting.userLang)}]]`,
-				username: userObj.username,
-				userslug: userObj.userslug,
-				notifications: unreadNotifs,
-				publicRooms: publicRooms,
-				recent: topics.recent,
-				topTopics: topics.top,
-				popularTopics: topics.popular,
-				interval: data.interval,
-				showUnsubscribe: true,
-			}).catch((err) => {
+			try {
+				await emailer.send('digest', userObj.uid, {
+					subject: `[[email:digest.subject, ${date.toLocaleDateString(userSetting.userLang)}]]`,
+					username: userObj.username,
+					userslug: userObj.userslug,
+					notifications: unreadNotifs,
+					publicRooms: publicRooms,
+					recent: topics.recent,
+					topTopics: topics.top,
+					popularTopics: topics.popular,
+					interval: data.interval,
+					showUnsubscribe: true,
+				});
+				emailsSent += 1;
+				successfullUids.push(userObj.uid);
+			} catch (err) {
+				emailsFailed += 1;
 				if (!errorLogged) {
 					winston.error(`[user/jobs] Could not send digest email\n[emailer.send] ${err.stack}`);
 					errorLogged = true;
 				}
-			});
+			}
 		}));
-		if (data.interval !== 'alltime') {
+
+		if (data.interval !== 'alltime' && successfullUids.length) {
 			const now = Date.now();
-			await db.sortedSetAdd('digest:delivery', userData.map(() => now), userData.map(u => u.uid));
+			await db.sortedSetAdd('digest:delivery', successfullUids.map(() => now), successfullUids);
 		}
 	}, {
 		interval: 1000,
 		batch: 100,
 	});
-	winston.info(`[user/jobs] Digest (${data.interval}) sending completed. ${emailsSent} emails sent.`);
+	winston.info(`[user/jobs] Digest (${data.interval}) sending completed. ${emailsSent} emails sent. ${emailsFailed} failures.`);
 	return emailsSent;
 };
 
 Digest.getDeliveryTimes = async (start, stop) => {
-	const count = await db.sortedSetCard('users:joindate');
-	const uids = await user.getUidsFromSet('users:joindate', start, stop);
+	const [count, uids] = await Promise.all([
+		db.sortedSetCard('users:joindate'),
+		user.getUidsFromSet('users:joindate', start, stop),
+	]);
 	if (!uids.length) {
-		return [];
+		return { users: [], count };
 	}
 
-	const [scores, settings] = await Promise.all([
+	const [scores, settings, userData] = await Promise.all([
 		// Grab the last time a digest was successfully delivered to these uids
 		db.sortedSetScores('digest:delivery', uids),
 		// Get users' digest settings
 		Digest.getUsersInterval(uids),
+		user.getUsersFields(uids, ['username', 'picture']),
 	]);
 
-	// Populate user data
-	let userData = await user.getUsersFields(uids, ['username', 'picture']);
-	userData = userData.map((user, idx) => {
+	userData.forEach((user, idx) => {
 		user.lastDelivery = scores[idx] ? new Date(scores[idx]).toISOString() : '[[admin/manage/digest:null]]';
 		user.setting = settings[idx];
-		return user;
 	});
 
 	return {

src/webserver.js

@@ -137,6 +137,12 @@ function setupExpressApp(app) {
 	app.set('view engine', 'tpl');
 	app.set('views', viewsDir);
 	app.set('json spaces', process.env.NODE_ENV === 'development' ? 4 : 0);
+
+	// https://github.com/NodeBB/NodeBB/issues/13918
+	const qs = require('qs');
+	app.set('query parser', str => qs.parse(str, {
+		arrayLimit: Math.min(100, nconf.get('queryParser:arrayLimit') || 50),
+	}));
 	app.use(flash());
 
 	app.enable('view cache');