Merge commit 'e63f1234a7fa4c5d9d9be7dafd07be498dcbe1ee' into v4.x

CHANGELOG.md

@@ -1,3 +1,44 @@
+#### v4.0.4 (2025-02-17)
+
+##### Chores
+
+*  up harmony (0fed9a76)
+*  up harmony (ef2c606d)
+*  up harmony (f1da510f)
+*  up deps (fa366095)
+*  up harmony (df07fcfa)
+*  up harmony (de5caf8f)
+*  up harmony (d1f78295)
+*  incrementing version number - v4.0.3 (2b65c735)
+*  update changelog for v4.0.3 (123e1635)
+*  incrementing version number - v4.0.2 (73fe5fcf)
+*  incrementing version number - v4.0.1 (a461b758)
+*  incrementing version number - v4.0.0 (c1eaee45)
+* **i18n:**  fallback strings for new resources: nodebb.themes-harmony (99210918)
+
+##### Bug Fixes
+
+*  clear parsed post cache when updating a post's attachments, #13164 (33d7b9b3)
+*  logic failure causing remote posts with image to not parse properly, #13164 (d936d5c0)
+*  change the passed-in notificatiom id for `notifyTagFollowers` to contain the list of matched tags (04f51cc6)
+*  actor.prune, dont try deleting same users (ffbe4b7b)
+*  getLocalFollowCounts, show non existing deletes (cfbb8ff8)
+*  return null if field isn't in hash (70a9f6d3)
+*  getUserField so that it always returns null (e85662a5)
+*  isArray check (224910b1)
+*  sanity-check the id when mocking a post (5cbf3dd7)
+*  missing actor on some local activities when federating out (040584f0)
+
+##### Performance Improvements
+
+*  closes #13145, reduce calls in actors.prune (d590c2af)
+
+##### Refactors
+
+*  single remove (77dd6dd0)
+*  cleanup ip:recent (d8724708)
+*  hooks button (c4b01330)
+
 #### v4.0.3 (2025-02-09)
 
 ##### Chores

install/package.json

@@ -99,8 +99,8 @@
         "multiparty": "4.2.3",
         "nconf": "0.12.1",
         "nodebb-plugin-2factor": "7.5.9",
-        "nodebb-plugin-composer-default": "10.2.45",
-        "nodebb-plugin-dbsearch": "6.2.9",
+        "nodebb-plugin-composer-default": "10.2.46",
+        "nodebb-plugin-dbsearch": "6.2.12",
         "nodebb-plugin-emoji": "6.0.2",
         "nodebb-plugin-emoji-android": "4.1.1",
         "nodebb-plugin-markdown": "13.1.0",
@@ -108,11 +108,11 @@
         "nodebb-plugin-spam-be-gone": "2.3.1",
         "nodebb-plugin-web-push": "0.7.2",
         "nodebb-rewards-essentials": "1.0.1",
-        "nodebb-theme-harmony": "2.0.25",
+        "nodebb-theme-harmony": "2.0.28",
         "nodebb-theme-lavender": "7.1.17",
-        "nodebb-theme-peace": "2.2.38",
-        "nodebb-theme-persona": "14.0.14",
-        "nodebb-widget-essentials": "7.0.32",
+        "nodebb-theme-peace": "2.2.39",
+        "nodebb-theme-persona": "14.0.15",
+        "nodebb-widget-essentials": "7.0.34",
         "nodemailer": "6.9.16",
         "nprogress": "0.2.0",
         "passport": "0.7.0",
@@ -200,4 +200,4 @@
             "url": "https://github.com/barisusakli"
         }
     ]
-}
+}

public/src/modules/quickreply.js

@@ -38,6 +38,7 @@ define('quickreply', [
 		});
 
 		uploadHelpers.init({
+			uploadBtnEl: $('[component="topic/quickreply/upload/button"]'),
 			dragDropAreaEl: $('[component="topic/quickreply/container"] .quickreply-message'),
 			pasteEl: element,
 			uploadFormEl: $('[component="topic/quickreply/upload"]'),

public/src/modules/uploadHelpers.js

@@ -41,6 +41,7 @@ define('uploadHelpers', ['alerts'], function (alerts) {
 			const fileInput = formEl.find('input[name="files[]"]');
 			options.uploadBtnEl.on('click', function () {
 				fileInput.trigger('click');
+				return false;
 			});
 			fileInput.on('change', function (e) {
 				const files = (e.target || {}).files ||

src/activitypub/contexts.js

@@ -55,8 +55,20 @@ Contexts.getItems = async (uid, id, options) => {
 		options.root = true;
 	}
 
-	activitypub.helpers.log(`[activitypub/context] Retrieving context ${id}`);
-	let { type, items, orderedItems, first, next } = await activitypub.get('uid', uid, id);
+	// Page object instead of id
+	let object;
+	if (!id && options.object) {
+		object = options.object;
+	} else {
+		activitypub.helpers.log(`[activitypub/context] Retrieving context/page ${id}`);
+		try {
+			object = await activitypub.get('uid', uid, id);
+		} catch (e) {
+			return false;
+		}
+	}
+	let { type, items, orderedItems, first, next } = object;
+
 	if (!acceptableTypes.includes(type)) {
 		return false;
 	}
@@ -84,14 +96,18 @@ Contexts.getItems = async (uid, id, options) => {
 
 	if (next) {
 		activitypub.helpers.log('[activitypub/context] Fetching next page...');
+		const isUrl = activitypub.helpers.isUri(next);
 		Array
-			.from(await Contexts.getItems(uid, next, {
+			.from(await Contexts.getItems(uid, isUrl && next, {
 				...options,
 				root: false,
+				object: !isUrl && next,
 			}))
 			.forEach((item) => {
 				chain.add(item);
 			});
+
+		return chain;
 	}
 
 	// Handle special case where originating object is not actually part of the context collection

src/activitypub/helpers.js

@@ -28,6 +28,16 @@ const sha256 = payload => crypto.createHash('sha256').update(payload).digest('he
 
 const Helpers = module.exports;
 
+Helpers._test = (method, args) => {
+	// because I am lazy and I probably wrote some variant of this below code 1000 times already
+	setTimeout(async () => {
+		console.log(await method.apply(method, args));
+	}, 2500);
+};
+// process.nextTick(() => {
+// Helpers._test(activitypub.notes.assert, [1, `https://`]);
+// });
+
 let _lastLog;
 Helpers.log = (message) => {
 	if (!message) {
@@ -54,6 +64,11 @@ Helpers.isUri = (value) => {
 	});
 };
 
+Helpers.assertAccept = accept => (accept && accept.split(',').some((value) => {
+	const parts = value.split(';').map(v => v.trim());
+	return activitypub._constants.acceptableTypes.includes(value || parts[0]);
+}));
+
 Helpers.isWebfinger = (value) => {
 	// N.B. returns normalized handle, so truthy check!
 	if (webfingerRegex.test(value) && !Helpers.isUri(value)) {

src/activitypub/index.js

@@ -59,14 +59,21 @@ ActivityPub.instances = require('./instances');
 ActivityPub.startJobs = () => {
 	ActivityPub.helpers.log('[activitypub/jobs] Registering jobs.');
 	new CronJob('0 0 * * *', async () => {
+		if (!meta.config.activitypubEnabled) {
+			return;
+		}
 		try {
 			await ActivityPub.notes.prune();
+			await db.sortedSetsRemoveRangeByScore(['activities:datetime'], '-inf', Date.now() - 604800000);
 		} catch (err) {
 			winston.error(err.stack);
 		}
 	}, null, true, null, null, false); // change last argument to true for debugging
 
 	new CronJob('*/30 * * * *', async () => {
+		if (!meta.config.activitypubEnabled) {
+			return;
+		}
 		try {
 			await ActivityPub.actors.prune();
 		} catch (err) {

src/activitypub/mocks.js

@@ -341,23 +341,24 @@ Mocks.actors.category = async (cid) => {
 	} = await categories.getCategoryData(cid);
 	const publicKey = await activitypub.getPublicKey('cid', cid);
 
-	let image;
+	let icon;
 	if (backgroundImage) {
 		const filename = path.basename(utils.decodeHTMLEntities(backgroundImage));
-		image = {
+		icon = {
 			type: 'Image',
 			mediaType: mime.getType(filename),
 			url: `${nconf.get('url')}${utils.decodeHTMLEntities(backgroundImage)}`,
 		};
+	} else {
+		icon = await categories.icons.get(cid);
+		icon = icon.get('png');
+		icon = {
+			type: 'Image',
+			mediaType: 'image/png',
+			url: `${nconf.get('url')}${icon}`,
+		};
 	}
 
-	let icon = await categories.icons.get(cid);
-	icon = icon.get('png');
-	icon = {
-		type: 'Image',
-		mediaType: 'image/png',
-		url: `${nconf.get('url')}${icon}`,
-	};
 
 	return {
 		'@context': [
@@ -375,7 +376,7 @@ Mocks.actors.category = async (cid) => {
 		name,
 		preferredUsername,
 		summary,
-		image,
+		// image, // todo once categories have cover photos
 		icon,
 
 		publicKey: {

src/activitypub/notes.js

@@ -71,7 +71,7 @@ Notes.assert = async (uid, input, options = { skipChecks: false }) => {
 	}
 
 	// Reorder chain items by timestamp
-	// chain = chain.sort((a, b) => a.timestamp - b.timestamp);
+	chain = chain.sort((a, b) => a.timestamp - b.timestamp);
 
 	const mainPost = chain[0];
 	let { pid: mainPid, tid, uid: authorId, timestamp, name, content, sourceContent, _activitypub } = mainPost;
@@ -229,6 +229,10 @@ Notes.assertPrivate = async (object) => {
 	// Given an object, adds it to an existing chat or creates a new chat otherwise
 	// todo: context stuff
 
+	if (!object || !object.id || !activitypub.helpers.isUri(object.id)) {
+		return null;
+	}
+
 	const localUids = [];
 	const recipients = new Set([...object.to, ...object.cc]);
 	await Promise.all(Array.from(recipients).map(async (value) => {

src/analytics.js

@@ -28,8 +28,11 @@ const total = _.cloneDeep(local);
 
 const runJobs = nconf.get('runJobs');
 
+Analytics.pause = false;
+
 Analytics.init = async function () {
 	new cronJob('*/10 * * * * *', (async () => {
+		if (Analytics.pause) return;
 		publishLocalAnalytics();
 		if (runJobs) {
 			await sleep(2000);

src/api/utils.js

@@ -52,6 +52,9 @@ utils.tokens.get = async (tokens) => {
 };
 
 utils.tokens.generate = async ({ uid, description }) => {
+	if (!srcUtils.isNumber(uid)) {
+		throw new Error('[[error:invalid-uid]]');
+	}
 	if (parseInt(uid, 10) !== 0) {
 		const uidExists = await user.exists(uid);
 		if (!uidExists) {
@@ -66,7 +69,7 @@ utils.tokens.generate = async ({ uid, description }) => {
 };
 
 utils.tokens.add = async ({ token, uid, description = '', timestamp = Date.now() }) => {
-	if (!token || uid === undefined) {
+	if (!token || uid === undefined || !srcUtils.isNumber(uid)) {
 		throw new Error('[[error:invalid-data]]');
 	}
 
@@ -80,6 +83,9 @@ utils.tokens.add = async ({ token, uid, description = '', timestamp = Date.now()
 };
 
 utils.tokens.update = async (token, { uid, description }) => {
+	if (!srcUtils.isNumber(uid)) {
+		throw new Error('[[error:invalid-uid]]');
+	}
 	await Promise.all([
 		db.setObject(`token:${token}`, { uid, description }),
 		db.sortedSetAdd(`tokens:uid`, uid, token),

src/categories/delete.js

@@ -31,6 +31,9 @@ module.exports = function (Categories) {
 		if (categoryData && categoryData.name) {
 			bulkRemove.push(['categories:name', `${categoryData.name.slice(0, 200).toLowerCase()}:${cid}`]);
 		}
+		if (categoryData && categoryData.handle) {
+			bulkRemove.push(['categoryhandle:cid', categoryData.handle]);
+		}
 		await db.sortedSetRemoveBulk(bulkRemove);
 
 		await removeFromParent(cid);

src/controllers/404.js

@@ -6,6 +6,7 @@ const validator = require('validator');
 
 const meta = require('../meta');
 const plugins = require('../plugins');
+const activitypub = require('../activitypub');
 const middleware = require('../middleware');
 const helpers = require('../middleware/helpers');
 const { secureRandom } = require('../utils');
@@ -24,6 +25,12 @@ exports.handle404 = helpers.try(async (req, res) => {
 
 	if (isClientScript.test(req.url)) {
 		res.type('text/javascript').status(404).send('Not Found');
+	} else if (
+		activitypub.helpers.assertAccept(req.headers.accept) ||
+		(req.headers['Content-Type'] && activitypub._constants.acceptableTypes.includes(req.headers['Content-Type']))
+	) {
+		// todo: separate logging of AP 404s
+		res.sendStatus(404);
 	} else if (
 		!res.locals.isAPI && (
 			req.path.startsWith(`${relativePath}/assets/uploads`) ||

src/controllers/globalmods.js

@@ -1,5 +1,7 @@
 'use strict';
 
+const validator = require('validator');
+
 const user = require('../user');
 const meta = require('../meta');
 const analytics = require('../analytics');
@@ -20,7 +22,7 @@ globalModsController.ipBlacklist = async function (req, res, next) {
 	]);
 	res.render('ip-blacklist', {
 		title: '[[pages:ip-blacklist]]',
-		rules: rules,
+		rules: validator.escape(String(rules)),
 		analytics: analyticsData,
 		breadcrumbs: helpers.buildBreadcrumbs([{ text: '[[pages:ip-blacklist]]' }]),
 	});

src/middleware/activitypub.js

@@ -16,10 +16,8 @@ middleware.assertS2S = async function (req, res, next) {
 		return next('route');
 	}
 
-	const pass = (accept && accept.split(',').some((value) => {
-		const parts = value.split(';').map(v => v.trim());
-		return activitypub._constants.acceptableTypes.includes(value || parts[0]);
-	})) || (contentType && activitypub._constants.acceptableTypes.includes(contentType));
+	const pass = activitypub.helpers.assertAccept(accept) ||
+		(contentType && activitypub._constants.acceptableTypes.includes(contentType));
 
 	if (!pass) {
 		return next('route');

src/pagination.js

@@ -29,8 +29,7 @@ pagination.create = function (currentPage, pageCount, queryObj) {
 	if (startPage > pageCount - 5) {
 		startPage -= 2 - (pageCount - currentPage);
 	}
-	let i;
-	for (i = 0; i < 5; i += 1) {
+	for (let i = 0; i < 5; i += 1) {
 		pagesToShow.push(startPage + i);
 	}
 
@@ -45,10 +44,11 @@ pagination.create = function (currentPage, pageCount, queryObj) {
 		return { page: page, active: page === currentPage, qs: qs.stringify(queryObj) };
 	});
 
-	for (i = pages.length - 1; i > 0; i -= 1) {
+	for (let i = pages.length - 1; i > 0; i -= 1) {
+		const prevPage = pages[i].page - 1;
 		if (pages[i].page - 2 === pages[i - 1].page) {
-			pages.splice(i, 0, { page: pages[i].page - 1, active: false, qs: qs.stringify(queryObj) });
-		} else if (pages[i].page - 1 !== pages[i - 1].page) {
+			pages.splice(i, 0, { page: prevPage, active: false, qs: qs.stringify({ ...queryObj, page: prevPage }) });
+		} else if (prevPage !== pages[i - 1].page) {
 			pages.splice(i, 0, { separator: true });
 		}
 	}

src/posts/create.js

@@ -11,8 +11,6 @@ const privileges = require('../privileges');
 const activitypub = require('../activitypub');
 const utils = require('../utils');
 
-const isEmojiShortcode = /^:[\w]+:$/;
-
 module.exports = function (Posts) {
 	Posts.create = async function (data) {
 		// This is an internal method, consider using Topics.reply instead
@@ -54,9 +52,15 @@ module.exports = function (Posts) {
 		if (_activitypub && _activitypub.tag && Array.isArray(_activitypub.tag)) {
 			_activitypub.tag
 				.filter(tag => tag.type === 'Emoji' &&
-					isEmojiShortcode.test(tag.name) &&
-					tag.icon && tag.icon.mediaType && tag.icon.mediaType.startsWith('image/'))
+					tag.icon && tag.icon.type === 'Image')
 				.forEach((tag) => {
+					if (!tag.name.startsWith(':')) {
+						tag.name = `:${tag.name}`;
+					}
+					if (!tag.name.endsWith(':')) {
+						tag.name = `${tag.name}:`;
+					}
+
 					postData.content = postData.content.replace(new RegExp(tag.name, 'g'), `<img class="not-responsive emoji" src="${tag.icon.url}" title="${tag.name}" />`);
 				});
 		}

src/topics/posts.js

@@ -192,7 +192,7 @@ module.exports = function (Topics) {
 		const pidToPrivs = _.zipObject(parentPids, postPrivileges);
 
 		parentPids = parentPids.filter(p => pidToPrivs[p]['topics:read']);
-		const parentPosts = await posts.getPostsFields(parentPids, ['uid', 'pid', 'timestamp', 'content', 'deleted']);
+		const parentPosts = await posts.getPostsFields(parentPids, ['uid', 'pid', 'timestamp', 'content', 'sourceContent', 'deleted']);
 		const parentUids = _.uniq(parentPosts.map(postObj => postObj && postObj.uid));
 		const userData = await user.getUsersFields(parentUids, ['username', 'userslug', 'picture']);
 

src/views/admin/partials/edit-token-modal.tpl

@@ -1,13 +1,13 @@
 <form role="form">
     <div class="mb-3">
         <label class="form-label" for="uid">[[admin/settings/api:uid]]</label>
-        <input type="text" inputmode="numeric" pattern="\d+" name="uid" class="form-control" placeholder="0" value="{./uid}" />
+        <input id="uid" type="number" inputmode="numeric" pattern="\d+" name="uid" class="form-control" placeholder="0" value="{./uid}" />
         <p class="form-text">
             [[admin/settings/api:uid-help-text]]
         </p>
     </div>
     <div class="mb-3">
         <label class="form-label" for="description">[[admin/settings/api:description]]</label>
-        <input type="text" name="description" class="form-control" placeholder="Description" value="{./description}" />
+        <input id="description" type="text" name="description" class="form-control" placeholder="Description" value="{./description}" />
     </div>
 </form>

test/activitypub.js

@@ -350,6 +350,79 @@ describe('ActivityPub integration', () => {
 		});
 	});
 
+	describe.only('Category Actor endpoint', () => {
+		let cid;
+		let slug;
+		let description;
+
+		beforeEach(async () => {
+			slug = slugify(utils.generateUUID().slice(0, 8));
+			description = utils.generateUUID();
+			({ cid } = await categories.create({
+				name: slug,
+				description,
+			}));
+		});
+
+		it('should return a valid ActivityPub Actor JSON-LD payload', async () => {
+			const { response, body } = await request.get(`${nconf.get('url')}/category/${cid}`, {
+				headers: {
+					Accept: 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"',
+				},
+			});
+
+			assert(response);
+			assert.strictEqual(response.statusCode, 200);
+			assert(body.hasOwnProperty('@context'));
+			assert(body['@context'].includes('https://www.w3.org/ns/activitystreams'));
+
+			['id', 'url', /* 'followers', 'following', */ 'inbox', 'outbox'].forEach((prop) => {
+				assert(body.hasOwnProperty(prop));
+				assert(body[prop]);
+			});
+
+			assert.strictEqual(body.id, `${nconf.get('url')}/category/${cid}`);
+			assert.strictEqual(body.type, 'Group');
+			assert.strictEqual(body.summary, description);
+			assert.deepStrictEqual(body.icon, {
+				type: 'Image',
+				mediaType: 'image/png',
+				url: `${nconf.get('url')}/assets/uploads/category/category-${cid}-icon.png`,
+			});
+		});
+
+		it('should contain a `publicKey` property with a public key', async () => {
+			const { body } = await request.get(`${nconf.get('url')}/category/${cid}`, {
+				headers: {
+					Accept: 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"',
+				},
+			});
+
+			assert(body.hasOwnProperty('publicKey'));
+			assert(['id', 'owner', 'publicKeyPem'].every(prop => body.publicKey.hasOwnProperty(prop)));
+		});
+
+		it('should serve the the backgroundImage in `icon` if set', async () => {
+			const payload = {};
+			payload[cid] = {
+				backgroundImage: `/assets/uploads/files/test.png`,
+			};
+			await categories.update(payload);
+
+			const { body } = await request.get(`${nconf.get('url')}/category/${cid}`, {
+				headers: {
+					Accept: 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"',
+				},
+			});
+
+			assert.deepStrictEqual(body.icon, {
+				type: 'Image',
+				mediaType: 'image/png',
+				url: `${nconf.get('url')}/assets/uploads/files/test.png`,
+			});
+		});
+	});
+
 	describe('Instance Actor endpoint', () => {
 		let response;
 		let body;

test/activitypub/analytics.js

@@ -126,9 +126,9 @@ describe('Analytics', () => {
 
 	it('should increment various metrics', async () => {
 		let counters;
+		analytics.pause = true;
 		({ counters } = analytics.peek());
 		const before = { ...counters };
-
 		const id = `https://example.org/activity/${utils.generateUUID()}`;
 		await controllers.activitypub.postInbox({
 			body: {
@@ -147,8 +147,9 @@ describe('Analytics', () => {
 
 		const metrics = ['activities', 'activities:byType:Like', 'activities:byHost:example.org'];
 		metrics.forEach((metric) => {
-			assert(before[metric] && after[metric]);
+			assert(before[metric] && after[metric], JSON.stringify({ before, after }, null, 2));
 			assert(before[metric] < after[metric]);
 		});
+		analytics.pause = false;
 	});
 });

test/pagination.js

@@ -26,6 +26,18 @@ describe('Pagination', () => {
 		done();
 	});
 
+	it('should create pagination for 18 pages and should not turn page 3 into separator', (done) => {
+		const data = pagination.create(6, 18);
+		// [1, 2, 3, 4, 5, (6), 7, 8, seperator, 17, 18]
+		assert.equal(data.pages.length, 11);
+		assert.equal(data.rel.length, 2);
+		assert.strictEqual(data.pages[2].qs, 'page=3');
+		assert.equal(data.pageCount, 18);
+		assert.equal(data.prev.page, 5);
+		assert.equal(data.next.page, 7);
+		done();
+	});
+
 	it('should create pagination for 3 pages with query params', (done) => {
 		const data = pagination.create(1, 3, { key: 'value' });
 		assert.equal(data.pages.length, 3);