From 7a9e09a696589d64712e5f16ef4ee7d3c363752e Mon Sep 17 00:00:00 2001 From: Misty Release Bot Date: Thu, 4 Sep 2025 16:02:47 +0000 Subject: [PATCH 01/34] chore: incrementing version number - v4.5.1 --- install/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install/package.json b/install/package.json index 14e65fabd7..6a5457e093 100644 --- a/install/package.json +++ b/install/package.json @@ -2,7 +2,7 @@ "name": "nodebb", "license": "GPL-3.0", "description": "NodeBB Forum", - "version": "4.5.0", + "version": "4.5.1", "homepage": "https://www.nodebb.org", "repository": { "type": "git", From a9fffd7ca07416184a2f6ec512b7cca95502e8b2 Mon Sep 17 00:00:00 2001 From: Misty Release Bot Date: Thu, 4 Sep 2025 16:02:47 +0000 Subject: [PATCH 02/34] chore: update changelog for v4.5.1 --- CHANGELOG.md | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 96dd2a28f1..beaab4c40c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,45 @@ +#### v4.5.1 (2025-09-04) + +##### Chores + +* up dbsearch (c07e81d2) +* incrementing version number - v4.5.0 (f05c5d06) +* update changelog for v4.5.0 (86d03b1e) +* incrementing version number - v4.4.6 (074043ad) +* incrementing version number - v4.4.5 (6f106923) +* incrementing version number - v4.4.4 (d323af44) +* incrementing version number - v4.4.3 (d354c2eb) +* incrementing version number - v4.4.2 (55c510ae) +* incrementing version number - v4.4.1 (5ae79b4e) +* incrementing version number - v4.4.0 (0a75eee3) +* incrementing version number - v4.3.2 (b92b5d80) +* incrementing version number - v4.3.1 (308e6b9f) +* incrementing version number - v4.3.0 (bff291db) +* incrementing version number - v4.2.2 (17fecc24) +* incrementing version number - v4.2.1 (852a270c) +* incrementing version number - v4.2.0 (87581958) +* incrementing version number - v4.1.1 (b2afbb16) +* incrementing version number - v4.1.0 (36c80850) +* incrementing version number - v4.0.6 (4a52fb2e) +* incrementing version number - v4.0.5 (1792a62b) +* incrementing version number - v4.0.4 (b1125cce) +* incrementing version number - v4.0.3 (2b65c735) +* incrementing version number - v4.0.2 (73fe5fcf) +* incrementing version number - v4.0.1 (a461b758) +* incrementing version number - v4.0.0 (c1eaee45) + +##### New Features + +* use _variables.scss overrides from acp in custom skins and bootswatch skins as well (0c48e0e9) + +##### Bug Fixes + +* remove unused dependency (8d7e3537) +* remove test for 1b12 announce on topic move (as this no longer occurs) (9221d34f) +* use existing id if checkHeader returns false (e6996846) +* regression that caused Piefed (or potentially others) content to be dropped on receipt (86d9016f) +* remove faulty code that tried to announce a remote object but couldn't as the ID was not a number (7adfe39e) + #### v4.5.0 (2025-09-03) ##### Chores From 290a9395c095f2589804f954fc0f45bebb74b0ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bar=C4=B1=C5=9F=20Soner=20U=C5=9Fakl=C4=B1?= Date: Sat, 6 Sep 2025 13:47:46 -0400 Subject: [PATCH 03/34] fix: pass object to.auth --- src/database/redis/connection.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/database/redis/connection.js b/src/database/redis/connection.js index fb2aceb33b..8792bbdf3f 100644 --- a/src/database/redis/connection.js +++ b/src/database/redis/connection.js @@ -71,7 +71,7 @@ connection.connect = async function (options) { }); if (options.password) { - cxn.auth(options.password); + cxn.auth({ password: options.password }); } }); }; From b3ffa00789f17d7a33aade67ac2f0ee6b8d29a10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bar=C4=B1=C5=9F=20Soner=20U=C5=9Fakl=C4=B1?= Date: Mon, 8 Sep 2025 09:29:32 -0400 Subject: [PATCH 04/34] fix: closes #13641, log test email sending errors server side --- src/socket.io/admin/email.js | 96 +++++++++++++++++++----------------- 1 file changed, 51 insertions(+), 45 deletions(-) diff --git a/src/socket.io/admin/email.js b/src/socket.io/admin/email.js index ed5bce7a60..b2a160b1f3 100644 --- a/src/socket.io/admin/email.js +++ b/src/socket.io/admin/email.js @@ -1,5 +1,7 @@ 'use strict'; +const winston = require('winston'); + const meta = require('../../meta'); const userDigest = require('../../user/digest'); const userEmail = require('../../user/email'); @@ -14,55 +16,59 @@ Email.test = async function (socket, data) { ...(data.payload || {}), subject: '[[email:test-email.subject]]', }; + try { + switch (data.template) { + case 'digest': + await userDigest.execute({ + interval: 'month', + subscribers: [socket.uid], + }); + break; - switch (data.template) { - case 'digest': - await userDigest.execute({ - interval: 'month', - subscribers: [socket.uid], - }); - break; + case 'banned': + Object.assign(payload, { + username: 'test-user', + until: utils.toISOString(Date.now()), + reason: 'Test Reason', + }); + await emailer.send(data.template, socket.uid, payload); + break; - case 'banned': - Object.assign(payload, { - username: 'test-user', - until: utils.toISOString(Date.now()), - reason: 'Test Reason', - }); - await emailer.send(data.template, socket.uid, payload); - break; + case 'verify-email': + case 'welcome': + await userEmail.sendValidationEmail(socket.uid, { + force: 1, + template: data.template, + subject: data.template === 'welcome' ? `[[email:welcome-to, ${meta.config.title || meta.config.browserTitle || 'NodeBB'}]]` : undefined, + }); + break; - case 'verify-email': - case 'welcome': - await userEmail.sendValidationEmail(socket.uid, { - force: 1, - template: data.template, - subject: data.template === 'welcome' ? `[[email:welcome-to, ${meta.config.title || meta.config.browserTitle || 'NodeBB'}]]` : undefined, - }); - break; + case 'notification': { + const notification = await notifications.create({ + type: 'test', + bodyShort: '[[email:notif.test.short]]', + bodyLong: '[[email:notif.test.long]]', + nid: `uid:${socket.uid}:test`, + path: '/', + from: socket.uid, + }); + await emailer.send('notification', socket.uid, { + path: notification.path, + subject: utils.stripHTMLTags(notification.subject || '[[notifications:new-notification]]'), + intro: utils.stripHTMLTags(notification.bodyShort), + body: notification.bodyLong || '', + notification, + showUnsubscribe: true, + }); + break; + } - case 'notification': { - const notification = await notifications.create({ - type: 'test', - bodyShort: '[[email:notif.test.short]]', - bodyLong: '[[email:notif.test.long]]', - nid: `uid:${socket.uid}:test`, - path: '/', - from: socket.uid, - }); - await emailer.send('notification', socket.uid, { - path: notification.path, - subject: utils.stripHTMLTags(notification.subject || '[[notifications:new-notification]]'), - intro: utils.stripHTMLTags(notification.bodyShort), - body: notification.bodyLong || '', - notification, - showUnsubscribe: true, - }); - break; + default: + await emailer.send(data.template, socket.uid, payload); + break; } - - default: - await emailer.send(data.template, socket.uid, payload); - break; + } catch (err) { + winston.error(err.stack); + throw err; } }; From 527f27af2948af908006af7fe3683a2aee207fc5 Mon Sep 17 00:00:00 2001 From: Julian Lam Date: Mon, 8 Sep 2025 12:00:32 -0400 Subject: [PATCH 05/34] fix: make auto-categorization logic case-insensitive --- src/activitypub/notes.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/activitypub/notes.js b/src/activitypub/notes.js index ce9371fd26..3b15ab0137 100644 --- a/src/activitypub/notes.js +++ b/src/activitypub/notes.js @@ -399,15 +399,18 @@ async function assertRelation(post) { } async function assignCategory(post) { + activitypub.helpers.log('[activitypub] Checking auto-categorization rules.'); let cid = undefined; const rules = await activitypub.rules.list(); - const tags = await Notes._normalizeTags(post._activitypub.tag || []); + let tags = await Notes._normalizeTags(post._activitypub.tag || []); + tags = tags.map(tag => tag.toLowerCase()); cid = rules.reduce((cid, { type, value, cid: target }) => { if (!cid) { switch (type) { case 'hashtag': { - if (tags.includes(value)) { + if (tags.includes(value.toLowerCase())) { + activitypub.helpers.log(`[activitypub] - Rule match: #${value}; cid: ${target}`); return target; } break; From 0311b98ed7d926502f27a7d84b00c02b73712dc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bar=C4=B1=C5=9F=20Soner=20U=C5=9Fakl=C4=B1?= Date: Wed, 10 Sep 2025 09:46:39 -0400 Subject: [PATCH 06/34] feat: add topic templates per category, closes #13649 --- install/package.json | 2 +- public/language/en-GB/admin/manage/categories.json | 2 ++ src/views/admin/manage/category.tpl | 10 ++++++++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/install/package.json b/install/package.json index 6a5457e093..9e94fa1ed3 100644 --- a/install/package.json +++ b/install/package.json @@ -97,7 +97,7 @@ "multer": "2.0.2", "nconf": "0.13.0", "nodebb-plugin-2factor": "7.5.10", - "nodebb-plugin-composer-default": "10.3.0", + "nodebb-plugin-composer-default": "10.3.1", "nodebb-plugin-dbsearch": "6.3.2", "nodebb-plugin-emoji": "6.0.3", "nodebb-plugin-emoji-android": "4.1.1", diff --git a/public/language/en-GB/admin/manage/categories.json b/public/language/en-GB/admin/manage/categories.json index d66dd814a1..7532cd9cd1 100644 --- a/public/language/en-GB/admin/manage/categories.json +++ b/public/language/en-GB/admin/manage/categories.json @@ -17,6 +17,8 @@ "federatedDescription": "Federated Description", "federatedDescription.help": "This text will be appended to the category description when queried by other websites/apps.", "federatedDescription.default": "This is a forum category containing topical discussion. You can start new discussions by mentioning this category.", + "topic-template": "Topic Template", + "topic-template.help": "Define a template for new topics created in this category.", "bg-color": "Background Colour", "text-color": "Text Colour", "bg-image-size": "Background Image Size", diff --git a/src/views/admin/manage/category.tpl b/src/views/admin/manage/category.tpl index 5d3b84c639..b935638ac4 100644 --- a/src/views/admin/manage/category.tpl +++ b/src/views/admin/manage/category.tpl @@ -46,6 +46,16 @@

+
+ + +

+ [[admin/manage/categories:topic-template.help]] +

+
+
From 953c051c2e546364a8a28d4097ebceac1876bcac Mon Sep 17 00:00:00 2001 From: Julian Lam Date: Wed, 10 Sep 2025 14:59:13 -0400 Subject: [PATCH 07/34] fix: perform Link header check on note assertion only when skipChecks is falsy --- src/activitypub/notes.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/activitypub/notes.js b/src/activitypub/notes.js index 3b15ab0137..50ce01bf3c 100644 --- a/src/activitypub/notes.js +++ b/src/activitypub/notes.js @@ -63,7 +63,9 @@ Notes.assert = async (uid, input, options = { skipChecks: false }) => { return null; } - id = (await activitypub.checkHeader(id)) || id; + if (!options.skipChecks) { + id = (await activitypub.checkHeader(id)) || id; + } let chain; let context = await activitypub.contexts.get(uid, id); From f9688b36b67e1706cafacce69f784a9e8e5aeaa0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bar=C4=B1=C5=9F=20Soner=20U=C5=9Fakl=C4=B1?= Date: Thu, 11 Sep 2025 17:44:34 -0400 Subject: [PATCH 08/34] fix: port the try/catch for notes.assert from develop --- src/activitypub/notes.js | 393 ++++++++++++++++++++------------------- 1 file changed, 200 insertions(+), 193 deletions(-) diff --git a/src/activitypub/notes.js b/src/activitypub/notes.js index 50ce01bf3c..733aff354f 100644 --- a/src/activitypub/notes.js +++ b/src/activitypub/notes.js @@ -63,212 +63,219 @@ Notes.assert = async (uid, input, options = { skipChecks: false }) => { return null; } - if (!options.skipChecks) { - id = (await activitypub.checkHeader(id)) || id; - } - - let chain; - let context = await activitypub.contexts.get(uid, id); - if (context.tid) { - await unlock(id); - const { tid } = context; - return { tid, count: 0 }; - } else if (context.context) { - chain = Array.from(await activitypub.contexts.getItems(uid, context.context, { input })); - if (chain && chain.length) { - // Context resolves, use in later topic creation - context = context.context; - } - } else { - context = undefined; - } - - if (!chain || !chain.length) { - // Fall back to inReplyTo traversal on context retrieval failure - chain = Array.from(await Notes.getParentChain(uid, input)); - chain.reverse(); - } - - // Can't resolve — give up. - if (!chain.length) { - await unlock(id); - return null; - } - - // Reorder chain items by timestamp - chain = chain.sort((a, b) => a.timestamp - b.timestamp); - - const mainPost = chain[0]; - let { pid: mainPid, tid, uid: authorId, timestamp, title, content, sourceContent, _activitypub } = mainPost; - const hasTid = !!tid; - - const cid = hasTid ? await topics.getTopicField(tid, 'cid') : options.cid || -1; - - if (options.cid && cid === -1) { - // Move topic if currently uncategorized - await topics.tools.move(tid, { cid: options.cid, uid: 'system' }); - } - - const members = await db.isSortedSetMembers(`tid:${tid}:posts`, chain.slice(1).map(p => p.pid)); - members.unshift(await posts.exists(mainPid)); - if (tid && members.every(Boolean)) { - // All cached, return early. - activitypub.helpers.log('[notes/assert] No new notes to process.'); - await unlock(id); - return { tid, count: 0 }; - } - - if (hasTid) { - mainPid = await topics.getTopicField(tid, 'mainPid'); - } else { - // Check recipients/audience for category (local or remote) - const set = activitypub.helpers.makeSet(_activitypub, ['to', 'cc', 'audience']); - await activitypub.actors.assert(Array.from(set)); - - // Local - const resolved = await Promise.all(Array.from(set).map(async id => await activitypub.helpers.resolveLocalId(id))); - const recipientCids = resolved - .filter(Boolean) - .filter(({ type }) => type === 'category') - .map(obj => obj.id); - - // Remote - let remoteCid; - const assertedGroups = await categories.exists(Array.from(set)); - try { - const { hostname } = new URL(mainPid); - remoteCid = Array.from(set).filter((id, idx) => { - const { hostname: cidHostname } = new URL(id); - return assertedGroups[idx] && cidHostname === hostname; - }).shift(); - } catch (e) { - // noop - winston.error('[activitypub/notes.assert] Could not parse URL of mainPid', e.stack); + try { + if (!options.skipChecks) { + id = (await activitypub.checkHeader(id)) || id; } - if (remoteCid || recipientCids.length) { - // Overrides passed-in value, respect addressing from main post over booster - options.cid = remoteCid || recipientCids.shift(); + let chain; + let context = await activitypub.contexts.get(uid, id); + if (context.tid) { + await unlock(id); + const { tid } = context; + return { tid, count: 0 }; + } else if (context.context) { + chain = Array.from(await activitypub.contexts.getItems(uid, context.context, { input })); + if (chain && chain.length) { + // Context resolves, use in later topic creation + context = context.context; + } + } else { + context = undefined; } - // Auto-categorization (takes place only if all other categorization efforts fail) - if (!options.cid) { - options.cid = await assignCategory(mainPost); + if (!chain || !chain.length) { + // Fall back to inReplyTo traversal on context retrieval failure + chain = Array.from(await Notes.getParentChain(uid, input)); + chain.reverse(); } - // mainPid ok to leave as-is - if (!title) { - const sentences = tokenizer.sentences(content || sourceContent, { sanitize: true }); - title = sentences.shift(); - } - - // Remove any custom emoji from title - if (_activitypub && _activitypub.tag && Array.isArray(_activitypub.tag)) { - _activitypub.tag - .filter(tag => tag.type === 'Emoji') - .forEach((tag) => { - title = title.replace(new RegExp(tag.name, 'g'), ''); - }); - } - } - mainPid = utils.isNumber(mainPid) ? parseInt(mainPid, 10) : mainPid; - - // Relation & privilege check for local categories - const inputIndex = chain.map(n => n.pid).indexOf(id); - const hasRelation = - uid || hasTid || - options.skipChecks || options.cid || - await assertRelation(chain[inputIndex !== -1 ? inputIndex : 0]); - - const privilege = `topics:${tid ? 'reply' : 'create'}`; - const allowed = await privileges.categories.can(privilege, options.cid || cid, activitypub._constants.uid); - if (!hasRelation || !allowed) { - if (!hasRelation) { - activitypub.helpers.log(`[activitypub/notes.assert] Not asserting ${id} as it has no relation to existing tracked content.`); - } - - await unlock(id); - return null; - } - - tid = tid || utils.generateUUID(); - mainPost.tid = tid; - - const urlMap = chain.reduce((map, post) => (post.url ? map.set(post.url, post.id) : map), new Map()); - const unprocessed = chain.map((post) => { - post.tid = tid; // add tid to post hash - - // Ensure toPids in replies are ids - if (urlMap.has(post.toPid)) { - post.toPid = urlMap.get(post.toPid); - } - - return post; - }).filter((p, idx) => !members[idx]); - const count = unprocessed.length; - activitypub.helpers.log(`[notes/assert] ${count} new note(s) found.`); - - if (!hasTid) { - const { to, cc, attachment } = mainPost._activitypub; - const tags = await Notes._normalizeTags(mainPost._activitypub.tag || []); - - try { - await topics.post({ - tid, - uid: authorId, - cid: options.cid || cid, - pid: mainPid, - title, - timestamp, - tags, - content: mainPost.content, - sourceContent: mainPost.sourceContent, - _activitypub: mainPost._activitypub, - }); - unprocessed.shift(); - } catch (e) { - activitypub.helpers.log(`[activitypub/notes.assert] Could not post topic (${mainPost.pid}): ${e.message}`); + // Can't resolve — give up. + if (!chain.length) { + await unlock(id); return null; } - // These must come after topic is posted + // Reorder chain items by timestamp + chain = chain.sort((a, b) => a.timestamp - b.timestamp); + + const mainPost = chain[0]; + let { pid: mainPid, tid, uid: authorId, timestamp, title, content, sourceContent, _activitypub } = mainPost; + const hasTid = !!tid; + + const cid = hasTid ? await topics.getTopicField(tid, 'cid') : options.cid || -1; + + if (options.cid && cid === -1) { + // Move topic if currently uncategorized + await topics.tools.move(tid, { cid: options.cid, uid: 'system' }); + } + + const members = await db.isSortedSetMembers(`tid:${tid}:posts`, chain.slice(1).map(p => p.pid)); + members.unshift(await posts.exists(mainPid)); + if (tid && members.every(Boolean)) { + // All cached, return early. + activitypub.helpers.log('[notes/assert] No new notes to process.'); + await unlock(id); + return { tid, count: 0 }; + } + + if (hasTid) { + mainPid = await topics.getTopicField(tid, 'mainPid'); + } else { + // Check recipients/audience for category (local or remote) + const set = activitypub.helpers.makeSet(_activitypub, ['to', 'cc', 'audience']); + await activitypub.actors.assert(Array.from(set)); + + // Local + const resolved = await Promise.all(Array.from(set).map(async id => await activitypub.helpers.resolveLocalId(id))); + const recipientCids = resolved + .filter(Boolean) + .filter(({ type }) => type === 'category') + .map(obj => obj.id); + + // Remote + let remoteCid; + const assertedGroups = await categories.exists(Array.from(set)); + try { + const { hostname } = new URL(mainPid); + remoteCid = Array.from(set).filter((id, idx) => { + const { hostname: cidHostname } = new URL(id); + return assertedGroups[idx] && cidHostname === hostname; + }).shift(); + } catch (e) { + // noop + winston.error('[activitypub/notes.assert] Could not parse URL of mainPid', e.stack); + } + + if (remoteCid || recipientCids.length) { + // Overrides passed-in value, respect addressing from main post over booster + options.cid = remoteCid || recipientCids.shift(); + } + + // Auto-categorization (takes place only if all other categorization efforts fail) + if (!options.cid) { + options.cid = await assignCategory(mainPost); + } + + // mainPid ok to leave as-is + if (!title) { + const sentences = tokenizer.sentences(content || sourceContent, { sanitize: true }); + title = sentences.shift(); + } + + // Remove any custom emoji from title + if (_activitypub && _activitypub.tag && Array.isArray(_activitypub.tag)) { + _activitypub.tag + .filter(tag => tag.type === 'Emoji') + .forEach((tag) => { + title = title.replace(new RegExp(tag.name, 'g'), ''); + }); + } + } + mainPid = utils.isNumber(mainPid) ? parseInt(mainPid, 10) : mainPid; + + // Relation & privilege check for local categories + const inputIndex = chain.map(n => n.pid).indexOf(id); + const hasRelation = + uid || hasTid || + options.skipChecks || options.cid || + await assertRelation(chain[inputIndex !== -1 ? inputIndex : 0]); + + const privilege = `topics:${tid ? 'reply' : 'create'}`; + const allowed = await privileges.categories.can(privilege, options.cid || cid, activitypub._constants.uid); + if (!hasRelation || !allowed) { + if (!hasRelation) { + activitypub.helpers.log(`[activitypub/notes.assert] Not asserting ${id} as it has no relation to existing tracked content.`); + } + + await unlock(id); + return null; + } + + tid = tid || utils.generateUUID(); + mainPost.tid = tid; + + const urlMap = chain.reduce((map, post) => (post.url ? map.set(post.url, post.id) : map), new Map()); + const unprocessed = chain.map((post) => { + post.tid = tid; // add tid to post hash + + // Ensure toPids in replies are ids + if (urlMap.has(post.toPid)) { + post.toPid = urlMap.get(post.toPid); + } + + return post; + }).filter((p, idx) => !members[idx]); + const count = unprocessed.length; + activitypub.helpers.log(`[notes/assert] ${count} new note(s) found.`); + + if (!hasTid) { + const { to, cc, attachment } = mainPost._activitypub; + const tags = await Notes._normalizeTags(mainPost._activitypub.tag || []); + + try { + await topics.post({ + tid, + uid: authorId, + cid: options.cid || cid, + pid: mainPid, + title, + timestamp, + tags, + content: mainPost.content, + sourceContent: mainPost.sourceContent, + _activitypub: mainPost._activitypub, + }); + unprocessed.shift(); + } catch (e) { + activitypub.helpers.log(`[activitypub/notes.assert] Could not post topic (${mainPost.pid}): ${e.message}`); + await unlock(id); + return null; + } + + // These must come after topic is posted + await Promise.all([ + Notes.updateLocalRecipients(mainPid, { to, cc }), + mainPost._activitypub.image ? topics.thumbs.associate({ + id: tid, + path: mainPost._activitypub.image, + }) : null, + posts.attachments.update(mainPid, attachment), + ]); + + if (context) { + activitypub.helpers.log(`[activitypub/notes.assert] Associating tid ${tid} with context ${context}`); + await topics.setTopicField(tid, 'context', context); + } + } + + for (const post of unprocessed) { + const { to, cc, attachment } = post._activitypub; + + try { + // eslint-disable-next-line no-await-in-loop + await topics.reply(post); + // eslint-disable-next-line no-await-in-loop + await Promise.all([ + Notes.updateLocalRecipients(post.pid, { to, cc }), + posts.attachments.update(post.pid, attachment), + ]); + } catch (e) { + activitypub.helpers.log(`[activitypub/notes.assert] Could not add reply (${post.pid}): ${e.message}`); + } + } + await Promise.all([ - Notes.updateLocalRecipients(mainPid, { to, cc }), - mainPost._activitypub.image ? topics.thumbs.associate({ - id: tid, - path: mainPost._activitypub.image, - }) : null, - posts.attachments.update(mainPid, attachment), + Notes.syncUserInboxes(tid, uid), + unlock(id), ]); - if (context) { - activitypub.helpers.log(`[activitypub/notes.assert] Associating tid ${tid} with context ${context}`); - await topics.setTopicField(tid, 'context', context); - } + return { tid, count }; + } catch (e) { + winston.warn(`[activitypub/notes.assert] Could not assert ${id} (${e.message}), releasing lock.`); + await unlock(id); + return null; } - - for (const post of unprocessed) { - const { to, cc, attachment } = post._activitypub; - - try { - // eslint-disable-next-line no-await-in-loop - await topics.reply(post); - // eslint-disable-next-line no-await-in-loop - await Promise.all([ - Notes.updateLocalRecipients(post.pid, { to, cc }), - posts.attachments.update(post.pid, attachment), - ]); - } catch (e) { - activitypub.helpers.log(`[activitypub/notes.assert] Could not add reply (${post.pid}): ${e.message}`); - } - } - - await Promise.all([ - Notes.syncUserInboxes(tid, uid), - unlock(id), - ]); - - return { tid, count }; }; Notes.assertPrivate = async (object) => { From f9ddbebacc76c2601d5dbb8adf97b44cb32395b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bar=C4=B1=C5=9F=20Soner=20U=C5=9Fakl=C4=B1?= Date: Fri, 12 Sep 2025 11:33:53 -0400 Subject: [PATCH 09/34] fix: remove .auth call --- src/database/redis/connection.js | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/database/redis/connection.js b/src/database/redis/connection.js index 8792bbdf3f..7f0b1912f8 100644 --- a/src/database/redis/connection.js +++ b/src/database/redis/connection.js @@ -69,10 +69,6 @@ connection.connect = async function (options) { }).catch((err) => { winston.error('Error connecting to Redis:', err); }); - - if (options.password) { - cxn.auth({ password: options.password }); - } }); }; From 56fad0be0d0da2c1f3a12562d07a8f2adab3bffa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bar=C4=B1=C5=9F=20Soner=20U=C5=9Fakl=C4=B1?= Date: Fri, 12 Sep 2025 19:19:52 -0400 Subject: [PATCH 10/34] fix: check brand:touchIcon for correct path --- src/middleware/index.js | 14 ++++++++++---- test/controllers.js | 12 +++++++++++- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/src/middleware/index.js b/src/middleware/index.js index 67d8e2faa0..14cb3138e1 100644 --- a/src/middleware/index.js +++ b/src/middleware/index.js @@ -145,12 +145,18 @@ middleware.logApiUsage = async function logApiUsage(req, res, next) { }; middleware.routeTouchIcon = function routeTouchIcon(req, res) { - if (meta.config['brand:touchIcon'] && validator.isURL(meta.config['brand:touchIcon'])) { - return res.redirect(meta.config['brand:touchIcon']); + const brandTouchIcon = meta.config['brand:touchIcon']; + if (brandTouchIcon && validator.isURL(brandTouchIcon)) { + return res.redirect(brandTouchIcon); } + let iconPath = ''; - if (meta.config['brand:touchIcon']) { - iconPath = path.join(nconf.get('upload_path'), meta.config['brand:touchIcon'].replace(/assets\/uploads/, '')); + if (brandTouchIcon) { + const uploadPath = nconf.get('upload_path'); + iconPath = path.join(uploadPath, brandTouchIcon.replace(/assets\/uploads/, '')); + if (!iconPath.startsWith(uploadPath)) { + return res.status(404).send('Not found'); + } } else { iconPath = path.join(nconf.get('base_dir'), 'public/images/touch/512.png'); } diff --git a/test/controllers.js b/test/controllers.js index b2174d8cf9..5ef313a6d2 100644 --- a/test/controllers.js +++ b/test/controllers.js @@ -6,8 +6,8 @@ const fs = require('fs'); const path = require('path'); const util = require('util'); -const request = require('../src/request'); const db = require('./mocks/databasemock'); +const request = require('../src/request'); const api = require('../src/api'); const categories = require('../src/categories'); const topics = require('../src/topics'); @@ -692,6 +692,16 @@ describe('Controllers', () => { assert(body); }); + it('should 404 if brand:touchIcon is not valid', async () => { + const oldValue = meta.config['brand:touchIcon']; + meta.config['brand:touchIcon'] = '../../not/valid'; + + const { response, body } = await request.get(`${nconf.get('url')}/apple-touch-icon`); + assert.strictEqual(response.statusCode, 404); + assert.strictEqual(body, 'Not found'); + meta.config['brand:touchIcon'] = oldValue; + }) + it('should error if guests do not have search privilege', async () => { const { response, body } = await request.get(`${nconf.get('url')}/api/users?query=bar§ion=sort-posts`); From a37521b0167bf3c10c1b856ed317589f4a08bede Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bar=C4=B1=C5=9F=20Soner=20U=C5=9Fakl=C4=B1?= Date: Fri, 12 Sep 2025 19:27:07 -0400 Subject: [PATCH 11/34] lint: fix --- test/controllers.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/controllers.js b/test/controllers.js index 5ef313a6d2..d79cf6de04 100644 --- a/test/controllers.js +++ b/test/controllers.js @@ -700,7 +700,7 @@ describe('Controllers', () => { assert.strictEqual(response.statusCode, 404); assert.strictEqual(body, 'Not found'); meta.config['brand:touchIcon'] = oldValue; - }) + }); it('should error if guests do not have search privilege', async () => { From e2dc592c4f21a48bd5f6c10b58773d0b755e32bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bar=C4=B1=C5=9F=20Soner=20U=C5=9Fakl=C4=B1?= Date: Fri, 12 Sep 2025 19:50:19 -0400 Subject: [PATCH 12/34] fix: favicon path --- src/webserver.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/webserver.js b/src/webserver.js index ab8dc5bc0d..18f57faa40 100644 --- a/src/webserver.js +++ b/src/webserver.js @@ -227,6 +227,9 @@ function setupHelmet(app) { function setupFavicon(app) { let faviconPath = meta.config['brand:favicon'] || 'favicon.ico'; faviconPath = path.join(nconf.get('base_dir'), 'public', faviconPath.replace(/assets\/uploads/, 'uploads')); + if (!faviconPath.startsWith(nconf.get('upload_path'))) { + faviconPath = path.join(nconf.get('base_dir'), 'public', 'favicon.ico'); + } if (file.existsSync(faviconPath)) { app.use(nconf.get('relative_path'), favicon(faviconPath)); } From 8a786c717e5366620d12c7b49d34fb0dffbe67a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bar=C4=B1=C5=9F=20Soner=20U=C5=9Fakl=C4=B1?= Date: Sat, 13 Sep 2025 17:40:09 -0400 Subject: [PATCH 13/34] fix: if reputation is disabled hide votes on /recent they were only hidden on category page --- install/package.json | 4 ++-- public/src/modules/topicList.js | 8 +++++--- src/controllers/recent.js | 1 + 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/install/package.json b/install/package.json index 9e94fa1ed3..55dee022a8 100644 --- a/install/package.json +++ b/install/package.json @@ -106,10 +106,10 @@ "nodebb-plugin-spam-be-gone": "2.3.2", "nodebb-plugin-web-push": "0.7.5", "nodebb-rewards-essentials": "1.0.2", - "nodebb-theme-harmony": "2.1.18", + "nodebb-theme-harmony": "2.1.19", "nodebb-theme-lavender": "7.1.19", "nodebb-theme-peace": "2.2.48", - "nodebb-theme-persona": "14.1.12", + "nodebb-theme-persona": "14.1.13", "nodebb-widget-essentials": "7.0.40", "nodemailer": "7.0.6", "nprogress": "0.2.0", diff --git a/public/src/modules/topicList.js b/public/src/modules/topicList.js index 6396bb9e8e..bc5a6e4abd 100644 --- a/public/src/modules/topicList.js +++ b/public/src/modules/topicList.js @@ -53,7 +53,7 @@ define('topicList', [ handleBack.init(function (after, handleBackCallback) { loadTopicsCallback(after, 1, function (data, loadCallback) { - onTopicsLoaded(templateName, data.topics, ajaxify.data.showSelect, 1, function () { + onTopicsLoaded(templateName, data, ajaxify.data.showSelect, 1, function () { handleBackCallback(); loadCallback(); }); @@ -166,7 +166,7 @@ define('topicList', [ } loadTopicsCallback(after, direction, function (data, done) { - onTopicsLoaded(templateName, data.topics, ajaxify.data.showSelect, direction, done); + onTopicsLoaded(templateName, data, ajaxify.data.showSelect, direction, done); }); }; @@ -187,7 +187,8 @@ define('topicList', [ }); } - function onTopicsLoaded(templateName, topics, showSelect, direction, callback) { + function onTopicsLoaded(templateName, data, showSelect, direction, callback) { + let { topics } = data; if (!topics || !topics.length) { $('#load-more-btn').hide(); return callback(); @@ -212,6 +213,7 @@ define('topicList', [ const tplData = { topics: topics, showSelect: showSelect, + 'reputation:disabled': data['reputation:disabled'], template: { name: templateName, }, diff --git a/src/controllers/recent.js b/src/controllers/recent.js index 5699fee1b7..656958463b 100644 --- a/src/controllers/recent.js +++ b/src/controllers/recent.js @@ -79,6 +79,7 @@ recentController.getData = async function (req, url, sort) { data.selectedTag = tagData.selectedTag; data.selectedTags = tagData.selectedTags; data['feeds:disableRSS'] = meta.config['feeds:disableRSS'] || 0; + data['reputation:disabled'] = meta.config['reputation:disabled']; if (!meta.config['feeds:disableRSS']) { data.rssFeedUrl = `${relative_path}/${url}.rss`; if (req.loggedIn) { From dfe19a98c193bdfdd601f4f7e4be06e67f8431b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bar=C4=B1=C5=9F=20Soner=20U=C5=9Fakl=C4=B1?= Date: Sat, 13 Sep 2025 17:51:25 -0400 Subject: [PATCH 14/34] fix: don't show votes on unread if rep system disabled add openapi spec --- public/openapi/read/popular.yaml | 2 ++ public/openapi/read/recent.yaml | 2 ++ public/openapi/read/top.yaml | 2 ++ public/openapi/read/unread.yaml | 2 ++ src/controllers/unread.js | 1 + 5 files changed, 9 insertions(+) diff --git a/public/openapi/read/popular.yaml b/public/openapi/read/popular.yaml index 67c7d5030f..fe6a0a6480 100644 --- a/public/openapi/read/popular.yaml +++ b/public/openapi/read/popular.yaml @@ -60,6 +60,8 @@ get: type: string feeds:disableRSS: type: number + reputation:disabled: + type: number rssFeedUrl: type: string title: diff --git a/public/openapi/read/recent.yaml b/public/openapi/read/recent.yaml index 74d3d91a27..848a306b79 100644 --- a/public/openapi/read/recent.yaml +++ b/public/openapi/read/recent.yaml @@ -58,6 +58,8 @@ get: type: string feeds:disableRSS: type: number + reputation:disabled: + type: number rssFeedUrl: type: string title: diff --git a/public/openapi/read/top.yaml b/public/openapi/read/top.yaml index 8594ca9f14..6f0cbc9bc1 100644 --- a/public/openapi/read/top.yaml +++ b/public/openapi/read/top.yaml @@ -71,6 +71,8 @@ get: type: string feeds:disableRSS: type: number + reputation:disabled: + type: number rssFeedUrl: type: string title: diff --git a/public/openapi/read/unread.yaml b/public/openapi/read/unread.yaml index ea69f666dd..77e9ec44f6 100644 --- a/public/openapi/read/unread.yaml +++ b/public/openapi/read/unread.yaml @@ -19,6 +19,8 @@ get: type: boolean showTopicTools: type: boolean + reputation:disabled: + type: number nextStart: type: number topics: diff --git a/src/controllers/unread.js b/src/controllers/unread.js index 9ff73da6ff..871f3252c0 100644 --- a/src/controllers/unread.js +++ b/src/controllers/unread.js @@ -75,6 +75,7 @@ unreadController.get = async function (req, res) { data.selectedTags = tagData.selectedTags; data.filters = helpers.buildFilters(baseUrl, filter, req.query); data.selectedFilter = data.filters.find(filter => filter && filter.selected); + data['reputation:disabled'] = meta.config['reputation:disabled']; res.render('unread', data); }; From 405d2172acc49321f6ec0109bda6b28f5340d39d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bar=C4=B1=C5=9F=20Soner=20U=C5=9Fakl=C4=B1?= Date: Mon, 15 Sep 2025 09:32:05 -0400 Subject: [PATCH 15/34] chore: up persona --- install/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install/package.json b/install/package.json index 55dee022a8..cd5d08f5c1 100644 --- a/install/package.json +++ b/install/package.json @@ -109,7 +109,7 @@ "nodebb-theme-harmony": "2.1.19", "nodebb-theme-lavender": "7.1.19", "nodebb-theme-peace": "2.2.48", - "nodebb-theme-persona": "14.1.13", + "nodebb-theme-persona": "14.1.14", "nodebb-widget-essentials": "7.0.40", "nodemailer": "7.0.6", "nprogress": "0.2.0", From 225bf85e941c95dd0d724ac5b47467918661f806 Mon Sep 17 00:00:00 2001 From: Julian Lam Date: Mon, 15 Sep 2025 12:47:49 -0400 Subject: [PATCH 16/34] fix: #13657, fix remote category data inconsistency in `sendNotificationToPostOwner` --- src/notifications.js | 20 +++++++++++++++++++- src/socket.io/helpers.js | 11 ++++++----- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/src/notifications.js b/src/notifications.js index bcf27c7d66..e71366417e 100644 --- a/src/notifications.js +++ b/src/notifications.js @@ -9,6 +9,7 @@ const _ = require('lodash'); const db = require('./database'); const User = require('./user'); +const categories = require('./categories'); const posts = require('./posts'); const groups = require('./groups'); const meta = require('./meta'); @@ -84,7 +85,24 @@ Notifications.getMultiple = async function (nids) { const notifications = await db.getObjects(keys); const userKeys = notifications.map(n => n && n.from); - const usersData = await User.getUsersFields(userKeys, ['username', 'userslug', 'picture']); + let [usersData, categoriesData] = await Promise.all([ + User.getUsersFields(userKeys, ['username', 'userslug', 'picture']), + categories.getCategoriesFields(userKeys, ['cid', 'name', 'slug', 'picture']), + ]); + // Merge valid categoriesData into usersData + usersData = usersData.map((userData, idx) => { + const categoryData = categoriesData[idx]; + if (!userData.uid && categoryData.cid) { + return { + username: categoryData.slug, + displayname: categoryData.name, + userslug: categoryData.slug, + picture: categoryData.picture, + }; + } + + return userData; + }); notifications.forEach((notification, index) => { if (notification) { diff --git a/src/socket.io/helpers.js b/src/socket.io/helpers.js index d80638458e..5def5138d4 100644 --- a/src/socket.io/helpers.js +++ b/src/socket.io/helpers.js @@ -94,7 +94,10 @@ SocketHelpers.sendNotificationToPostOwner = async function (pid, fromuid, comman return; } fromuid = utils.isNumber(fromuid) ? parseInt(fromuid, 10) : fromuid; - const postData = await posts.getPostFields(pid, ['tid', 'uid', 'content']); + const [postData, fromCategory] = await Promise.all([ + posts.getPostFields(pid, ['tid', 'uid', 'content']), + !utils.isNumber(fromuid) && categories.exists(fromuid), + ]); const [canRead, isIgnoring] = await Promise.all([ privileges.posts.can('topics:read', pid, postData.uid), topics.isIgnoring([postData.tid], postData.uid), @@ -103,19 +106,17 @@ SocketHelpers.sendNotificationToPostOwner = async function (pid, fromuid, comman return; } const [userData, topicTitle, postObj] = await Promise.all([ - user.getUserFields(fromuid, ['username']), + fromCategory ? categories.getCategoryFields(fromuid, ['name']) : user.getUserFields(fromuid, ['username']), topics.getTopicField(postData.tid, 'title'), posts.parsePost(postData), ]); - const { displayname } = userData; - const title = utils.decodeHTMLEntities(topicTitle); const titleEscaped = title.replace(/%/g, '%').replace(/,/g, ','); const notifObj = await notifications.create({ type: command, - bodyShort: `[[${notification}, ${displayname}, ${titleEscaped}]]`, + bodyShort: `[[${notification}, ${userData.displayname || userData.name}, ${titleEscaped}]]`, bodyLong: postObj.content, pid: pid, tid: postData.tid, From 52fec49310db9957677c3ab3a3a98ae8ad08f903 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bar=C4=B1=C5=9F=20Soner=20U=C5=9Fakl=C4=B1?= Date: Mon, 15 Sep 2025 12:57:29 -0400 Subject: [PATCH 17/34] chore: remove obsolete deprecation --- src/meta/minifier.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/meta/minifier.js b/src/meta/minifier.js index 0ab268dc4f..324b827797 100644 --- a/src/meta/minifier.js +++ b/src/meta/minifier.js @@ -166,7 +166,7 @@ actions.buildCSS = async function buildCSS(data) { }; if (data.minify) { opts.silenceDeprecations = [ - 'legacy-js-api', 'mixed-decls', 'color-functions', + 'legacy-js-api', 'color-functions', 'global-builtin', 'import', ]; } From f67942caecbefab7f062379f653615f33b074b2f Mon Sep 17 00:00:00 2001 From: Julian Lam Date: Mon, 15 Sep 2025 13:53:27 -0400 Subject: [PATCH 18/34] fix: local pids not always converted to absolute URLs on topic actor controller --- src/controllers/activitypub/actors.js | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/controllers/activitypub/actors.js b/src/controllers/activitypub/actors.js index 0f7a348990..850585e379 100644 --- a/src/controllers/activitypub/actors.js +++ b/src/controllers/activitypub/actors.js @@ -136,6 +136,7 @@ Actors.topic = async function (req, res, next) { let collection; let pids; try { + // pids are used in generation of digest only. ([collection, pids] = await Promise.all([ activitypub.helpers.generateCollection({ set: `tid:${req.params.tid}:posts`, @@ -151,7 +152,6 @@ Actors.topic = async function (req, res, next) { } pids.push(mainPid); pids = pids.map(pid => (utils.isNumber(pid) ? `${nconf.get('url')}/post/${pid}` : pid)); - collection.totalItems += 1; // account for mainPid // Generate digest for ETag const digest = activitypub.helpers.generateDigest(new Set(pids)); @@ -168,15 +168,18 @@ Actors.topic = async function (req, res, next) { } res.set('ETag', digest); - // Convert pids to urls + // Add OP to collection on first (or only) page if (page || collection.totalItems < perPage) { collection.orderedItems = collection.orderedItems || []; - if (!page || page === 1) { // add OP to collection + if (!page || page === 1) { collection.orderedItems.unshift(mainPid); + collection.totalItems += 1; } - collection.orderedItems = collection.orderedItems.map(pid => (utils.isNumber(pid) ? `${nconf.get('url')}/post/${pid}` : pid)); } + // Convert pids to urls + collection.orderedItems = collection.orderedItems.map(pid => (utils.isNumber(pid) ? `${nconf.get('url')}/post/${pid}` : pid)); + const object = { '@context': 'https://www.w3.org/ns/activitystreams', id: `${nconf.get('url')}/topic/${req.params.tid}${collection.orderedItems && page ? `?page=${page}` : ''}`, From b66c30a2a73d06c45a6b6d97607b0d9378d56b87 Mon Sep 17 00:00:00 2001 From: Julian Lam Date: Mon, 15 Sep 2025 14:10:02 -0400 Subject: [PATCH 19/34] fix: handle cases where incoming ap object tag can be a non-array --- src/activitypub/notes.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/activitypub/notes.js b/src/activitypub/notes.js index 733aff354f..ef4abe9add 100644 --- a/src/activitypub/notes.js +++ b/src/activitypub/notes.js @@ -31,7 +31,13 @@ async function unlock(value) { Notes._normalizeTags = async (tag, cid) => { const systemTags = (meta.config.systemTags || '').split(','); const maxTags = await categories.getCategoryField(cid, 'maxTags'); - const tags = (tag || []) + let tags = tag || []; + + if (!Array.isArray(tags)) { // the "|| []" should handle null/undefined values... #famouslastwords + tags = [tags]; + } + + tags = tags .map((tag) => { tag.name = tag.name.startsWith('#') ? tag.name.slice(1) : tag.name; return tag; From 68a8db856a9dcad42fb58e505aaacaa173a1401a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bar=C4=B1=C5=9F=20Soner=20U=C5=9Fakl=C4=B1?= Date: Tue, 16 Sep 2025 11:23:31 -0400 Subject: [PATCH 20/34] feat: add a new hook to override generateUrl in navigator.js --- public/src/modules/navigator.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/public/src/modules/navigator.js b/public/src/modules/navigator.js index 8c557985c3..0776f04c4b 100644 --- a/public/src/modules/navigator.js +++ b/public/src/modules/navigator.js @@ -441,7 +441,13 @@ define('navigator', [ function generateUrl(index) { const pathname = window.location.pathname.replace(config.relative_path, ''); const parts = pathname.split('/'); - return parts[1] + '/' + parts[2] + '/' + parts[3] + (index ? '/' + index : ''); + const newUrl = parts[1] + '/' + parts[2] + '/' + parts[3] + (index ? '/' + index : ''); + const data = { + newUrl, + index, + }; + hooks.fire('filter:navigator.generateUrl', data); + return data.newUrl; } navigator.getCount = () => count; From 9c18c6fe49be49975fb5ea0eef73045890544415 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bar=C4=B1=C5=9F=20Soner=20U=C5=9Fakl=C4=B1?= Date: Tue, 16 Sep 2025 11:24:14 -0400 Subject: [PATCH 21/34] feat: add a term param to recent controller so it can be controller without req.query.term --- src/controllers/recent.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/controllers/recent.js b/src/controllers/recent.js index 656958463b..73d5348c0d 100644 --- a/src/controllers/recent.js +++ b/src/controllers/recent.js @@ -22,9 +22,9 @@ recentController.get = async function (req, res, next) { res.render('recent', data); }; -recentController.getData = async function (req, url, sort) { +recentController.getData = async function (req, url, sort, selectedTerm = 'alltime') { const page = parseInt(req.query.page, 10) || 1; - let term = helpers.terms[req.query.term]; + let term = helpers.terms[req.query.term || selectedTerm]; const { cid, tag } = req.query; const filter = req.query.filter || ''; From f7bbec7ccfee65cd21118d430f710a87ffcabcce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bar=C4=B1=C5=9F=20Soner=20U=C5=9Fakl=C4=B1?= Date: Tue, 16 Sep 2025 11:48:39 -0400 Subject: [PATCH 22/34] fix: switch to action --- public/src/modules/navigator.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/src/modules/navigator.js b/public/src/modules/navigator.js index 0776f04c4b..0532beb3b4 100644 --- a/public/src/modules/navigator.js +++ b/public/src/modules/navigator.js @@ -446,7 +446,7 @@ define('navigator', [ newUrl, index, }; - hooks.fire('filter:navigator.generateUrl', data); + hooks.fire('action:navigator.generateUrl', data); return data.newUrl; } From f7c47429879f757e08975b5cd003416db00f5568 Mon Sep 17 00:00:00 2001 From: Julian Lam Date: Wed, 17 Sep 2025 10:44:51 -0400 Subject: [PATCH 23/34] fix: add pre-processing step to title generation logic so sbd doesn't fall over so badly --- src/activitypub/helpers.js | 46 -------------------------------------- src/activitypub/notes.js | 6 ++++- 2 files changed, 5 insertions(+), 47 deletions(-) diff --git a/src/activitypub/helpers.js b/src/activitypub/helpers.js index e6eb2e1c08..01cfd86d10 100644 --- a/src/activitypub/helpers.js +++ b/src/activitypub/helpers.js @@ -339,52 +339,6 @@ Helpers.resolveObjects = async (ids) => { return objects.length === 1 ? objects[0] : objects; }; -const titleishTags = ['h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'title', 'p', 'span']; -const titleRegex = new RegExp(`<(${titleishTags.join('|')})>(.+?)`, 'm'); -Helpers.generateTitle = (html) => { - // Given an html string, generates a more appropriate title if possible - let title; - - // Try the first paragraph-like element - const match = html.match(titleRegex); - if (match && match.index === 0) { - title = match[2]; - } - - // Fall back to newline splitting (i.e. if no paragraph elements) - title = title || html.split('\n').filter(Boolean).shift(); - - // Discard everything after a line break element - title = title.replace(/.*/g, ''); - - // Strip html - title = utils.stripHTMLTags(title); - - // Split sentences and use only first one - const sentences = title - .split(/(\.|\?|!)\s/) - .reduce((memo, cur, idx, sentences) => { - if (idx % 2) { - memo.push(`${sentences[idx - 1]}${cur}`); - } else if (idx === sentences.length - 1) { - memo.push(cur); - } - - return memo; - }, []); - - if (sentences.length > 1) { - title = sentences.shift(); - } - - // Truncate down if too long - if (title.length > meta.config.maximumTitleLength) { - title = `${title.slice(0, meta.config.maximumTitleLength - 3)}...`; - } - - return title; -}; - Helpers.remoteAnchorToLocalProfile = async (content, isMarkdown = false) => { let anchorRegex; if (isMarkdown) { diff --git a/src/activitypub/notes.js b/src/activitypub/notes.js index ef4abe9add..6ccb2ca209 100644 --- a/src/activitypub/notes.js +++ b/src/activitypub/notes.js @@ -165,7 +165,11 @@ Notes.assert = async (uid, input, options = { skipChecks: false }) => { // mainPid ok to leave as-is if (!title) { - const sentences = tokenizer.sentences(content || sourceContent, { sanitize: true }); + // Naive pre-processing prior to sbd tokenization + let sbdInput = content || sourceContent; + sbdInput = sbdInput.replace('

', '

\n

'); + + const sentences = tokenizer.sentences(sbdInput, { sanitize: true, newline_boundaries: true }); title = sentences.shift(); } From 6cca55e37f0bce389c3094c5aae07ed1bbed3297 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bar=C4=B1=C5=9F=20Soner=20U=C5=9Fakl=C4=B1?= Date: Wed, 17 Sep 2025 10:50:35 -0400 Subject: [PATCH 24/34] fix: use parameterized query for key lookup --- src/database/postgres/main.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/database/postgres/main.js b/src/database/postgres/main.js index c0838b45a0..5b3c7f7e9d 100644 --- a/src/database/postgres/main.js +++ b/src/database/postgres/main.js @@ -85,7 +85,8 @@ module.exports = function (module) { text: ` SELECT o."_key" FROM "legacy_object_live" o - WHERE o."_key" LIKE '${match}'`, + WHERE o."_key" LIKE $1`, + values: [match], }); return res.rows.map(r => r._key); From 532653110c9e0400967c42352415be6dccb8e6a4 Mon Sep 17 00:00:00 2001 From: Julian Lam Date: Wed, 17 Sep 2025 10:58:07 -0400 Subject: [PATCH 25/34] Revert "fix: add pre-processing step to title generation logic so sbd doesn't fall over so badly" This reverts commit f7c47429879f757e08975b5cd003416db00f5568. --- src/activitypub/helpers.js | 46 ++++++++++++++++++++++++++++++++++++++ src/activitypub/notes.js | 6 +---- 2 files changed, 47 insertions(+), 5 deletions(-) diff --git a/src/activitypub/helpers.js b/src/activitypub/helpers.js index 01cfd86d10..e6eb2e1c08 100644 --- a/src/activitypub/helpers.js +++ b/src/activitypub/helpers.js @@ -339,6 +339,52 @@ Helpers.resolveObjects = async (ids) => { return objects.length === 1 ? objects[0] : objects; }; +const titleishTags = ['h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'title', 'p', 'span']; +const titleRegex = new RegExp(`<(${titleishTags.join('|')})>(.+?)`, 'm'); +Helpers.generateTitle = (html) => { + // Given an html string, generates a more appropriate title if possible + let title; + + // Try the first paragraph-like element + const match = html.match(titleRegex); + if (match && match.index === 0) { + title = match[2]; + } + + // Fall back to newline splitting (i.e. if no paragraph elements) + title = title || html.split('\n').filter(Boolean).shift(); + + // Discard everything after a line break element + title = title.replace(/.*/g, ''); + + // Strip html + title = utils.stripHTMLTags(title); + + // Split sentences and use only first one + const sentences = title + .split(/(\.|\?|!)\s/) + .reduce((memo, cur, idx, sentences) => { + if (idx % 2) { + memo.push(`${sentences[idx - 1]}${cur}`); + } else if (idx === sentences.length - 1) { + memo.push(cur); + } + + return memo; + }, []); + + if (sentences.length > 1) { + title = sentences.shift(); + } + + // Truncate down if too long + if (title.length > meta.config.maximumTitleLength) { + title = `${title.slice(0, meta.config.maximumTitleLength - 3)}...`; + } + + return title; +}; + Helpers.remoteAnchorToLocalProfile = async (content, isMarkdown = false) => { let anchorRegex; if (isMarkdown) { diff --git a/src/activitypub/notes.js b/src/activitypub/notes.js index 6ccb2ca209..ef4abe9add 100644 --- a/src/activitypub/notes.js +++ b/src/activitypub/notes.js @@ -165,11 +165,7 @@ Notes.assert = async (uid, input, options = { skipChecks: false }) => { // mainPid ok to leave as-is if (!title) { - // Naive pre-processing prior to sbd tokenization - let sbdInput = content || sourceContent; - sbdInput = sbdInput.replace('

', '

\n

'); - - const sentences = tokenizer.sentences(sbdInput, { sanitize: true, newline_boundaries: true }); + const sentences = tokenizer.sentences(content || sourceContent, { sanitize: true }); title = sentences.shift(); } From a6674f67a1cfb92f6236e76447e5e9213b1b5710 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bar=C4=B1=C5=9F=20Soner=20U=C5=9Fakl=C4=B1?= Date: Wed, 17 Sep 2025 10:58:26 -0400 Subject: [PATCH 26/34] lint: remove unused --- src/activitypub/helpers.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/activitypub/helpers.js b/src/activitypub/helpers.js index 01cfd86d10..f24d18b730 100644 --- a/src/activitypub/helpers.js +++ b/src/activitypub/helpers.js @@ -8,7 +8,6 @@ const validator = require('validator'); // const cheerio = require('cheerio'); const crypto = require('crypto'); -const meta = require('../meta'); const posts = require('../posts'); const categories = require('../categories'); const messaging = require('../messaging'); @@ -16,7 +15,6 @@ const request = require('../request'); const db = require('../database'); const ttl = require('../cache/ttl'); const user = require('../user'); -const utils = require('../utils'); const activitypub = require('.'); const webfingerRegex = /^(@|acct:)?[\w-.]+@.+$/; From 5beeedd67cc2fc08b6dda77f237ba7892b5329b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bar=C4=B1=C5=9F=20Soner=20U=C5=9Fakl=C4=B1?= Date: Wed, 17 Sep 2025 11:09:02 -0400 Subject: [PATCH 27/34] Revert "lint: remove unused" This reverts commit a6674f67a1cfb92f6236e76447e5e9213b1b5710. --- src/activitypub/helpers.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/activitypub/helpers.js b/src/activitypub/helpers.js index e0f96fe3aa..e6eb2e1c08 100644 --- a/src/activitypub/helpers.js +++ b/src/activitypub/helpers.js @@ -8,6 +8,7 @@ const validator = require('validator'); // const cheerio = require('cheerio'); const crypto = require('crypto'); +const meta = require('../meta'); const posts = require('../posts'); const categories = require('../categories'); const messaging = require('../messaging'); @@ -15,6 +16,7 @@ const request = require('../request'); const db = require('../database'); const ttl = require('../cache/ttl'); const user = require('../user'); +const utils = require('../utils'); const activitypub = require('.'); const webfingerRegex = /^(@|acct:)?[\w-.]+@.+$/; From d1f5060f11a257388690d1441726efd58ca88b5a Mon Sep 17 00:00:00 2001 From: Julian Lam Date: Thu, 18 Sep 2025 13:33:16 -0400 Subject: [PATCH 28/34] fix(deps): bump 2factor to 7.6.0 --- install/package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/install/package.json b/install/package.json index cd5d08f5c1..f9e5b8b0ed 100644 --- a/install/package.json +++ b/install/package.json @@ -96,7 +96,7 @@ "mousetrap": "1.6.5", "multer": "2.0.2", "nconf": "0.13.0", - "nodebb-plugin-2factor": "7.5.10", + "nodebb-plugin-2factor": "7.6.0", "nodebb-plugin-composer-default": "10.3.1", "nodebb-plugin-dbsearch": "6.3.2", "nodebb-plugin-emoji": "6.0.3", @@ -201,4 +201,4 @@ "url": "https://github.com/barisusakli" } ] -} \ No newline at end of file +} From f9edb13f6209b075d4a53c130d1bba166ae188fa Mon Sep 17 00:00:00 2001 From: Julian Lam Date: Fri, 19 Sep 2025 14:43:04 -0400 Subject: [PATCH 29/34] fix: missing actor assertion on 1b12 announced upboat --- src/activitypub/inbox.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/activitypub/inbox.js b/src/activitypub/inbox.js index 754720f208..ea5b032a1a 100644 --- a/src/activitypub/inbox.js +++ b/src/activitypub/inbox.js @@ -298,6 +298,7 @@ inbox.announce = async (req) => { const exists = await posts.exists(localId || id); if (exists) { try { + await activitypub.actors.assert(object.actor); const result = await posts.upvote(localId || id, object.actor); if (localId) { socketHelpers.upvote(result, 'notifications:upvoted-your-post-in'); From 4d68e3fe145e49f38d1f3dc2b45d8409bb3945f6 Mon Sep 17 00:00:00 2001 From: Julian Lam Date: Mon, 22 Sep 2025 11:56:55 -0400 Subject: [PATCH 30/34] fix: re-jig handling of ap tag values so that only hashtags are considered (not Piefed community tags, etc.) --- src/activitypub/notes.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/activitypub/notes.js b/src/activitypub/notes.js index ef4abe9add..832b8d152d 100644 --- a/src/activitypub/notes.js +++ b/src/activitypub/notes.js @@ -38,11 +38,12 @@ Notes._normalizeTags = async (tag, cid) => { } tags = tags + .filter(({ type }) => type === 'Hashtag') .map((tag) => { tag.name = tag.name.startsWith('#') ? tag.name.slice(1) : tag.name; return tag; }) - .filter(o => o.type === 'Hashtag' && !systemTags.includes(o.name)) + .filter(({ name }) => !systemTags.includes(name)) .map(t => t.name); if (tags.length > maxTags) { From d0c058263f5ffbdd7be821d15b3fd3bcbdb5fa12 Mon Sep 17 00:00:00 2001 From: Julian Lam Date: Mon, 22 Sep 2025 12:14:14 -0400 Subject: [PATCH 31/34] fix: update note assertion topic members check to simpler posts.exists check The original logic checked that each member of the resolved chain was part of the resolved topic. That isn't always the case, especially when topics splinter due to network timeouts/unavailability. This ended up causing issues where already asserted posts were re-asserted but failed because they no longer served an _activitypub object since it was already asserted and the data was just pulled from the db. --- src/activitypub/notes.js | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/activitypub/notes.js b/src/activitypub/notes.js index 832b8d152d..e1fc16c175 100644 --- a/src/activitypub/notes.js +++ b/src/activitypub/notes.js @@ -117,9 +117,8 @@ Notes.assert = async (uid, input, options = { skipChecks: false }) => { await topics.tools.move(tid, { cid: options.cid, uid: 'system' }); } - const members = await db.isSortedSetMembers(`tid:${tid}:posts`, chain.slice(1).map(p => p.pid)); - members.unshift(await posts.exists(mainPid)); - if (tid && members.every(Boolean)) { + const exists = await posts.exists(chain.map(p => p.pid)); + if (tid && exists.every(Boolean)) { // All cached, return early. activitypub.helpers.log('[notes/assert] No new notes to process.'); await unlock(id); @@ -212,7 +211,7 @@ Notes.assert = async (uid, input, options = { skipChecks: false }) => { } return post; - }).filter((p, idx) => !members[idx]); + }).filter((p, idx) => !exists[idx]); const count = unprocessed.length; activitypub.helpers.log(`[notes/assert] ${count} new note(s) found.`); From 32d0ee480844b350cd5502922bfe1109ad913d19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bar=C4=B1=C5=9F=20Soner=20U=C5=9Fakl=C4=B1?= Date: Thu, 25 Sep 2025 02:03:14 -0400 Subject: [PATCH 32/34] perf: update old upgrade scripts to use bulkSet/Add fix a missing await --- src/upgrades/1.10.2/upgrade_bans_to_hashes.js | 34 ++++++---- src/upgrades/1.10.2/username_email_history.js | 45 +++++++------ .../1.12.1/clear_username_email_history.js | 53 ++++++---------- .../1.12.1/moderation_notes_refactor.js | 10 +-- src/upgrades/1.13.0/clean_post_topic_hash.js | 3 +- .../1.6.2/topics_lastposttime_zset.js | 35 ++++++----- src/upgrades/1.7.1/notification-settings.js | 35 ++++++++--- src/upgrades/1.7.3/topic_votes.js | 56 ++++++++++------- src/upgrades/1.8.1/diffs_zset_to_listhash.js | 63 +++++++------------ .../1.9.0/refresh_post_upload_associations.js | 19 +++--- src/upgrades/2.8.7/fix-email-sorted-sets.js | 2 +- src/upgrades/3.7.0/category-read-by-uid.js | 2 +- 12 files changed, 186 insertions(+), 171 deletions(-) diff --git a/src/upgrades/1.10.2/upgrade_bans_to_hashes.js b/src/upgrades/1.10.2/upgrade_bans_to_hashes.js index 84c7a0ed4d..2bc55b4667 100644 --- a/src/upgrades/1.10.2/upgrade_bans_to_hashes.js +++ b/src/upgrades/1.10.2/upgrade_bans_to_hashes.js @@ -11,14 +11,23 @@ module.exports = { method: async function () { const { progress } = this; + progress.total = await db.sortedSetCard('users:joindate'); + await batch.processSortedSet('users:joindate', async (uids) => { - for (const uid of uids) { - progress.incr(); - const [bans, reasons, userData] = await Promise.all([ - db.getSortedSetRevRangeWithScores(`uid:${uid}:bans`, 0, -1), - db.getSortedSetRevRangeWithScores(`banned:${uid}:reasons`, 0, -1), - db.getObjectFields(`user:${uid}`, ['banned', 'banned:expire', 'joindate', 'lastposttime', 'lastonline']), - ]); + progress.incr(uids.length); + const [allUserData, allBans] = await Promise.all([ + db.getObjectsFields( + uids.map(uid => `user:${uid}`), + ['banned', 'banned:expire', 'joindate', 'lastposttime', 'lastonline'], + ), + db.getSortedSetsMembersWithScores( + uids.map(uid => `uid:${uid}:bans`) + ), + ]); + + await Promise.all(uids.map(async (uid, index) => { + const userData = allUserData[index]; + const bans = allBans[index] || []; // has no history, but is banned, create plain object with just uid and timestmap if (!bans.length && parseInt(userData.banned, 10)) { @@ -31,6 +40,7 @@ module.exports = { const banKey = `uid:${uid}:ban:${banTimestamp}`; await addBan(uid, banKey, { uid: uid, timestamp: banTimestamp }); } else if (bans.length) { + const reasons = await db.getSortedSetRevRangeWithScores(`banned:${uid}:reasons`, 0, -1); // process ban history for (const ban of bans) { const reasonData = reasons.find(reasonData => reasonData.score === ban.score); @@ -46,14 +56,16 @@ module.exports = { await addBan(uid, banKey, data); } } - } + })); }, { - progress: this.progress, + batch: 500, }); }, }; async function addBan(uid, key, data) { - await db.setObject(key, data); - await db.sortedSetAdd(`uid:${uid}:bans:timestamp`, data.timestamp, key); + await Promise.all([ + db.setObject(key, data), + db.sortedSetAdd(`uid:${uid}:bans:timestamp`, data.timestamp, key), + ]); } diff --git a/src/upgrades/1.10.2/username_email_history.js b/src/upgrades/1.10.2/username_email_history.js index 3b03568a69..8ee4306e3d 100644 --- a/src/upgrades/1.10.2/username_email_history.js +++ b/src/upgrades/1.10.2/username_email_history.js @@ -11,27 +11,34 @@ module.exports = { method: async function () { const { progress } = this; - await batch.processSortedSet('users:joindate', async (uids) => { - async function updateHistory(uid, set, fieldName) { - const count = await db.sortedSetCard(set); - if (count <= 0) { - // User has not changed their username/email before, record original username - const userData = await user.getUserFields(uid, [fieldName, 'joindate']); - if (userData && userData.joindate && userData[fieldName]) { - await db.sortedSetAdd(set, userData.joindate, [userData[fieldName], userData.joindate].join(':')); - } - } - } + progress.total = await db.sortedSetCard('users:joindate'); - await Promise.all(uids.map(async (uid) => { - await Promise.all([ - updateHistory(uid, `user:${uid}:usernames`, 'username'), - updateHistory(uid, `user:${uid}:emails`, 'email'), - ]); - progress.incr(); - })); + await batch.processSortedSet('users:joindate', async (uids) => { + const [usernameHistory, emailHistory, userData] = await Promise.all([ + db.sortedSetsCard(uids.map(uid => `user:${uid}:usernames`)), + db.sortedSetsCard(uids.map(uid => `user:${uid}:emails`)), + user.getUsersFields(uids, ['uid', 'username', 'email', 'joindate']), + ]); + + const bulkAdd = []; + userData.forEach((data, index) => { + const thisUsernameHistory = usernameHistory[index]; + const thisEmailHistory = emailHistory[index]; + if (thisUsernameHistory <= 0 && data && data.joindate && data.username) { + bulkAdd.push([ + `user:${data.uid}:usernames`, data.joindate, [data.username, data.joindate].join(':'), + ]); + } + if (thisEmailHistory <= 0 && data && data.joindate && data.email) { + bulkAdd.push([ + `user:${data.uid}:emails`, data.joindate, [data.email, data.joindate].join(':'), + ]); + } + }); + await db.sortedSetAddBulk(bulkAdd); + progress.incr(uids.length); }, { - progress: this.progress, + batch: 500, }); }, }; diff --git a/src/upgrades/1.12.1/clear_username_email_history.js b/src/upgrades/1.12.1/clear_username_email_history.js index 822b500884..0d36534502 100644 --- a/src/upgrades/1.12.1/clear_username_email_history.js +++ b/src/upgrades/1.12.1/clear_username_email_history.js @@ -1,45 +1,32 @@ 'use strict'; -const async = require('async'); + const db = require('../../database'); const user = require('../../user'); +const batch = require('../../batch'); module.exports = { name: 'Delete username email history for deleted users', timestamp: Date.UTC(2019, 2, 25), - method: function (callback) { + method: async function () { const { progress } = this; - let currentUid = 1; - db.getObjectField('global', 'nextUid', (err, nextUid) => { - if (err) { - return callback(err); - } - progress.total = nextUid; - async.whilst((next) => { - next(null, currentUid < nextUid); - }, - (next) => { - progress.incr(); - user.exists(currentUid, (err, exists) => { - if (err) { - return next(err); - } - if (exists) { - currentUid += 1; - return next(); - } - db.deleteAll([`user:${currentUid}:usernames`, `user:${currentUid}:emails`], (err) => { - if (err) { - return next(err); - } - currentUid += 1; - next(); - }); - }); - }, - (err) => { - callback(err); - }); + + progress.total = await db.getObjectField('global', 'nextUid'); + const allUids = []; + for (let i = 1; i < progress.total; i += 1) { + allUids.push(i); + } + await batch.processArray(allUids, async (uids) => { + const exists = await user.exists(uids); + const missingUids = uids.filter((uid, index) => !exists[index]); + const keysToDelete = [ + ...missingUids.map(uid => `user:${uid}:usernames`), + ...missingUids.map(uid => `user:${uid}:emails`), + ]; + await db.deleteAll(keysToDelete); + progress.incr(uids.length); + }, { + batch: 500, }); }, }; diff --git a/src/upgrades/1.12.1/moderation_notes_refactor.js b/src/upgrades/1.12.1/moderation_notes_refactor.js index 390273d74a..85118a9a0c 100644 --- a/src/upgrades/1.12.1/moderation_notes_refactor.js +++ b/src/upgrades/1.12.1/moderation_notes_refactor.js @@ -12,10 +12,12 @@ module.exports = { const { progress } = this; await batch.processSortedSet('users:joindate', async (uids) => { - await Promise.all(uids.map(async (uid) => { - progress.incr(); - - const notes = await db.getSortedSetRevRange(`uid:${uid}:moderation:notes`, 0, -1); + progress.incr(uids.length); + const allNotes = await db.getSortedSetsMembers( + uids.map(uid => `uid:${uid}:moderation:notes`) + ); + await Promise.all(uids.map(async (uid, index) => { + const notes = allNotes[index]; for (const note of notes) { const noteData = JSON.parse(note); noteData.timestamp = noteData.timestamp || Date.now(); diff --git a/src/upgrades/1.13.0/clean_post_topic_hash.js b/src/upgrades/1.13.0/clean_post_topic_hash.js index caa6dbd8f6..20cfd78c22 100644 --- a/src/upgrades/1.13.0/clean_post_topic_hash.js +++ b/src/upgrades/1.13.0/clean_post_topic_hash.js @@ -8,6 +8,7 @@ module.exports = { timestamp: Date.UTC(2019, 9, 7), method: async function () { const { progress } = this; + progress.total = await db.sortedSetCard('posts:pid') + await db.sortedSetCard('topics:tid'); await cleanPost(progress); await cleanTopic(progress); }, @@ -51,7 +52,6 @@ async function cleanPost(progress) { })); }, { batch: 500, - progress: progress, }); } @@ -90,6 +90,5 @@ async function cleanTopic(progress) { })); }, { batch: 500, - progress: progress, }); } diff --git a/src/upgrades/1.6.2/topics_lastposttime_zset.js b/src/upgrades/1.6.2/topics_lastposttime_zset.js index 1dee9feb1a..f299b19c01 100644 --- a/src/upgrades/1.6.2/topics_lastposttime_zset.js +++ b/src/upgrades/1.6.2/topics_lastposttime_zset.js @@ -1,29 +1,30 @@ 'use strict'; -const async = require('async'); - const db = require('../../database'); +const batch = require('../../batch'); module.exports = { name: 'New sorted set cid::tids:lastposttime', timestamp: Date.UTC(2017, 9, 30), - method: function (callback) { + method: async function () { const { progress } = this; + progress.total = await db.sortedSetCard('topics:tid'); - require('../../batch').processSortedSet('topics:tid', (tids, next) => { - async.eachSeries(tids, (tid, next) => { - db.getObjectFields(`topic:${tid}`, ['cid', 'timestamp', 'lastposttime'], (err, topicData) => { - if (err || !topicData) { - return next(err); - } - progress.incr(); - - const timestamp = topicData.lastposttime || topicData.timestamp || Date.now(); - db.sortedSetAdd(`cid:${topicData.cid}:tids:lastposttime`, timestamp, tid, next); - }, next); - }, next); + await batch.processSortedSet('topics:tid', async (tids) => { + const topicData = await db.getObjectsFields( + tids.map(tid => `topic:${tid}`), ['tid', 'cid', 'timestamp', 'lastposttime'] + ); + const bulkAdd = []; + topicData.forEach((data) => { + if (data && data.cid && data.tid) { + const timestamp = data.lastposttime || data.timestamp || Date.now(); + bulkAdd.push([`cid:${data.cid}:tids:lastposttime`, timestamp, data.tid]); + } + }); + await db.sortedSetAddBulk(bulkAdd); + progress.incr(tids.length); }, { - progress: this.progress, - }, callback); + batch: 500, + }); }, }; diff --git a/src/upgrades/1.7.1/notification-settings.js b/src/upgrades/1.7.1/notification-settings.js index fed592effb..e3693d4f04 100644 --- a/src/upgrades/1.7.1/notification-settings.js +++ b/src/upgrades/1.7.1/notification-settings.js @@ -8,23 +8,38 @@ module.exports = { timestamp: Date.UTC(2017, 10, 15), method: async function () { const { progress } = this; - + progress.total = await db.sortedSetCard('users:joindate'); await batch.processSortedSet('users:joindate', async (uids) => { - await Promise.all(uids.map(async (uid) => { - progress.incr(); - const userSettings = await db.getObjectFields(`user:${uid}:settings`, ['sendChatNotifications', 'sendPostNotifications']); - if (userSettings) { + + const userSettings = await db.getObjectsFields( + uids.map(uid => `user:${uid}:settings`), + ['sendChatNotifications', 'sendPostNotifications'], + ); + + const bulkSet = []; + userSettings.forEach((settings, index) => { + const set = {}; + if (settings) { if (parseInt(userSettings.sendChatNotifications, 10) === 1) { - await db.setObjectField(`user:${uid}:settings`, 'notificationType_new-chat', 'notificationemail'); + set['notificationType_new-chat'] = 'notificationemail'; } if (parseInt(userSettings.sendPostNotifications, 10) === 1) { - await db.setObjectField(`user:${uid}:settings`, 'notificationType_new-reply', 'notificationemail'); + set['notificationType_new-reply'] = 'notificationemail'; + } + if (Object.keys(set).length) { + bulkSet.push([`user:${uids[index]}:settings`, set]); } } - await db.deleteObjectFields(`user:${uid}:settings`, ['sendChatNotifications', 'sendPostNotifications']); - })); + }); + await db.setObjectBulk(bulkSet); + + await db.deleteObjectFields( + uids.map(uid => `user:${uid}:settings`), + ['sendChatNotifications', 'sendPostNotifications'], + ); + + progress.incr(uids.length); }, { - progress: progress, batch: 500, }); }, diff --git a/src/upgrades/1.7.3/topic_votes.js b/src/upgrades/1.7.3/topic_votes.js index 008aaece0a..d5f6b9fd57 100644 --- a/src/upgrades/1.7.3/topic_votes.js +++ b/src/upgrades/1.7.3/topic_votes.js @@ -10,32 +10,42 @@ module.exports = { method: async function () { const { progress } = this; - batch.processSortedSet('topics:tid', async (tids) => { - await Promise.all(tids.map(async (tid) => { - progress.incr(); - const topicData = await db.getObjectFields(`topic:${tid}`, ['mainPid', 'cid', 'pinned']); - if (topicData.mainPid && topicData.cid) { - const postData = await db.getObject(`post:${topicData.mainPid}`); - if (postData) { - const upvotes = parseInt(postData.upvotes, 10) || 0; - const downvotes = parseInt(postData.downvotes, 10) || 0; - const data = { - upvotes: upvotes, - downvotes: downvotes, - }; - const votes = upvotes - downvotes; - await Promise.all([ - db.setObject(`topic:${tid}`, data), - db.sortedSetAdd('topics:votes', votes, tid), - ]); - if (parseInt(topicData.pinned, 10) !== 1) { - await db.sortedSetAdd(`cid:${topicData.cid}:tids:votes`, votes, tid); - } + progress.total = await db.sortedSetCard('topics:tid'); + + await batch.processSortedSet('topics:tid', async (tids) => { + const topicsData = await db.getObjectsFields( + tids.map(tid => `topic:${tid}`), + ['tid', 'mainPid', 'cid', 'pinned'], + ); + const mainPids = topicsData.map(topicData => topicData && topicData.mainPid); + const mainPosts = await db.getObjects(mainPids.map(pid => `post:${pid}`)); + + const bulkSet = []; + const bulkAdd = []; + + topicsData.forEach((topicData, index) => { + const mainPost = mainPosts[index]; + if (mainPost && topicData && topicData.cid) { + const upvotes = parseInt(mainPost.upvotes, 10) || 0; + const downvotes = parseInt(mainPost.downvotes, 10) || 0; + const data = { + upvotes: upvotes, + downvotes: downvotes, + }; + const votes = upvotes - downvotes; + bulkSet.push([`topic:${topicData.tid}`, data]); + bulkAdd.push(['topics:votes', votes, topicData.tid]); + if (parseInt(topicData.pinned, 10) !== 1) { + bulkAdd.push([`cid:${topicData.cid}:tids:votes`, votes, topicData.tid]); } } - })); + }); + + await db.setObjectBulk(bulkSet); + await db.sortedSetAddBulk('topics:votes', bulkAdd); + + progress.incr(tids.length); }, { - progress: progress, batch: 500, }); }, diff --git a/src/upgrades/1.8.1/diffs_zset_to_listhash.js b/src/upgrades/1.8.1/diffs_zset_to_listhash.js index 370242fba1..277418a79e 100644 --- a/src/upgrades/1.8.1/diffs_zset_to_listhash.js +++ b/src/upgrades/1.8.1/diffs_zset_to_listhash.js @@ -1,57 +1,40 @@ 'use strict'; -const async = require('async'); const db = require('../../database'); const batch = require('../../batch'); - module.exports = { name: 'Reformatting post diffs to be stored in lists and hash instead of single zset', timestamp: Date.UTC(2018, 2, 15), - method: function (callback) { + method: async function () { const { progress } = this; - batch.processSortedSet('posts:pid', (pids, next) => { - async.each(pids, (pid, next) => { - db.getSortedSetRangeWithScores(`post:${pid}:diffs`, 0, -1, (err, diffs) => { - if (err) { - return next(err); - } + progress.total = await db.sortedSetCard('posts:pid'); - if (!diffs || !diffs.length) { - progress.incr(); - return next(); - } + await batch.processSortedSet('posts:pid', async (pids) => { + const postDiffs = await db.getSortedSetsMembersWithScores( + pids.map(pid => `post:${pid}:diffs`), + ); - // For each diff, push to list - async.each(diffs, (diff, next) => { - async.series([ - async.apply(db.delete.bind(db), `post:${pid}:diffs`), - async.apply(db.listPrepend.bind(db), `post:${pid}:diffs`, diff.score), - async.apply(db.setObject.bind(db), `diff:${pid}.${diff.score}`, { - pid: pid, - patch: diff.value, - }), - ], next); - }, (err) => { - if (err) { - return next(err); - } + await db.deleteAll(pids.map(pid => `post:${pid}:diffs`)); - progress.incr(); - return next(); - }); - }); - }, (err) => { - if (err) { - // Probably type error, ok to incr and continue - progress.incr(); + await Promise.all(postDiffs.map(async (diffs, index) => { + if (!diffs || !diffs.length) { + return; } - - return next(); - }); + diffs.reverse(); + const pid = pids[index]; + await db.listAppend(`post:${pid}:diffs`, diffs.map(d => d.score)); + await db.setObjectBulk( + diffs.map(d => ([`diff:${pid}.${d.score}`, { + pid: pid, + patch: d.value, + }])) + ); + })); + progress.incr(pids.length); }, { - progress: progress, - }, callback); + batch: 500, + }); }, }; diff --git a/src/upgrades/1.9.0/refresh_post_upload_associations.js b/src/upgrades/1.9.0/refresh_post_upload_associations.js index 44acfc079f..6183529641 100644 --- a/src/upgrades/1.9.0/refresh_post_upload_associations.js +++ b/src/upgrades/1.9.0/refresh_post_upload_associations.js @@ -1,21 +1,20 @@ 'use strict'; -const async = require('async'); +const db = require('../../database'); const posts = require('../../posts'); +const batch = require('../../batch'); module.exports = { name: 'Refresh post-upload associations', timestamp: Date.UTC(2018, 3, 16), - method: function (callback) { + method: async function () { const { progress } = this; - - require('../../batch').processSortedSet('posts:pid', (pids, next) => { - async.each(pids, (pid, next) => { - posts.uploads.sync(pid, next); - progress.incr(); - }, next); + progress.total = await db.sortedSetCard('posts:pid'); + await batch.processSortedSet('posts:pid', async (pids) => { + await Promise.all(pids.map(pid => posts.uploads.sync(pid))); + progress.incr(pids.length); }, { - progress: this.progress, - }, callback); + batch: 500, + }); }, }; diff --git a/src/upgrades/2.8.7/fix-email-sorted-sets.js b/src/upgrades/2.8.7/fix-email-sorted-sets.js index fcab69a8f4..84919e6774 100644 --- a/src/upgrades/2.8.7/fix-email-sorted-sets.js +++ b/src/upgrades/2.8.7/fix-email-sorted-sets.js @@ -26,7 +26,7 @@ module.exports = { } // user has email but doesn't match whats stored in user hash, gh#11259 - if (userData.email && userData.email.toLowerCase() !== email.toLowerCase()) { + if (userData.email && email && String(userData.email).toLowerCase() !== email.toLowerCase()) { bulkRemove.push(['email:uid', email]); bulkRemove.push(['email:sorted', `${email.toLowerCase()}:${uid}`]); } diff --git a/src/upgrades/3.7.0/category-read-by-uid.js b/src/upgrades/3.7.0/category-read-by-uid.js index 4ef564f53a..971620613e 100644 --- a/src/upgrades/3.7.0/category-read-by-uid.js +++ b/src/upgrades/3.7.0/category-read-by-uid.js @@ -9,6 +9,7 @@ module.exports = { method: async function () { const { progress } = this; const nextCid = await db.getObjectField('global', 'nextCid'); + progress.total = nextCid; const allCids = []; for (let i = 1; i <= nextCid; i++) { allCids.push(i); @@ -18,7 +19,6 @@ module.exports = { progress.incr(cids.length); }, { batch: 500, - progress, }); }, }; From 0a2fa45da1768c175f1821ea79e9eebdfb83faab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bar=C4=B1=C5=9F=20Soner=20U=C5=9Fakl=C4=B1?= Date: Thu, 25 Sep 2025 11:02:12 -0400 Subject: [PATCH 33/34] perf: update upgrade script to use bulk methods add missing progress.total --- src/upgrades/1.10.0/view_deleted_privilege.js | 1 + .../1.10.2/fix_category_topic_zsets.js | 24 +++++++++++-------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/src/upgrades/1.10.0/view_deleted_privilege.js b/src/upgrades/1.10.0/view_deleted_privilege.js index a483bcf417..3b65f2d5b7 100644 --- a/src/upgrades/1.10.0/view_deleted_privilege.js +++ b/src/upgrades/1.10.0/view_deleted_privilege.js @@ -11,6 +11,7 @@ module.exports = { method: async function () { const { progress } = this; const cids = await db.getSortedSetRange('categories:cid', 0, -1); + progress.total = cids.length; for (const cid of cids) { const uids = await db.getSortedSetRange(`group:cid:${cid}:privileges:moderate:members`, 0, -1); for (const uid of uids) { diff --git a/src/upgrades/1.10.2/fix_category_topic_zsets.js b/src/upgrades/1.10.2/fix_category_topic_zsets.js index 999383feac..83b4d7b27f 100644 --- a/src/upgrades/1.10.2/fix_category_topic_zsets.js +++ b/src/upgrades/1.10.2/fix_category_topic_zsets.js @@ -1,5 +1,3 @@ -/* eslint-disable no-await-in-loop */ - 'use strict'; const db = require('../../database'); @@ -13,18 +11,24 @@ module.exports = { const { progress } = this; const topics = require('../../topics'); + progress.total = await db.sortedSetCard('topics:tid'); await batch.processSortedSet('topics:tid', async (tids) => { - for (const tid of tids) { - progress.incr(); - const topicData = await db.getObjectFields(`topic:${tid}`, ['cid', 'pinned', 'postcount']); - if (parseInt(topicData.pinned, 10) !== 1) { + progress.incr(tids.length); + const topicData = await db.getObjectFields( + tids.map(tid => `topic:${tid}`), + ['tid', 'cid', 'pinned', 'postcount'], + ); + const bulkAdd = []; + topicData.forEach((topic) => { + if (topic && parseInt(topic.pinned, 10) !== 1) { topicData.postcount = parseInt(topicData.postcount, 10) || 0; - await db.sortedSetAdd(`cid:${topicData.cid}:tids:posts`, topicData.postcount, tid); + bulkAdd.push([`cid:${topicData.cid}:tids:posts`, topicData.postcount, topicData.tid]); } - await topics.updateLastPostTimeFromLastPid(tid); - } + }); + await db.sortedSetAddBulk(bulkAdd); + await Promise.all(tids.map(tid => topics.updateLastPostTimeFromLastPid(tid))); }, { - progress: progress, + batch: 500, }); }, }; From 7abdfd86ac87bcd4f5df7d17ae0cdf157177a991 Mon Sep 17 00:00:00 2001 From: Julian Lam Date: Thu, 25 Sep 2025 11:56:38 -0400 Subject: [PATCH 34/34] fix: skip header checking during note assertion if test runner is active --- src/activitypub/notes.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/activitypub/notes.js b/src/activitypub/notes.js index e1fc16c175..b85d686f1e 100644 --- a/src/activitypub/notes.js +++ b/src/activitypub/notes.js @@ -71,7 +71,7 @@ Notes.assert = async (uid, input, options = { skipChecks: false }) => { } try { - if (!options.skipChecks) { + if (!(options.skipChecks || process.env.hasOwnProperty('CI'))) { id = (await activitypub.checkHeader(id)) || id; }