refactor: allow passing an array to topics.purge and topics.purgePost… (#14018)

* refactor: allow passing an array to topics.purge and topics.purgePostsAndTopic

deprecate action:topic.purge, add action:topics.purge with array of deleted topics
update usage of topics.purge to pass in an array
fix an issue in posts/delete where cids were passed to parseInt, caused AP cids to get saved into category:NaN

* lint

* refactor: change style

* use array of tids
This commit is contained in:
Barış Uşaklı
2026-02-26 13:09:42 -05:00
committed by GitHub
parent b460506e4d
commit e4c945f636
14 changed files with 269 additions and 115 deletions

View File

@@ -748,11 +748,11 @@ Mocks.notes.public = async (post) => {
if (isArticle) {
// Preview is not adopted by anybody, so is left commented-out for now
// preview = {
// type: 'Note',
// attributedTo: `${nconf.get('url')}/uid/${post.user.uid}`,
// content: post.content,
// published,
// attachment,
// type: 'Note',
// attributedTo: `${nconf.get('url')}/uid/${post.user.uid}`,
// content: post.content,
// published,
// attachment,
// };
const breakString = '[...]';

View File

@@ -786,7 +786,7 @@ async function pruneCidTids(cid, cuttoff) {
winston.info(`[notes/prune] ${tidsWithNoEngagement.length} topics eligible in cid:${cid} for pruning`);
await batch.processArray(tidsWithNoEngagement, async (tids) => {
await Promise.all(tids.map(async tid => await topics.purgePostsAndTopic(tid, 0)));
await topics.purgePostsAndTopic(tids, 0);
}, { batch: 100 });
}

View File

@@ -87,11 +87,21 @@ module.exports = function (Categories) {
};
Categories.setCategoryField = async function (cid, field, value) {
await db.setObjectField(`${utils.isNumber(cid) ? 'category' : 'categoryRemote'}:${cid}`, field, value);
await db.setObjectField(
utils.isNumber(cid) ? `category:${cid}` : `categoryRemote:${cid}`,
field, value
);
};
Categories.setCategoryFields = async function (cid, fields) {
await db.setObject(
utils.isNumber(cid) ? `category:${cid}` : `categoryRemote:${cid}`,
fields
);
};
Categories.incrementCategoryFieldBy = async function (cid, field, value) {
await db.incrObjectFieldBy(`${utils.isNumber(cid) ? 'category' : 'categoryRemote'}:${cid}`, field, value);
await db.incrObjectFieldBy(utils.isNumber(cid) ? `category:${cid}` : `categoryRemote:${cid}`, field, value);
};
};

View File

@@ -1,6 +1,5 @@
'use strict';
const async = require('async');
const db = require('../database');
const batch = require('../batch');
const plugins = require('../plugins');
@@ -14,17 +13,14 @@ const utils = require('../utils');
module.exports = function (Categories) {
Categories.purge = async function (cid, uid) {
await batch.processSortedSet(`cid:${cid}:tids`, async (tids) => {
await async.eachLimit(tids, 10, async (tid) => {
await topics.purgePostsAndTopic(tid, uid);
});
await topics.purgePostsAndTopic(tids, uid);
await db.sortedSetRemove(`cid:${cid}:tids`, tids);
}, { alwaysStartAt: 0 });
const pinnedTids = await db.getSortedSetRevRange(`cid:${cid}:tids:pinned`, 0, -1);
await async.eachLimit(pinnedTids, 10, async (tid) => {
await topics.purgePostsAndTopic(tid, uid);
});
await topics.purgePostsAndTopic(pinnedTids, uid);
await db.sortedSetRemove(`cid:${cid}:tids:pinned`, pinnedTids);
const categoryData = await Categories.getCategoryData(cid);
await purgeCategory(cid, categoryData);
plugins.hooks.fire('action:category.delete', { cid: cid, uid: uid, category: categoryData });

View File

@@ -198,13 +198,15 @@ module.exports = function (Categories) {
Categories.onTopicsMoved = async (cids) => {
await Promise.all(cids.map(async (cid) => {
const [topicCount, postCount] = await db.sortedSetsCard([
`cid:${cid}:tids:lastposttime`,
`cid:${cid}:pids`,
]);
await Promise.all([
Categories.setCategoryField(
cid, 'topic_count', await db.sortedSetCard(`cid:${cid}:tids:lastposttime`)
),
Categories.setCategoryField(
cid, 'post_count', await db.sortedSetCard(`cid:${cid}:pids`)
),
Categories.setCategoryFields(cid, {
topic_count: topicCount,
post_count: postCount,
}),
Categories.updateRecentTidForCid(cid),
]);
}));

View File

@@ -13,6 +13,11 @@ Hooks._deprecated = new Map([
since: 'v4.0.0',
until: 'v5.0.0',
}], */
['action:topic.purge', {
new: 'action:topics.purge',
since: 'v4.9.0',
until: 'v5.0.0',
}],
]);
Hooks.internals = {

View File

@@ -108,9 +108,12 @@ module.exports = function (Posts) {
const localCount = postData.filter(p => utils.isNumber(p.pid)).length;
const incrObjectBulk = [['global', { postCount: -localCount }]];
const postsByCategory = _.groupBy(postData, p => parseInt(p.cid, 10));
const postsByCategory = _.groupBy(postData, p => String(p.cid));
for (const [cid, posts] of Object.entries(postsByCategory)) {
incrObjectBulk.push([`category:${cid}`, { post_count: -posts.length }]);
incrObjectBulk.push([
utils.isNumber(cid) ? `category:${cid}` : `categoryRemote:${cid}`,
{ post_count: -posts.length },
]);
}
const postsByTopic = _.groupBy(postData, p => String(p.tid));

View File

@@ -1,5 +1,6 @@
'use strict';
const winston = require('winston');
const _ = require('lodash');
const db = require('../database');
const topics = require('.');
@@ -117,8 +118,10 @@ Crossposts.add = async function (tid, cid, uid) {
Crossposts.remove = async function (tid, cid, uid) {
let crossposts = await Crossposts.get(tid);
const isPrivileged = await user.isAdminOrGlobalMod(uid);
const isMod = await user.isModerator(uid, cid);
const [isPrivileged, isMod] = await Promise.all([
user.isAdminOrGlobalMod(uid),
user.isModerator(uid, cid),
]);
const crosspostId = crossposts.reduce((id, { id: _id, cid: _cid, uid: _uid }) => {
if (String(cid) === String(_cid) && (isPrivileged || isMod || String(uid) === String(_uid))) {
id = _id;
@@ -156,18 +159,24 @@ Crossposts.remove = async function (tid, cid, uid) {
topics.events.find(tid, { uid, toCid: cid, type: 'crosspost' }).then((eventIds) => {
topics.events.purge(tid, eventIds);
});
}).catch(err => winston.error(err));
crossposts = await Crossposts.get(tid);
return crossposts;
};
Crossposts.removeAll = async function (tid) {
const crosspostIds = await db.getSortedSetMembers(`tid:${tid}:crossposts`);
const crossposts = await db.getObjects(crosspostIds.map(id => `crosspost:${id}`));
await Promise.all(crossposts.map(async ({ tid, cid, uid }) => {
return Crossposts.remove(tid, cid, uid);
}));
Crossposts.removeAll = async function (tids) {
if (!Array.isArray(tids)) {
tids = [tids];
}
const allCrosspostIds = (await db.getSortedSetsMembers(
tids.map(tid => `tid:${tid}:crossposts`)
)).flat();
const crossposts = (await db.getObjects(
allCrosspostIds.map(id => `crosspost:${id}`)
)).filter(Boolean);
return [];
await Promise.all(
crossposts.map(({ tid, cid, uid }) => Crossposts.remove(tid, cid, uid))
);
};

View File

@@ -1,7 +1,8 @@
'use strict';
const db = require('../database');
const _ = require('lodash');
const db = require('../database');
const user = require('../user');
const posts = require('../posts');
const categories = require('../categories');
@@ -62,90 +63,189 @@ module.exports = function (Topics) {
await categories.updateRecentTidForCid(cid);
};
Topics.purgePostsAndTopic = async function (tid, uid) {
const mainPid = await Topics.getTopicField(tid, 'mainPid');
await batch.processSortedSet(`tid:${tid}:posts`, async (pids) => {
await posts.purge(pids, uid);
await db.sortedSetRemove(`tid:${tid}:posts`, pids); // Guard against infinite loop if pid already does not exist in db
}, { alwaysStartAt: 0, batch: 500 });
await posts.purge(mainPid, uid);
await Topics.purge(tid, uid);
Topics.purgePostsAndTopic = async function (tids, uid) {
if (!Array.isArray(tids)) {
tids = [tids];
}
let topicData = await Topics.getTopicsFields(tids, ['tid', 'mainPid']);
topicData = topicData.filter(t => t && t.tid);
const tidsToDelete = topicData.map(t => t.tid);
await Promise.all(tidsToDelete.map(async (tid) => {
await batch.processSortedSet(`tid:${tid}:posts`, async (pids) => {
await posts.purge(pids, uid);
await db.sortedSetRemove(`tid:${tid}:posts`, pids); // Guard against infinite loop if pid already does not exist in db
}, { alwaysStartAt: 0, batch: 500 });
}));
await posts.purge(topicData.map(t => t.mainPid), uid);
await Topics.purge(tidsToDelete, uid);
};
Topics.purge = async function (tid, uid) {
const deletedTopic = await Topics.getTopicData(tid);
if (!deletedTopic) {
Topics.purge = async function (tids, uid) {
if (!Array.isArray(tids)) {
tids = [tids];
}
const deletedTopics = (await Topics.getTopicsData(tids)).filter(Boolean);
if (!deletedTopics.length) {
return;
}
deletedTopic.tags = deletedTopic.tags.map(tag => tag.value);
await deleteFromFollowersIgnorers(tid);
const tidsToDelete = deletedTopics.map(t => t.tid);
deletedTopics.forEach((t) => {
t.tags = t.tags.map(tag => tag.value);
});
await deleteFromFollowersIgnorers(tidsToDelete);
const remoteTids = [];
const localTids = [];
for (const tid of tidsToDelete) {
if (utils.isNumber(tid)) {
localTids.push(tid);
} else {
remoteTids.push(tid);
}
}
await Promise.all([
db.deleteAll([
`tid:${tid}:followers`,
`tid:${tid}:ignorers`,
`tid:${tid}:posts`,
`tid:${tid}:posts:votes`,
`tid:${tid}:bookmarks`,
`tid:${tid}:posters`,
]),
deleteKeys(tidsToDelete),
db.sortedSetsRemove([
utils.isNumber(tid) ? 'topics:tid' : 'topicsRemote:tid',
'topics:recent',
'topics:scheduled',
], tid),
db.sortedSetsRemove(['views', 'posts', 'votes'].map(prop => `${utils.isNumber(tid) ? 'topics' : 'topicsRemote'}:${prop}`), tid),
deleteTopicFromCategoryAndUser(tid),
Topics.deleteTopicTags(tid),
Topics.events.purge(tid),
Topics.thumbs.deleteAll(tid),
Topics.crossposts.removeAll(tid),
reduceCounters(tid),
], tidsToDelete),
db.sortedSetsRemove([
'topics:tid',
'topics:views',
'topics:posts',
'topics:votes',
], localTids),
db.sortedSetsRemove([
'topicsRemote:tid',
'topicsRemote:views',
'topicsRemote:posts',
'topicsRemote:votes',
], remoteTids),
deleteTopicsFromCategoryAndUser(deletedTopics),
deleteFromTags(deletedTopics),
Topics.events.purge(tidsToDelete),
Topics.crossposts.removeAll(tidsToDelete),
reduceCounters(deletedTopics),
]);
plugins.hooks.fire('action:topic.purge', { topic: deletedTopic, uid: uid });
await db.delete(`topic:${tid}`);
// DEPRECATED hook
deletedTopics.forEach((topic) => {
plugins.hooks.fire('action:topic.purge', { topic, uid });
});
// new hook
plugins.hooks.fire('action:topics.purge', { topics: deletedTopics, uid });
await db.deleteAll(tids.map(tid => `topic:${tid}`));
};
async function deleteFromFollowersIgnorers(tid) {
async function deleteFromFollowersIgnorers(tids) {
const [followers, ignorers] = await Promise.all([
db.getSetMembers(`tid:${tid}:followers`),
db.getSetMembers(`tid:${tid}:ignorers`),
db.getSetsMembers(tids.map(tid => `tid:${tid}:followers`)),
db.getSetsMembers(tids.map(tid => `tid:${tid}:ignorers`)),
]);
const followerKeys = followers.map(uid => `uid:${uid}:followed_tids`);
const ignorerKeys = ignorers.map(uid => `uid:${uid}ignored_tids`);
await db.sortedSetsRemove(followerKeys.concat(ignorerKeys), tid);
const bulkRemove = [];
tids.forEach((tid, index) => {
followers[index].forEach((uid) => {
bulkRemove.push([`uid:${uid}:followed_tids`, tid]);
});
ignorers[index].forEach((uid) => {
bulkRemove.push([`uid:${uid}:followed_tids`, tid]);
});
});
await db.sortedSetRemoveBulk(bulkRemove);
}
async function deleteTopicFromCategoryAndUser(tid) {
const topicData = await Topics.getTopicFields(tid, ['cid', 'uid']);
await Promise.all([
db.sortedSetsRemove([
`cid:${topicData.cid}:tids`,
`cid:${topicData.cid}:tids:pinned`,
`cid:${topicData.cid}:tids:create`,
`cid:${topicData.cid}:tids:posts`,
`cid:${topicData.cid}:tids:lastposttime`,
`cid:${topicData.cid}:tids:votes`,
`cid:${topicData.cid}:tids:views`,
`cid:${topicData.cid}:recent_tids`,
`cid:${topicData.cid}:uid:${topicData.uid}:tids`,
`uid:${topicData.uid}:topics`,
], tid),
user.decrementUserFieldBy(topicData.uid, 'topiccount', 1),
async function deleteKeys(tids) {
await db.deleteAll([
...tids.map(tid => `tid:${tid}:followers`),
...tids.map(tid => `tid:${tid}:ignorers`),
...tids.map(tid => `tid:${tid}:posts`),
...tids.map(tid => `tid:${tid}:posts:votes`),
...tids.map(tid => `tid:${tid}:bookmarks`),
...tids.map(tid => `tid:${tid}:posters`),
]);
await categories.updateRecentTidForCid(topicData.cid);
}
async function reduceCounters(tid) {
const incr = -1;
const { cid, postcount } = await Topics.getTopicFields(tid, ['cid', 'postcount']);
const postCountChange = incr * postcount;
const categoryKey = `${utils.isNumber(cid) ? 'category' : 'categoryRemote'}:${cid}`;
const bulkIncr = [
[categoryKey, { post_count: postCountChange, topic_count: incr }],
];
if (utils.isNumber(tid)) {
bulkIncr.push(['global', { postCount: postCountChange, topicCount: incr }]);
async function deleteTopicsFromCategoryAndUser(topicsData) {
const bulkRemove = [];
for (const topic of topicsData) {
bulkRemove.push([`cid:${topic.cid}:tids`, topic.tid]);
bulkRemove.push([`cid:${topic.cid}:tids:pinned`, topic.tid]);
bulkRemove.push([`cid:${topic.cid}:tids:create`, topic.tid]);
bulkRemove.push([`cid:${topic.cid}:tids:posts`, topic.tid]);
bulkRemove.push([`cid:${topic.cid}:tids:lastposttime`, topic.tid]);
bulkRemove.push([`cid:${topic.cid}:tids:votes`, topic.tid]);
bulkRemove.push([`cid:${topic.cid}:tids:views`, topic.tid]);
bulkRemove.push([`cid:${topic.cid}:recent_tids`, topic.tid]);
bulkRemove.push([`cid:${topic.cid}:uid:${topic.uid}:tids`, topic.tid]);
bulkRemove.push([`uid:${topic.uid}:topics`, topic.tid]);
}
await db.sortedSetRemoveBulk(bulkRemove);
const uniqCids = new Set();
const uniqUids = new Set();
topicsData.forEach((t) => {
uniqCids.add(String(t.cid));
uniqUids.add(String(t.uid));
});
await user.updateTopicCount(Array.from(uniqUids));
await Promise.all(Array.from(uniqCids).map(cid => categories.updateRecentTidForCid(cid)));
}
async function deleteFromTags(topicsData) {
const bulkRemove = [];
const uniqCids = new Set();
const uniqTags = new Set();
for (const topic of topicsData) {
for (const tag of topic.tags) {
bulkRemove.push([`tag:${tag}:topics`, topic.tid]);
bulkRemove.push([`cid:${topic.cid}:tag:${tag}:topics`, topic.tid]);
uniqTags.add(tag);
}
uniqCids.add(String(topic.cid));
}
await db.sortedSetRemoveBulk(bulkRemove);
await Topics.updateCategoryTagsCount(
Array.from(uniqCids),
Array.from(uniqTags)
);
await Topics.updateTagCount(uniqTags);
}
async function reduceCounters(topicsData) {
const bulkIncr = [];
let globalPostCountChange = 0;
let globalTopicCountChange = 0;
const topicsByCid = _.groupBy(topicsData, t => String(t.cid));
for (const [cid, topics] of Object.entries(topicsByCid)) {
const cidPostCountChange = Math.max(0, topics.reduce((acc, t) => acc + t.postcount, 0));
const categoryKey = `${utils.isNumber(cid) ? 'category' : 'categoryRemote'}:${cid}`;
bulkIncr.push([
categoryKey, { post_count: -cidPostCountChange, topic_count: -topics.length },
]);
for (const topic of topics) {
if (utils.isNumber(topic.tid)) {
globalPostCountChange += topic.postcount;
globalTopicCountChange += 1;
}
}
}
if (globalPostCountChange || globalTopicCountChange) {
bulkIncr.push([
'global', { postCount: -globalPostCountChange, topicCount: -globalTopicCountChange },
]);
}
await db.incrObjectFieldByBulk(bulkIncr);
}

View File

@@ -272,7 +272,11 @@ Events.log = async (tid, payload) => {
};
Events.purge = async (tid, eventIds = []) => {
if (eventIds.length) {
const isArray = Array.isArray(tid);
if (isArray && !tid.length) {
return;
}
if (eventIds.length && !isArray) {
const isTopicEvent = await db.isSortedSetMembers(`topic:${tid}:events`, eventIds);
eventIds = eventIds.filter((id, index) => isTopicEvent[index]);
await Promise.all([
@@ -280,8 +284,11 @@ Events.purge = async (tid, eventIds = []) => {
db.deleteAll(eventIds.map(id => `topicEvent:${id}`)),
]);
} else {
const keys = [`topic:${tid}:events`];
const eventIds = await db.getSortedSetRange(keys[0], 0, -1);
if (!isArray) {
tid = [tid];
}
const keys = tid.map(tid => `topic:${tid}:events`);
const eventIds = await db.getSortedSetRange(keys, 0, -1);
keys.push(...eventIds.map(id => `topicEvent:${id}`));
await db.deleteAll(keys);

View File

@@ -29,7 +29,7 @@ module.exports = function (Topics) {
);
await db.sortedSetsAdd(topicSets, timestamp, tid);
await Topics.updateCategoryTagsCount([cid], tags);
await Promise.all(tags.map(updateTagCount));
await updateTagCount(tags);
};
Topics.filterTags = async function (tags, cid) {
@@ -185,11 +185,21 @@ module.exports = function (Topics) {
await Topics.updateCategoryTagsCount(Object.keys(allCids), [newTagName]);
}
async function updateTagCount(tag) {
const count = await Topics.getTagTopicCount(tag);
await db.sortedSetAdd('tags:topic:count', count || 0, tag);
async function updateTagCount(tags) {
if (!Array.isArray(tags)) {
tags = [tags];
}
if (!tags.length) return;
const counts = await Promise.all(tags.map(tag => Topics.getTagTopicCount(tag)));
await db.sortedSetAdd(
'tags:topic:count',
tags.map((tag, index) => counts[index] || 0),
tags
);
cache.del('tags:topic:count');
}
Topics.updateTagCount = updateTagCount;
Topics.getTagTids = async function (tag, start, stop) {
const tids = await db.getSortedSetRevRange(`tag:${tag}:topics`, start, stop);
@@ -381,7 +391,7 @@ module.exports = function (Topics) {
db.setObjectBulk(bulkSet),
]);
await Promise.all(tags.map(updateTagCount));
await updateTagCount(tags);
await Topics.updateCategoryTagsCount(_.uniq(topicData.map(t => t.cid)), tags);
};
@@ -406,7 +416,7 @@ module.exports = function (Topics) {
db.setObjectBulk(bulkSet),
]);
await Promise.all(tags.map(updateTagCount));
await updateTagCount(tags);
await Topics.updateCategoryTagsCount(_.uniq(topicData.map(t => t.cid)), tags);
};
@@ -430,7 +440,7 @@ module.exports = function (Topics) {
await db.sortedSetsRemove(sets, tid);
await Topics.updateCategoryTagsCount([cid], tags);
await Promise.all(tags.map(updateTagCount));
await updateTagCount(tags);
};
Topics.searchTags = async function (data) {

View File

@@ -49,9 +49,7 @@ module.exports = function (User) {
async function deleteTopics(callerUid, uid) {
await batch.processSortedSet(`uid:${uid}:topics`, async (tids) => {
await async.eachSeries(tids, async (tid) => {
await topics.purge(tid, callerUid);
});
await topics.purge(tids, callerUid);
await db.sortedSetRemove(`uid:${uid}:topics`, tids);
}, { alwaysStartAt: 0, batch: 100 });
}

View File

@@ -111,12 +111,26 @@ module.exports = function (User) {
if (uids.length) {
const counts = await db.sortedSetsCard(uids.map(uid => `uid:${uid}:posts`));
await Promise.all([
db.setObjectBulk(uids.map((uid, index) => ([`user${activitypub.helpers.isUri(uid) ? 'Remote' : ''}:${uid}`, { postcount: counts[index] }]))),
db.setObjectBulk(
uids.map((uid, index) => ([activitypub.helpers.isUri(uid) ? `userRemote:${uid}` : `user:${uid}`, { postcount: counts[index] }]))
),
db.sortedSetAdd('users:postcount', counts, uids),
]);
}
};
User.updateTopicCount = async (uids) => {
uids = Array.isArray(uids) ? uids : [uids];
const exists = await User.exists(uids);
uids = uids.filter((uid, index) => exists[index]);
if (uids.length) {
const counts = await db.sortedSetsCard(uids.map(uid => `uid:${uid}:topics`));
await db.setObjectBulk(
uids.map((uid, index) => ([activitypub.helpers.isUri(uid) ? `userRemote:${uid}` : `user:${uid}`, { topiccount: counts[index] }]))
);
}
};
User.incrementUserPostCountBy = async function (uid, value) {
return await incrementUserFieldAndSetBy(uid, 'postcount', 'users:postcount', value);
};

View File

@@ -2342,7 +2342,7 @@ describe('Topic\'s', () => {
});
it('should create a scheduled topic as pinned, deleted, included in "topics:scheduled" zset and with a timestamp in future', async () => {
topicData = (await topics.post(topic)).topicData;
topicData = (await topics.post({ ...topic })).topicData;
topicData = await topics.getTopicData(topicData.tid);
assert(topicData.pinned);
@@ -2499,7 +2499,7 @@ describe('Topic\'s', () => {
});
it('should allow to purge a scheduled topic', async () => {
topicData = (await topics.post(topic)).topicData;
const { topicData } = await topics.post({ ...topic });
const { response } = await request.delete(`${nconf.get('url')}/api/v3/topics/${topicData.tid}`, adminApiOpts);
assert.strictEqual(response.statusCode, 200);
});