Nemcio/NodeBB
1
0
Fork 0
mirror of https://github.com/NodeBB/NodeBB.git synced 2026-07-04 16:08:59 +02:00
Code Issues Packages Projects Releases Wiki Activity

refactor: actor pruning logic

Browse Source 
Remove re-assertion set as it is expensive to re-assert all old user accounts.

Update actor assertion logic to always re-assert a passed-in id if their account's last crawl date is older than the configurable pruning threshold.

fixes #12636
This commit is contained in:
Julian Lam
2024-06-14 10:43:03 -04:00
parent 937caecdc4
commit 6227ab900e
1 changed files with 8 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
src/activitypub/actors.js
View File

@@ -62,10 +62,14 @@ Actors.assert = async (ids, options = {}) => {
// Filter out loopback uris // Filter out loopback uris
ids = ids.filter(uri => uri !== 'loopback' && new URL(uri).host !== nconf.get('url_parsed').host); ids = ids.filter(uri => uri !== 'loopback' && new URL(uri).host !== nconf.get('url_parsed').host);
// Filter out existing // Only assert those who haven't been seen recently (configurable), unless update flag passed in (force refresh)
if (!options.update) { if (!options.update) {
const exists = await db.isSortedSetMembers('usersRemote:lastCrawled', ids.map(id => ((typeof id === 'object' && id.hasOwnProperty('id')) ? id.id : id))); const upperBound = Date.now() - (1000 * 60 * 60 * 24 * meta.config.activitypubUserPruneDays);
ids = ids.filter((id, idx) => !exists[idx]); const lastCrawled = await db.sortedSetScores('usersRemote:lastCrawled', ids.map(id => ((typeof id === 'object' && id.hasOwnProperty('id')) ? id.id : id)));
ids = ids.filter((id, idx) => {
const timestamp = lastCrawled[idx];
return !timestamp || timestamp < upperBound;
});
} }
if (!ids.length) { if (!ids.length) {
@@ -272,7 +276,6 @@ Actors.prune = async () => {
winston.info(`[actors/prune] Found ${uids.length} remote users last crawled more than ${days} days ago`); winston.info(`[actors/prune] Found ${uids.length} remote users last crawled more than ${days} days ago`);
let deletionCount = 0; let deletionCount = 0;
const reassertionSet = new Set();
await batch.processArray(uids, async (uids) => { await batch.processArray(uids, async (uids) => {
const exists = await db.exists(uids.map(uid => `userRemote:${uid}`)); const exists = await db.exists(uids.map(uid => `userRemote:${uid}`));
@@ -292,8 +295,6 @@ Actors.prune = async () => {
} catch (err) { } catch (err) {
winston.error(err.stack); winston.error(err.stack);
} }
} else {
reassertionSet.add(uid);
} }
})); }));
}, { }, {
@@ -301,7 +302,5 @@ Actors.prune = async () => {
interval: 1000, interval: 1000,
}); });
winston.info(`[actors/prune] ${deletionCount} remote users pruned, re-asserting ${reassertionSet.size} remote users.`); winston.info(`[actors/prune] ${deletionCount} remote users pruned.`);
await Actors.assert(Array.from(reassertionSet), { update: true });
}; };