From 224910b133d32c85a09ef5d16e4702f2c72ac532 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bar=C4=B1=C5=9F=20Soner=20U=C5=9Fakl=C4=B1?= Date: Tue, 11 Feb 2025 10:48:42 -0500 Subject: [PATCH 1/3] fix: isArray check --- src/activitypub/actors.js | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/src/activitypub/actors.js b/src/activitypub/actors.js index 2208f03de7..b5ecdc8f96 100644 --- a/src/activitypub/actors.js +++ b/src/activitypub/actors.js @@ -230,19 +230,29 @@ Actors.getLocalFollowers = async (id) => { return response; }; -Actors.getLocalFollowCounts = async (actor) => { - let followers = 0; // x local followers - let following = 0; // following x local users - if (!activitypub.helpers.isUri(actor)) { - return { followers, following }; +Actors.getLocalFollowCounts = async (actors) => { + const isArray = Array.isArray(actors); + if (!isArray) { + actors = [actors]; } - [followers, following] = await Promise.all([ - db.sortedSetCard(`followersRemote:${actor}`), - db.sortedSetCard(`followingRemote:${actor}`), + const validActors = actors.filter(actor => activitypub.helpers.isUri(actor)); + const followerKeys = validActors.map(actor => `followersRemote:${actor}`); + const followingKeys = validActors.map(actor => `followingRemote:${actor}`); + + const [followersCounts, followingCounts] = await Promise.all([ + db.sortedSetsCard(followerKeys), + db.sortedSetsCard(followingKeys), ]); - return { followers, following }; + const results = actors.map((actor, index) => { + if (!validActors.includes(actor)) { + return { followers: 0, following: 0 }; + } + return { followers: followersCounts[index], following: followingCounts[index] }; + }); + + return isArray ? results : results[0]; }; Actors.remove = async (id) => { From d590c2afcf9a680323b2387d86378756e9b06d24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bar=C4=B1=C5=9F=20Soner=20U=C5=9Fakl=C4=B1?= Date: Tue, 11 Feb 2025 10:39:24 -0500 Subject: [PATCH 2/3] perf: closes #13145, reduce calls in actors.prune instead of deleting 18k users everyday delete max 500 every 30mins like notifications.prune refactor getLocalFollowCounts so it works with an array of actors dont make dbcalls for uids that dont exist --- src/activitypub/actors.js | 25 +++++++++++++------------ src/activitypub/index.js | 2 +- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/src/activitypub/actors.js b/src/activitypub/actors.js index b5ecdc8f96..ef4e1497c1 100644 --- a/src/activitypub/actors.js +++ b/src/activitypub/actors.js @@ -298,7 +298,7 @@ Actors.prune = async () => { const days = parseInt(meta.config.activitypubUserPruneDays, 10); const timestamp = Date.now() - (1000 * 60 * 60 * 24 * days); - const uids = await db.getSortedSetRangeByScore('usersRemote:lastCrawled', 0, -1, '-inf', timestamp); + const uids = await db.getSortedSetRangeByScore('usersRemote:lastCrawled', 0, 500, '-inf', timestamp); if (!uids.length) { winston.info('[actors/prune] No remote users to prune, all done.'); return; @@ -309,18 +309,19 @@ Actors.prune = async () => { await batch.processArray(uids, async (uids) => { const exists = await db.exists(uids.map(uid => `userRemote:${uid}`)); - const [postCounts, roomCounts] = await Promise.all([ - db.sortedSetsCard(uids.map(uid => `uid:${uid}:posts`)), - db.sortedSetsCard(uids.map(uid => `uid:${uid}:chat:rooms`)), - ]); - await Promise.all(uids.map(async (uid, idx) => { - if (!exists[idx]) { - // id in zset but not asserted, handle and return early - await db.sortedSetRemove('usersRemote:lastCrawled', uid); - return; - } - const { followers, following } = await Actors.getLocalFollowCounts(uid); + const uidsThatExist = uids.filter((uid, idx) => exists[idx]); + const uidsThatDontExist = uids.filter((uid, idx) => !exists[idx]); + await db.sortedSetRemove('usersRemote:lastCrawled', uidsThatDontExist); + + const [postCounts, roomCounts, followCounts] = await Promise.all([ + db.sortedSetsCard(uidsThatExist.map(uid => `uid:${uid}:posts`)), + db.sortedSetsCard(uidsThatExist.map(uid => `uid:${uid}:chat:rooms`)), + Actors.getLocalFollowCounts(uidsThatExist), + ]); + + await Promise.all(uidsThatExist.map(async (uid, idx) => { + const { followers, following } = followCounts[idx]; const postCount = postCounts[idx]; const roomCount = roomCounts[idx]; if ([postCount, roomCount, followers, following].every(metric => metric < 1)) { diff --git a/src/activitypub/index.js b/src/activitypub/index.js index 4a6d95b48b..3eec80eae1 100644 --- a/src/activitypub/index.js +++ b/src/activitypub/index.js @@ -66,7 +66,7 @@ ActivityPub.startJobs = () => { } }, null, true, null, null, false); // change last argument to true for debugging - new CronJob('0 1 * * *', async () => { + new CronJob('*/30 * * * *', async () => { try { await ActivityPub.actors.prune(); } catch (err) { From e85662a536bf352e023f8c8747ea812b0df59f8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bar=C4=B1=C5=9F=20Soner=20U=C5=9Fakl=C4=B1?= Date: Tue, 11 Feb 2025 13:28:25 -0500 Subject: [PATCH 3/3] fix: getUserField so that it always returns null if field doesn't exist user.getUserField('foo', 'bar') returned undefined vs user.getUserField('1', 'bar') which returned null --- src/user/data.js | 2 +- test/user.js | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/user/data.js b/src/user/data.js index ec643f729b..c9d8cb5ae6 100644 --- a/src/user/data.js +++ b/src/user/data.js @@ -161,7 +161,7 @@ module.exports = function (User) { User.getUserField = async function (uid, field) { const user = await User.getUserFields(uid, [field]); - return user ? user[field] : null; + return user && user.hasOwnProperty(field) ? user[field] : null; }; User.getUserFields = async function (uid, fields) { diff --git a/test/user.js b/test/user.js index 84d8711260..579386b166 100644 --- a/test/user.js +++ b/test/user.js @@ -680,6 +680,12 @@ describe('User', () => { done(); }); }); + + it('should return null if field or user doesn not exist', async () => { + assert.strictEqual(await User.getUserField('1', 'doesnotexist'), null); + assert.strictEqual(await User.getUserField('doesnotexistkey', 'doesnotexist'), null); + assert.strictEqual(await User.getUserField('0', 'doesnotexist'), null); + }); }); describe('profile methods', () => {