diff --git a/src/activitypub/actors.js b/src/activitypub/actors.js
index 4a19aabf8c..3c31fac0e4 100644
--- a/src/activitypub/actors.js
+++ b/src/activitypub/actors.js
@@ -207,12 +207,19 @@ Actors.getLocalFollowers = async (id) => {
 	return response;
 };
 
-Actors.getLocalFollowersCount = async (id) => {
-	if (!activitypub.helpers.isUri(id)) {
-		return false;
+Actors.getLocalFollowCounts = async (actor) => {
+	let followers = 0; // x local followers
+	let following = 0; // following x local users
+	if (!activitypub.helpers.isUri(actor)) {
+		return { followers, following };
 	}
 
-	return await db.sortedSetCard(`followersRemote:${id}`);
+	[followers, following] = await Promise.all([
+		db.sortedSetCard(`followersRemote:${actor}`),
+		db.sortedSetCard(`followingRemote:${actor}`),
+	]);
+
+	return { followers, following };
 };
 
 Actors.remove = async (id) => {
@@ -270,7 +277,7 @@ Actors.prune = async () => {
 
 	await batch.processArray(uids, async (uids) => {
 		const exists = await db.exists(uids.map(uid => `userRemote:${uid}`));
-		const counts = await db.sortedSetsCard(uids.map(uid => `uid:${uid}:posts`));
+		const postCounts = await db.sortedSetsCard(uids.map(uid => `uid:${uid}:posts`));
 		await Promise.all(uids.map(async (uid, idx) => {
 			if (!exists[idx]) {
 				// id in zset but not asserted, handle and return early
@@ -278,8 +285,9 @@ Actors.prune = async () => {
 				return;
 			}
 
-			const count = counts[idx];
-			if (count < 1) {
+			const { followers, following } = await Actors.getLocalFollowCounts(uid);
+			const postCount = postCounts[idx];
+			if ([postCount, followers, following].every(metric => metric < 1)) {
 				try {
 					await user.deleteAccount(uid);
 					deletionCount += 1;
diff --git a/src/activitypub/inbox.js b/src/activitypub/inbox.js
index c1002388ed..72ecb0412b 100644
--- a/src/activitypub/inbox.js
+++ b/src/activitypub/inbox.js
@@ -235,8 +235,8 @@ inbox.announce = async (req) => {
 	} else { // Remote object
 		// Follower check
 		if (!cid) {
-			const numFollowers = await activitypub.actors.getLocalFollowersCount(actor);
-			if (!numFollowers) {
+			const { followers } = await activitypub.actors.getLocalFollowCounts(actor);
+			if (!followers) {
 				winston.verbose(`[activitypub/inbox.announce] Rejecting ${object.id} via ${actor} due to no followers`);
 				reject('Announce', object, actor);
 				return;
@@ -300,6 +300,7 @@ inbox.follow = async (req) => {
 
 		const now = Date.now();
 		await db.sortedSetAdd(`followersRemote:${id}`, now, actor);
+		await db.sortedSetAdd(`followingRemote:${actor}`, now, id); // for following backreference (actor pruning)
 
 		const followerRemoteCount = await db.sortedSetCard(`followersRemote:${id}`);
 		await user.setUserField(id, 'followerRemoteCount', followerRemoteCount);
@@ -422,7 +423,10 @@ inbox.undo = async (req) => {
 						throw new Error('[[error:invalid-uid]]');
 					}
 
-					await db.sortedSetRemove(`followersRemote:${id}`, actor);
+					await Promise.all([
+						db.sortedSetRemove(`followersRemote:${id}`, actor),
+						db.sortedSetRemove(`followingRemote:${actor}`, id),
+					]);
 					const followerRemoteCount = await db.sortedSetCard(`followerRemote:${id}`);
 					await user.setUserField(id, 'followerRemoteCount', followerRemoteCount);
 					notifications.rescind(`follow:${id}:uid:${actor}`);
diff --git a/src/activitypub/mocks.js b/src/activitypub/mocks.js
index e90cd564c0..c8d32247f7 100644
--- a/src/activitypub/mocks.js
+++ b/src/activitypub/mocks.js
@@ -40,10 +40,10 @@ Mocks.profile = async (actors) => {
 		let hostname;
 		let {
 			url, preferredUsername, published, icon, image,
-			name, summary, followers, followerCount, followingCount,
-			inbox, endpoints,
+			name, summary, followers, inbox, endpoints,
 		} = actor;
 		preferredUsername = preferredUsername || slugify(name);
+		const { followers: followerCount, following: followingCount } = await activitypub.actors.getLocalFollowCounts(uid);
 
 		try {
 			({ hostname } = new URL(actor.id));
diff --git a/src/activitypub/notes.js b/src/activitypub/notes.js
index 772ba2412a..a51156dd1e 100644
--- a/src/activitypub/notes.js
+++ b/src/activitypub/notes.js
@@ -181,7 +181,7 @@ async function assertRelation(post) {
 	 */
 
 	// Is followed by at least one local user
-	const numFollowers = await activitypub.actors.getLocalFollowersCount(post.uid);
+	const { followers } = await activitypub.actors.getLocalFollowCounts(post.uid);
 
 	// Local user is mentioned
 	const { tag } = post._activitypub;
@@ -201,7 +201,7 @@ async function assertRelation(post) {
 		uids = uids.filter(Boolean);
 	}
 
-	return numFollowers > 0 || uids.length;
+	return followers > 0 || uids.length;
 }
 
 Notes.updateLocalRecipients = async (id, { to, cc }) => {
diff --git a/src/upgrades/4.0.0/follow_backreferences.js b/src/upgrades/4.0.0/follow_backreferences.js
new file mode 100644
index 0000000000..ef15570a1d
--- /dev/null
+++ b/src/upgrades/4.0.0/follow_backreferences.js
@@ -0,0 +1,41 @@
+'use strict';
+
+const db = require('../../database');
+const batch = require('../../batch');
+const activitypub = require('../../activitypub');
+
+module.exports = {
+	name: 'Establish follow backreference sorted sets for remote users',
+	timestamp: Date.UTC(2024, 4, 1),
+	method: async function () {
+		const { progress } = this;
+		const bulkOp = [];
+		const now = Date.now();
+		const reassert = [];
+
+		await batch.processSortedSet('users:joindate', async (uids) => {
+			const [_followers, _following] = await Promise.all([
+				db.getSortedSetsMembers(uids.map(uid => `followersRemote:${uid}`)),
+				db.getSortedSetsMembers(uids.map(uid => `followingRemote:${uid}`)),
+			]);
+
+			const toCheck = Array.from(new Set(_following.flat()));
+			const asserted = await db.isSortedSetMembers('usersRemote:lastCrawled', toCheck);
+			reassert.push(...toCheck.filter((actor, idx) => !asserted[idx]));
+
+			uids.forEach((uid, idx) => {
+				const followers = _followers[idx];
+				if (followers.length) {
+					bulkOp.push(...followers.map(actor => [`followingRemote:${actor}`, now, uid]))
+				}
+			});
+
+			progress.incr(uids.length);
+		}, { progress });
+
+		await Promise.all([
+			db.sortedSetAddBulk(bulkOp),
+			activitypub.actors.assert(Array.from(new Set(reassert))),
+		]);
+	},
+};