From 263c918088d70d861805880eeccc1fb464f584cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bar=C4=B1=C5=9F=20Soner=20U=C5=9Fakl=C4=B1?= Date: Wed, 12 Dec 2018 11:15:34 -0500 Subject: [PATCH] feat: remove uid::ignored:cids (#7099) use cid::ignorers instead --- src/categories/index.js | 3 +- src/database/mongo/sorted.js | 4 +- src/database/postgres/sorted.js | 4 +- src/database/redis/sorted.js | 3 ++ src/topics/sorted.js | 46 +++++++++---------- src/topics/unread.js | 5 +- .../1.11.1/remove_ignored_cids_per_user.js | 21 +++++++++ src/user/categories.js | 22 ++++++--- src/user/delete.js | 2 +- test/database/sorted.js | 9 ++++ 10 files changed, 80 insertions(+), 39 deletions(-) create mode 100644 src/upgrades/1.11.1/remove_ignored_cids_per_user.js diff --git a/src/categories/index.js b/src/categories/index.js index 3a325b33f1..cba2da02c8 100644 --- a/src/categories/index.js +++ b/src/categories/index.js @@ -80,7 +80,8 @@ Categories.isIgnored = function (cids, uid, callback) { if (parseInt(uid, 10) <= 0) { return setImmediate(callback, null, cids.map(() => false)); } - db.isSortedSetMembers('uid:' + uid + ':ignored:cids', cids, callback); + const keys = cids.map(cid => 'cid:' + cid + ':ignorers'); + db.isMemberOfSortedSets(keys, uid, callback); }; Categories.getAllCidsFromSet = function (key, callback) { diff --git a/src/database/mongo/sorted.js b/src/database/mongo/sorted.js index 0cfdf4a1d4..3fee97fa2f 100644 --- a/src/database/mongo/sorted.js +++ b/src/database/mongo/sorted.js @@ -339,8 +339,8 @@ module.exports = function (db, module) { }; module.isMemberOfSortedSets = function (keys, value, callback) { - if (!Array.isArray(keys)) { - return callback(); + if (!Array.isArray(keys) || !keys.length) { + return setImmediate(callback, null, []); } value = helpers.valueToString(value); db.collection('objects').find({ _key: { $in: keys }, value: value }, { projection: { _id: 0, score: 0 } }).toArray(function (err, results) { diff --git a/src/database/postgres/sorted.js b/src/database/postgres/sorted.js index ecc56e53d0..ad749c19bd 100644 --- a/src/database/postgres/sorted.js +++ b/src/database/postgres/sorted.js @@ -473,8 +473,8 @@ SELECT z."value" v }; module.isMemberOfSortedSets = function (keys, value, callback) { - if (!Array.isArray(keys)) { - return callback(); + if (!Array.isArray(keys) || !keys.length) { + return setImmediate(callback, null, []); } value = helpers.valueToString(value); diff --git a/src/database/redis/sorted.js b/src/database/redis/sorted.js index 883bf54455..471d896852 100644 --- a/src/database/redis/sorted.js +++ b/src/database/redis/sorted.js @@ -206,6 +206,9 @@ module.exports = function (redisClient, module) { }; module.isMemberOfSortedSets = function (keys, value, callback) { + if (!Array.isArray(keys) || !keys.length) { + return setImmediate(callback, null, []); + } helpers.execKeysValue(redisClient, 'batch', 'zscore', keys, value, function (err, results) { if (err) { return callback(err); diff --git a/src/topics/sorted.js b/src/topics/sorted.js index 2347134784..b5328d0c6d 100644 --- a/src/topics/sorted.js +++ b/src/topics/sorted.js @@ -1,13 +1,15 @@ 'use strict'; -var async = require('async'); +const async = require('async'); +const _ = require('lodash'); -var db = require('../database'); -var privileges = require('../privileges'); -var user = require('../user'); -var meta = require('../meta'); -var plugins = require('../plugins'); +const db = require('../database'); +const privileges = require('../privileges'); +const user = require('../user'); +const categories = require('../categories'); +const meta = require('../meta'); +const plugins = require('../plugins'); module.exports = function (Topics) { Topics.getSortedTopics = function (params, callback) { @@ -117,6 +119,8 @@ module.exports = function (Topics) { const filter = params.filter; const uid = params.uid; + let topicData; + let topicCids; async.waterfall([ function (next) { if (filter === 'watched') { @@ -133,35 +137,29 @@ module.exports = function (Topics) { privileges.topics.filterTids('read', tids, uid, next); }, function (tids, next) { + Topics.getTopicsFields(tids, ['uid', 'tid', 'cid'], next); + }, + function (_topicData, next) { + topicData = _topicData; + topicCids = _.uniq(topicData.map(topic => topic.cid)).filter(Boolean); + async.parallel({ ignoredCids: function (next) { if (filter === 'watched' || meta.config.disableRecentCategoryFilter) { return next(null, []); } - user.getIgnoredCategories(uid, next); - }, - topicData: function (next) { - Topics.getTopicsFields(tids, ['uid', 'tid', 'cid'], next); + categories.isIgnored(topicCids, uid, next); }, + filtered: async.apply(user.blocks.filter, uid, topicData), }, next); }, function (results, next) { - user.blocks.filter(uid, results.topicData, function (err, filtered) { - if (err) { - return next(err); - } + const isCidIgnored = _.zipObject(topicCids, results.ignoredCids); + topicData = results.filtered; - results.topicData = filtered; - next(null, results); - }); - }, - function (results, next) { const cids = params.cids && params.cids.map(String); - tids = results.topicData.filter(function (topic) { - if (topic && topic.cid) { - return !results.ignoredCids.includes(topic.cid.toString()) && (!cids || (cids.length && cids.includes(topic.cid.toString()))); - } - return false; + tids = topicData.filter(function (topic) { + return topic && topic.cid && !isCidIgnored[topic.cid] && (!cids || (cids.length && cids.includes(topic.cid.toString()))); }).map(topic => topic.tid); plugins.fireHook('filter:topics.filterSortedTids', { tids: tids, params: params }, next); }, diff --git a/src/topics/unread.js b/src/topics/unread.js index 5610b46a65..5e02e2e686 100644 --- a/src/topics/unread.js +++ b/src/topics/unread.js @@ -195,7 +195,7 @@ module.exports = function (Topics) { db.sortedSetScores('uid:' + uid + ':followed_tids', tids, next); }, ignoredCids: function (next) { - user.getIgnoredCategories(uid, next); + categories.isIgnored(cids, uid, next); }, readableCids: function (next) { privileges.categories.filterCids('read', cids, uid, next); @@ -205,6 +205,7 @@ module.exports = function (Topics) { function (results, next) { cid = cid && cid.map(String); results.readableCids = results.readableCids.map(String); + const isCidIgnored = _.zipObject(cids, results.ignoredCids); topicData.forEach(function (topic, index) { function cidMatch(topicCid) { @@ -213,7 +214,7 @@ module.exports = function (Topics) { if (topic && topic.cid && cidMatch(topic.cid) && !blockedUids.includes(parseInt(topic.uid, 10))) { topic.tid = parseInt(topic.tid, 10); - if ((results.isTopicsFollowed[index] || !results.ignoredCids.includes(String(topic.cid)))) { + if ((results.isTopicsFollowed[index] || !isCidIgnored[topic.cid])) { counts[''] += 1; tidsByFilter[''].push(topic.tid); } diff --git a/src/upgrades/1.11.1/remove_ignored_cids_per_user.js b/src/upgrades/1.11.1/remove_ignored_cids_per_user.js new file mode 100644 index 0000000000..f175e4074f --- /dev/null +++ b/src/upgrades/1.11.1/remove_ignored_cids_per_user.js @@ -0,0 +1,21 @@ +'use strict'; + +var db = require('../../database'); + +var batch = require('../../batch'); + +module.exports = { + name: 'Remove uid::ignored:cids', + timestamp: Date.UTC(2018, 11, 11), + method: function (callback) { + const progress = this.progress; + + batch.processSortedSet('users:joindate', function (uids, next) { + progress.incr(uids.length); + const keys = uids.map(uid => 'uid:' + uid + ':ignored:cids'); + db.deleteAll(keys, next); + }, { + progress: this.progress, + }, callback); + }, +}; diff --git a/src/user/categories.js b/src/user/categories.js index 8a825e2a61..b81bb8935e 100644 --- a/src/user/categories.js +++ b/src/user/categories.js @@ -10,7 +10,19 @@ module.exports = function (User) { if (parseInt(uid, 10) <= 0) { return setImmediate(callback, null, []); } - db.getSortedSetRange('uid:' + uid + ':ignored:cids', 0, -1, callback); + let cids; + async.waterfall([ + function (next) { + categories.getAllCidsFromSet('categories:cid', next); + }, + function (_cids, next) { + cids = _cids; + db.isMemberOfSortedSets(cids.map(cid => 'cid:' + cid + ':ignorers'), uid, next); + }, + function (isMembers, next) { + next(null, cids.filter((cid, index) => isMembers[index])); + }, + ], callback); }; User.getWatchedCategories = function (uid, callback) { @@ -46,9 +58,7 @@ module.exports = function (User) { if (!exists) { return next(new Error('[[error:no-category]]')); } - db.sortedSetAdd('uid:' + uid + ':ignored:cids', Date.now(), cid, next); - }, - function (next) { + db.sortedSetAdd('cid:' + cid + ':ignorers', Date.now(), uid, next); }, ], callback); @@ -67,9 +77,7 @@ module.exports = function (User) { if (!exists) { return next(new Error('[[error:no-category]]')); } - db.sortedSetRemove('uid:' + uid + ':ignored:cids', cid, next); - }, - function (next) { + db.sortedSetRemove('cid:' + cid + ':ignorers', uid, next); }, ], callback); diff --git a/src/user/delete.js b/src/user/delete.js index de82799505..3c29e23ecb 100644 --- a/src/user/delete.js +++ b/src/user/delete.js @@ -160,7 +160,7 @@ module.exports = function (User) { 'uid:' + uid + ':chats', 'uid:' + uid + ':chats:unread', 'uid:' + uid + ':chat:rooms', 'uid:' + uid + ':chat:rooms:unread', 'uid:' + uid + ':upvote', 'uid:' + uid + ':downvote', - 'uid:' + uid + ':ignored:cids', 'uid:' + uid + ':flag:pids', + 'uid:' + uid + ':flag:pids', 'uid:' + uid + ':sessions', 'uid:' + uid + ':sessionUUID:sessionId', ]; db.deleteAll(keys, next); diff --git a/test/database/sorted.js b/test/database/sorted.js index fcbea80e76..da2e25c951 100644 --- a/test/database/sorted.js +++ b/test/database/sorted.js @@ -627,6 +627,15 @@ describe('Sorted Set methods', function () { done(); }); }); + + it('should return empty array if keys is empty array', function (done) { + db.isMemberOfSortedSets([], 'value2', function (err, isMembers) { + assert.ifError(err); + assert.equal(arguments.length, 2); + assert.deepEqual(isMembers, []); + done(); + }); + }); }); describe('getSortedSetsMembers', function () {