From 6a9a73c86c9795fb2755fd1a71ba99044ae4e74e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bar=C4=B1=C5=9F=20Soner=20U=C5=9Fakl=C4=B1?= Date: Sat, 17 Mar 2018 18:49:33 -0400 Subject: [PATCH] changes to leaveAllGroups Groups.destroy can take an array of groupnames Groups.leave can take an array of groupnames db.incrObjectField/decrObjectField can take an array of keys db.sortedSetRemove can take an array of keys and values db.setRemove can take an array of keys --- src/database/mongo/hash.js | 28 +++++++ src/database/mongo/sets.js | 24 +++--- src/database/mongo/sorted.js | 2 +- src/database/mongo/sorted/remove.js | 8 +- src/database/redis/hash.js | 10 ++- src/database/redis/sets.js | 12 ++- src/database/redis/sorted/remove.js | 16 +++- src/groups/data.js | 12 ++- src/groups/delete.js | 60 +++++++++------ src/groups/membership.js | 109 +++++++++++++++++++--------- test/database/hash.js | 9 +++ test/database/sets.js | 21 ++++++ test/database/sorted.js | 36 +++++++++ 13 files changed, 266 insertions(+), 81 deletions(-) diff --git a/src/database/mongo/hash.js b/src/database/mongo/hash.js index 5eca32fd4e..64815d5a8f 100644 --- a/src/database/mongo/hash.js +++ b/src/database/mongo/hash.js @@ -1,5 +1,6 @@ 'use strict'; +var async = require('async'); var pubsub = require('../../pubsub'); module.exports = function (db, module) { @@ -286,6 +287,33 @@ module.exports = function (db, module) { field = helpers.fieldToString(field); data[field] = value; + if (Array.isArray(key)) { + var bulk = db.collection('objects').initializeUnorderedBulkOp(); + bulk.find({ _key: { $in: key } }).upsert().update({ $inc: data }); + async.waterfall([ + function (next) { + bulk.execute(function (err) { + next(err); + }); + }, + function (next) { + key.forEach(function (key) { + module.delObjectCache(key); + }); + + module.getObjectsFields(key, [field], next); + }, + function (data, next) { + data = data.map(function (data) { + return data && data[field]; + }); + next(null, data); + }, + ], callback); + return; + } + + db.collection('objects').findAndModify({ _key: key }, {}, { $inc: data }, { new: true, upsert: true }, function (err, result) { if (err) { return callback(err); diff --git a/src/database/mongo/sets.js b/src/database/mongo/sets.js index 6d6abbfbab..3315922b4a 100644 --- a/src/database/mongo/sets.js +++ b/src/database/mongo/sets.js @@ -47,7 +47,7 @@ module.exports = function (db, module) { var bulk = db.collection('objects').initializeUnorderedBulkOp(); for (var i = 0; i < keys.length; i += 1) { - bulk.find({ _key: keys[i] }).upsert().updateOne({ $addToSet: { + bulk.find({ _key: keys[i] }).upsert().updateOne({ $addToSet: { members: { $each: value, }, @@ -69,9 +69,15 @@ module.exports = function (db, module) { array[index] = helpers.valueToString(element); }); - db.collection('objects').update({ _key: key }, { $pullAll: { members: value } }, function (err) { - callback(err); - }); + if (Array.isArray(key)) { + db.collection('objects').updateMany({ _key: { $in: key } }, { $pullAll: { members: value } }, function (err) { + callback(err); + }); + } else { + db.collection('objects').update({ _key: key }, { $pullAll: { members: value } }, function (err) { + callback(err); + }); + } }; module.setsRemove = function (keys, value, callback) { @@ -81,15 +87,7 @@ module.exports = function (db, module) { } value = helpers.valueToString(value); - var bulk = db.collection('objects').initializeUnorderedBulkOp(); - - for (var i = 0; i < keys.length; i += 1) { - bulk.find({ _key: keys[i] }).updateOne({ $pull: { - members: value, - } }); - } - - bulk.execute(function (err) { + db.collection('objects').update({ _key: { $in: keys } }, { $pull: { members: value } }, { multi: true }, function (err) { callback(err); }); }; diff --git a/src/database/mongo/sorted.js b/src/database/mongo/sorted.js index ca1d53f63b..58eddb940e 100644 --- a/src/database/mongo/sorted.js +++ b/src/database/mongo/sorted.js @@ -384,7 +384,7 @@ module.exports = function (db, module) { if (!Array.isArray(keys) || !keys.length) { return callback(null, []); } - db.collection('objects').find({ _key: { $in: keys } }, { _id: 0, _key: 1, value: 1 }).toArray(function (err, data) { + db.collection('objects').find({ _key: { $in: keys } }, { _id: 0, _key: 1, value: 1 }).sort({ score: 1 }).toArray(function (err, data) { if (err) { return callback(err); } diff --git a/src/database/mongo/sorted/remove.js b/src/database/mongo/sorted/remove.js index 0a3fd87b6e..c9bf121e10 100644 --- a/src/database/mongo/sorted/remove.js +++ b/src/database/mongo/sorted/remove.js @@ -11,10 +11,14 @@ module.exports = function (db, module) { if (!key) { return callback(); } - - if (Array.isArray(value)) { + if (Array.isArray(key) && Array.isArray(value)) { + db.collection('objects').remove({ _key: { $in: key }, value: { $in: value } }, done); + } else if (Array.isArray(value)) { value = value.map(helpers.valueToString); db.collection('objects').remove({ _key: key, value: { $in: value } }, done); + } else if (Array.isArray(key)) { + value = helpers.valueToString(value); + db.collection('objects').remove({ _key: { $in: key }, value: value }, done); } else { value = helpers.valueToString(value); db.collection('objects').remove({ _key: key, value: value }, done); diff --git a/src/database/redis/hash.js b/src/database/redis/hash.js index 6f3c799027..9dd6276f88 100644 --- a/src/database/redis/hash.js +++ b/src/database/redis/hash.js @@ -129,6 +129,14 @@ module.exports = function (redisClient, module) { if (!key || isNaN(value)) { return callback(null, null); } - redisClient.hincrby(key, field, value, callback); + if (Array.isArray(key)) { + var multi = redisClient.multi(); + key.forEach(function (key) { + multi.hincrby(key, field, value); + }); + multi.exec(callback); + } else { + redisClient.hincrby(key, field, value, callback); + } }; }; diff --git a/src/database/redis/sets.js b/src/database/redis/sets.js index a5716438e4..1da6f4ff22 100644 --- a/src/database/redis/sets.js +++ b/src/database/redis/sets.js @@ -25,7 +25,17 @@ module.exports = function (redisClient, module) { module.setRemove = function (key, value, callback) { callback = callback || function () {}; - redisClient.srem(key, value, function (err) { + if (!Array.isArray(value)) { + value = [value]; + } + if (!Array.isArray(key)) { + key = [key]; + } + var multi = redisClient.multi(); + key.forEach(function (key) { + multi.srem(key, value); + }); + multi.exec(function (err) { callback(err); }); }; diff --git a/src/database/redis/sorted/remove.js b/src/database/redis/sorted/remove.js index eacb6ca861..886bd7cdb9 100644 --- a/src/database/redis/sorted/remove.js +++ b/src/database/redis/sorted/remove.js @@ -13,9 +13,19 @@ module.exports = function (redisClient, module) { value = [value]; } - helpers.multiKeyValues(redisClient, 'zrem', key, value, function (err) { - callback(err); - }); + if (Array.isArray(key)) { + var multi = redisClient.multi(); + key.forEach(function (key) { + multi.zrem(key, value); + }); + multi.exec(function (err) { + callback(err); + }); + } else { + helpers.multiKeyValues(redisClient, 'zrem', key, value, function (err) { + callback(err); + }); + } }; module.sortedSetsRemove = function (keys, value, callback) { diff --git a/src/groups/data.js b/src/groups/data.js index 919a257368..f791b1f5f1 100644 --- a/src/groups/data.js +++ b/src/groups/data.js @@ -2,6 +2,7 @@ var async = require('async'); var validator = require('validator'); +var winston = require('winston'); var db = require('../database'); var plugins = require('../plugins'); @@ -68,17 +69,22 @@ module.exports = function (Groups) { }; Groups.getGroupFields = function (groupName, fields, callback) { - Groups.getMultipleGroupFields([groupName], fields, function (err, groups) { + Groups.getGroupsFields([groupName], fields, function (err, groups) { callback(err, groups ? groups[0] : null); }); }; - Groups.getMultipleGroupFields = function (groups, fields, callback) { - db.getObjectsFields(groups.map(function (group) { + Groups.getGroupsFields = function (groupNames, fields, callback) { + db.getObjectsFields(groupNames.map(function (group) { return 'group:' + group; }), fields, callback); }; + Groups.getMultipleGroupFields = function (groups, fields, callback) { + winston.warn('[deprecated] Groups.getMultipleGroupFields is deprecated please use Groups.getGroupsFields'); + Groups.getGroupsFields(groups, fields, callback); + }; + Groups.setGroupField = function (groupName, field, value, callback) { async.waterfall([ function (next) { diff --git a/src/groups/delete.js b/src/groups/delete.js index b0203aa60e..ed55a1f383 100644 --- a/src/groups/delete.js +++ b/src/groups/delete.js @@ -7,44 +7,61 @@ var db = require('./../database'); var batch = require('../batch'); module.exports = function (Groups) { - Groups.destroy = function (groupName, callback) { + Groups.destroy = function (groupNames, callback) { + if (!Array.isArray(groupNames)) { + groupNames = [groupNames]; + } + var groupObj; + var groupsData; async.waterfall([ function (next) { - Groups.getGroupsData([groupName], next); + Groups.getGroupsData(groupNames, next); }, - function (groupsData, next) { - if (!groupsData[0]) { + function (_groupsData, next) { + groupsData = _groupsData.filter(Boolean); + if (!groupsData.length) { return callback(); } + // backwards compatibility groupObj = groupsData[0]; async.parallel([ function (next) { - db.deleteAll([ - 'group:' + groupName, - 'group:' + groupName + ':members', - 'group:' + groupName + ':pending', - 'group:' + groupName + ':invited', - 'group:' + groupName + ':owners', - 'group:' + groupName + ':member:pids', - ], next); + var keys = []; + groupNames.forEach(function (groupName) { + keys.push('group:' + groupName, + 'group:' + groupName + ':members', + 'group:' + groupName + ':pending', + 'group:' + groupName + ':invited', + 'group:' + groupName + ':owners', + 'group:' + groupName + ':member:pids' + ); + }); + + db.deleteAll(keys, next); }, function (next) { - db.sortedSetsRemove([ + db.sortedSetRemove([ 'groups:createtime', 'groups:visible:createtime', 'groups:visible:memberCount', - ], groupName, next); + ], groupNames, next); }, function (next) { - db.sortedSetRemove('groups:visible:name', groupName.toLowerCase() + ':' + groupName, next); + var keys = groupNames.map(function (groupName) { + return groupName.toLowerCase() + ':' + groupName; + }); + db.sortedSetRemove('groups:visible:name', keys, next); }, function (next) { - db.deleteObjectField('groupslug:groupname', utils.slugify(groupName), next); + var fields = groupNames.map(function (groupName) { + return utils.slugify(groupName); + }); + db.deleteObjectFields('groupslug:groupname', fields, next); }, function (next) { - removeGroupFromOtherGroups(groupName, next); + removeGroupsFromOtherGroups(groupNames, next); }, ], function (err) { next(err); @@ -53,17 +70,18 @@ module.exports = function (Groups) { function (next) { Groups.resetCache(); plugins.fireHook('action:group.destroy', { group: groupObj }); + plugins.fireHook('action:groups.destroy', { groups: groupsData }); next(); }, ], callback); }; - function removeGroupFromOtherGroups(groupName, callback) { - batch.processSortedSet('groups:createtime', function (groupNames, next) { - var keys = groupNames.map(function (group) { + function removeGroupsFromOtherGroups(groupNames, callback) { + batch.processSortedSet('groups:createtime', function (otherGroups, next) { + var keys = otherGroups.map(function (group) { return 'group:' + group + ':members'; }); - db.sortedSetsRemove(keys, groupName, next); + db.sortedSetRemove(keys, groupNames, next); }, { batch: 500, }, callback); diff --git a/src/groups/membership.js b/src/groups/membership.js index 6217311139..da312cd1b7 100644 --- a/src/groups/membership.js +++ b/src/groups/membership.js @@ -147,8 +147,16 @@ module.exports = function (Groups) { ], callback); }; - Groups.rejectMembership = function (groupName, uid, callback) { - db.setsRemove(['group:' + groupName + ':pending', 'group:' + groupName + ':invited'], uid, callback); + Groups.rejectMembership = function (groupNames, uid, callback) { + if (!Array.isArray(groupNames)) { + groupNames = [groupNames]; + } + var sets = []; + groupNames.forEach(function (groupName) { + sets.push('group:' + groupName + ':pending', 'group:' + groupName + ':invited'); + }); + + db.setsRemove(sets, uid, callback); }; Groups.invite = function (groupName, uid, callback) { @@ -206,49 +214,70 @@ module.exports = function (Groups) { ], callback); } - Groups.leave = function (groupName, uid, callback) { + Groups.leave = function (groupNames, uid, callback) { callback = callback || function () {}; + if (!Array.isArray(groupNames)) { + groupNames = [groupNames]; + } + async.waterfall([ function (next) { async.parallel({ - isMember: async.apply(Groups.isMember, uid, groupName), - exists: async.apply(Groups.exists, groupName), + isMembers: async.apply(Groups.isMemberOfGroups, uid, groupNames), + exists: async.apply(Groups.exists, groupNames), }, next); }, function (result, next) { - if (!result.isMember || !result.exists) { + groupNames = groupNames.filter(function (groupName, index) { + return result.isMembers[index] && result.exists[index]; + }); + + if (!groupNames.length) { return callback(); } async.parallel([ - async.apply(db.sortedSetRemove, 'group:' + groupName + ':members', uid), - async.apply(db.setRemove, 'group:' + groupName + ':owners', uid), - async.apply(db.decrObjectField, 'group:' + groupName, 'memberCount'), + async.apply(db.sortedSetRemove, groupNames.map(groupName => 'group:' + groupName + ':members'), uid), + async.apply(db.setRemove, groupNames.map(groupName => 'group:' + groupName + ':owners'), uid), + async.apply(db.decrObjectField, groupNames.map(groupName => 'group:' + groupName), 'memberCount'), ], next); }, function (results, next) { - clearCache(uid, groupName); - Groups.getGroupFields(groupName, ['hidden', 'memberCount'], next); + clearCache(uid, groupNames); + Groups.getGroupsFields(groupNames, ['name', 'hidden', 'memberCount'], next); }, function (groupData, next) { if (!groupData) { return callback(); } - if (Groups.isPrivilegeGroup(groupName) && parseInt(groupData.memberCount, 10) === 0) { - Groups.destroy(groupName, next); - } else if (parseInt(groupData.hidden, 10) !== 1) { - db.sortedSetAdd('groups:visible:memberCount', groupData.memberCount, groupName, next); - } else { - next(); + var tasks = []; + + var emptyPrivilegeGroups = groupData.filter(function (groupData) { + return groupData && Groups.isPrivilegeGroup(groupData.name) && parseInt(groupData.memberCount, 10) === 0; + }); + if (emptyPrivilegeGroups.length) { + tasks.push(async.apply(Groups.destroy, emptyPrivilegeGroups)); } + + var visibleGroups = groupData.filter(function (groupData) { + return groupData && parseInt(groupData.hidden, 10) !== 1; + }); + if (visibleGroups.length) { + tasks.push(async.apply(db.sortedSetAdd, 'groups:visible:memberCount', visibleGroups.map(groupData => groupData.memberCount), visibleGroups.map(groupData => groupData.name))); + } + + async.parallel(tasks, function (err) { + next(err); + }); }, function (next) { - clearGroupTitleIfSet(groupName, uid, next); + clearGroupTitleIfSet(groupNames, uid, next); }, function (next) { plugins.fireHook('action:group.leave', { - groupName: groupName, + groupName: groupNames[0], + groupNames: groupNames, uid: uid, }); next(); @@ -256,8 +285,11 @@ module.exports = function (Groups) { ], callback); }; - function clearGroupTitleIfSet(groupName, uid, callback) { - if (groupName === 'registered-users' || Groups.isPrivilegeGroup(groupName)) { + function clearGroupTitleIfSet(groupNames, uid, callback) { + groupNames = groupNames.filter(function (groupName) { + return groupName !== 'registered-users' && !Groups.isPrivilegeGroup(groupName); + }); + if (!groupNames.length) { return callback(); } async.waterfall([ @@ -265,7 +297,7 @@ module.exports = function (Groups) { db.getObjectField('user:' + uid, 'groupTitle', next); }, function (groupTitle, next) { - if (groupTitle === groupName) { + if (groupNames.includes(groupTitle)) { db.deleteObjectField('user:' + uid, 'groupTitle', next); } else { next(); @@ -280,16 +312,14 @@ module.exports = function (Groups) { db.getSortedSetRange('groups:createtime', 0, -1, next); }, function (groups, next) { - async.each(groups, function (groupName, next) { - async.parallel([ - function (next) { - Groups.leave(groupName, uid, next); - }, - function (next) { - Groups.rejectMembership(groupName, uid, next); - }, - ], next); - }, next); + async.parallel([ + function (next) { + Groups.leave(groups, uid, next); + }, + function (next) { + Groups.rejectMembership(groups, uid, next); + }, + ], next); }, ], callback); }; @@ -326,13 +356,20 @@ module.exports = function (Groups) { cache.reset(); }); - function clearCache(uid, groupName) { - pubsub.publish('group:cache:del', { uid: uid, groupName: groupName }); - cache.del(uid + ':' + groupName); + function clearCache(uid, groupNames) { + if (!Array.isArray(groupNames)) { + groupNames = [groupNames]; + } + pubsub.publish('group:cache:del', { uid: uid, groupNames: groupNames }); + groupNames.forEach(function (groupName) { + cache.del(uid + ':' + groupName); + }); } pubsub.on('group:cache:del', function (data) { - cache.del(data.uid + ':' + data.groupName); + data.groupNames.forEach(function (groupName) { + cache.del(data.uid + ':' + groupName); + }); }); Groups.isMember = function (uid, groupName, callback) { diff --git a/test/database/hash.js b/test/database/hash.js index cd64acba56..88a61812a9 100644 --- a/test/database/hash.js +++ b/test/database/hash.js @@ -406,6 +406,15 @@ describe('Hash methods', function () { done(); }); }); + + it('should decrement multiple objects field by 1 and return an array of new values', function (done) { + db.decrObjectField(['testObject13', 'testObject14'], 'age', function (err, data) { + assert.ifError(err); + assert.equal(data[0], 97); + assert.equal(data[1], -1); + done(); + }); + }); }); describe('incrObjectFieldBy()', function () { diff --git a/test/database/sets.js b/test/database/sets.js index 75e81fb662..daef0a6d4c 100644 --- a/test/database/sets.js +++ b/test/database/sets.js @@ -203,6 +203,27 @@ describe('Set methods', function () { }); }); }); + + it('should remove multiple values from multiple keys', function (done) { + db.setAdd('multiSetTest1', ['one', 'two', 'three', 'four'], function (err) { + assert.ifError(err); + db.setAdd('multiSetTest2', ['three', 'four', 'five', 'six'], function (err) { + assert.ifError(err); + db.setRemove(['multiSetTest1', 'multiSetTest2'], ['three', 'four', 'five', 'doesnt exist'], function (err) { + assert.ifError(err); + db.getSetsMembers(['multiSetTest1', 'multiSetTest2'], function (err, members) { + assert.ifError(err); + assert.equal(members[0].length, 2); + assert.equal(members[1].length, 1); + assert(members[0].includes('one')); + assert(members[0].includes('two')); + assert(members[1].includes('six')); + done(); + }); + }); + }); + }); + }); }); describe('setsRemove()', function () { diff --git a/test/database/sorted.js b/test/database/sorted.js index 08799513b2..f07eee3008 100644 --- a/test/database/sorted.js +++ b/test/database/sorted.js @@ -642,6 +642,42 @@ describe('Sorted Set methods', function () { }); }); }); + + it('should remove multiple values from multiple keys', function (done) { + db.sortedSetAdd('multiTest1', [1, 2, 3, 4], ['one', 'two', 'three', 'four'], function (err) { + assert.ifError(err); + db.sortedSetAdd('multiTest2', [3, 4, 5, 6], ['three', 'four', 'five', 'six'], function (err) { + assert.ifError(err); + db.sortedSetRemove(['multiTest1', 'multiTest2'], ['two', 'three', 'four', 'five', 'doesnt exist'], function (err) { + assert.ifError(err); + db.getSortedSetsMembers(['multiTest1', 'multiTest2'], function (err, members) { + assert.ifError(err); + assert.equal(members[0].length, 1); + assert.equal(members[1].length, 1); + assert.deepEqual(members, [['one'], ['six']]); + done(); + }); + }); + }); + }); + }); + + it('should remove value from multiple keys', function (done) { + db.sortedSetAdd('multiTest3', [1, 2, 3, 4], ['one', 'two', 'three', 'four'], function (err) { + assert.ifError(err); + db.sortedSetAdd('multiTest4', [3, 4, 5, 6], ['three', 'four', 'five', 'six'], function (err) { + assert.ifError(err); + db.sortedSetRemove(['multiTest3', 'multiTest4'], 'three', function (err) { + assert.ifError(err); + db.getSortedSetsMembers(['multiTest3', 'multiTest4'], function (err, members) { + assert.ifError(err); + assert.deepEqual(members, [['one', 'two', 'four'], ['four', 'five', 'six']]); + done(); + }); + }); + }); + }); + }); }); describe('sortedSetsRemove()', function () {