diff --git a/src/batch.js b/src/batch.js new file mode 100644 index 0000000000..1c3c75627a --- /dev/null +++ b/src/batch.js @@ -0,0 +1,61 @@ + + +'use strict'; + +var async = require('async'), + db = require('./database'), + utils = require('../public/src/utils'); + +(function(Batch) { + + var DEFAULT_BATCH_SIZE = 100; + + Batch.processSortedSet = function(setKey, process, options, callback) { + if (typeof options === 'function') { + callback = options; + options = {}; + } + + callback = typeof callback === 'function' ? callback : function(){}; + options = options || {}; + + if (typeof process !== 'function') { + return callback(new Error('[[error:process-not-a-function]]')); + } + + // custom done condition + options.doneIf = typeof options.doneIf === 'function' ? options.doneIf : function(){}; + + var batch = options.batch || DEFAULT_BATCH_SIZE; + var start = 0; + var end = batch; + var done = false; + + async.whilst( + function() { + return !done; + }, + function(next) { + db.getSortedSetRange(setKey, start, end, function(err, ids) { + if (err) { + return next(err); + } + if (!ids.length || options.doneIf(start, end, ids)) { + done = true; + return next(); + } + process(err, ids, function(err) { + if (err) { + return next(err); + } + start += utils.isNumber(options.alwaysStartAt) ? options.alwaysStartAt : batch + 1; + end = start + batch; + next(); + }); + }); + }, + callback + ); + }; + +}(exports)); \ No newline at end of file diff --git a/src/categories/delete.js b/src/categories/delete.js index 1783ae77b1..c4dfc06c21 100644 --- a/src/categories/delete.js +++ b/src/categories/delete.js @@ -2,27 +2,23 @@ var async = require('async'), db = require('../database'), + batch = require('../batch'), threadTools = require('../threadTools'); module.exports = function(Categories) { Categories.purge = function(cid, callback) { - - Categories.getTopicIds(cid, 0, -1, function(err, tids) { + batch.processSortedSet('categories:' + cid + ':tid', function(err, tids, next) { if (err) { return callback(err); } async.eachLimit(tids, 10, function(tid, next) { threadTools.purge(tid, 0, next); - }, function(err) { - if (err) { - return callback(err); - } - - purgeCategory(cid, callback); - }); + }, next); + }, {alwaysStartAt: 0}, function(err) { + purgeCategory(cid, callback); }); }; @@ -32,13 +28,7 @@ module.exports = function(Categories) { db.sortedSetRemove('categories:cid', cid, next); }, function(next) { - db.delete('categories:' + cid + ':tid', next); - }, - function(next) { - db.delete('categories:recent_posts:cid:' + cid, next); - }, - function(next) { - db.delete('category:' + cid, next); + db.deleteAll(['categories:' + cid + ':tid', 'categories:recent_posts:cid:' + cid, 'category:' + cid], next); } ], callback); } diff --git a/src/posts/delete.js b/src/posts/delete.js index 3fe36abf8e..dfcff4dc35 100644 --- a/src/posts/delete.js +++ b/src/posts/delete.js @@ -40,17 +40,11 @@ module.exports = function(Posts) { return callback(err); } - async.parallel([ - function(next) { - db.sortedSetRemove('tid:' + postData.tid + ':posts', pid, next); - }, - function(next) { - db.sortedSetRemove('tid:' + postData.tid + ':posts:votes', pid, next); - }, - function(next) { - db.sortedSetRemove('uid:' + postData.uid + ':posts', pid, next); - } - ], function(err) { + db.sortedSetsRemove([ + 'tid:' + postData.tid + ':posts', + 'tid:' + postData.tid + ':posts:votes', + 'uid:' + postData.uid + ':posts' + ], pid, function(err) { if (err) { return callback(err); } @@ -101,7 +95,7 @@ module.exports = function(Posts) { } var sets = uids.map(function(uid) { - return 'uid:' + uid + ':favourites' + return 'uid:' + uid + ':favourites'; }); db.sortedSetsRemove(sets, pid, function(err) { @@ -143,14 +137,10 @@ module.exports = function(Posts) { db.sortedSetsRemove(downvoterSets, pid, next); }, function(next) { - db.delete('pid:' + pid + ':upvote', next); - }, - function(next) { - db.delete('pid:' + pid + ':downvote', next); + db.deleteAll(['pid:' + pid + ':upvote', 'pid:' + pid + ':downvote'], next); } ], callback); }); } - }; diff --git a/src/threadTools.js b/src/threadTools.js index 613a456443..1e69868e62 100644 --- a/src/threadTools.js +++ b/src/threadTools.js @@ -13,7 +13,8 @@ var winston = require('winston'), meta = require('./meta'), websockets = require('./socket.io'), events = require('./events'), - plugins = require('./plugins'); + plugins = require('./plugins'), + batch = require('./batch'); (function(ThreadTools) { @@ -71,26 +72,24 @@ var winston = require('winston'), } ThreadTools.purge = function(tid, uid, callback) { - async.parallel({ - topic: function(next) { - topics.getTopicFields(tid, ['cid'], next); - }, - pids: function(next) { - topics.getPids(tid, next); - } - }, function(err, results) { + batch.processSortedSet('tid:' + tid + ':posts', function(err, pids, next) { + async.eachLimit(pids, 10, posts.purge, next); + }, {alwaysStartAt: 0}, function(err) { if (err) { return callback(err); } - async.parallel([ - function(next) { - async.eachLimit(results.pids, 10, posts.purge, next); - }, - function(next) { - topics.purge(tid, next); + topics.getTopicField(tid, 'mainPid', function(err, mainPid) { + if (err) { + return callback(err); } - ], callback); + posts.purge(mainPid, function(err) { + if (err) { + return callback(err); + } + topics.purge(tid, callback); + }); + }); }); }; diff --git a/src/topics/delete.js b/src/topics/delete.js index 2a654b551c..06e289bdaf 100644 --- a/src/topics/delete.js +++ b/src/topics/delete.js @@ -44,10 +44,7 @@ module.exports = function(Topics) { Topics.removeRecent(tid, next); }, function(next) { - db.sortedSetRemove('topics:posts', tid, next); - }, - function(next) { - db.sortedSetRemove('topics:views', tid, next); + db.sortedSetsRemove(['topics:posts', 'topics:views'], tid, next); } ], function(err) { if (err) { @@ -90,22 +87,10 @@ module.exports = function(Topics) { Topics.purge = function(tid, callback) { async.parallel([ function(next) { - db.delete('tid:' + tid + ':followers', next); + db.deleteAll(['tid:' + tid + ':followers', 'tid:' + tid + ':read_by_uid'], next); }, function(next) { - db.delete('tid:' + tid + ':read_by_uid', next); - }, - function(next) { - db.sortedSetRemove('topics:tid', tid, next); - }, - function(next) { - Topics.removeRecent(tid, next); - }, - function(next) { - db.sortedSetRemove('topics:posts', tid, next); - }, - function(next) { - db.sortedSetRemove('topics:views', tid, next); + db.sortedSetsRemove(['topics:tid', 'topics:recent', 'topics:posts', 'topics:views'], tid, next); }, function(next) { deleteTopicFromCategoryAndUser(tid, next); @@ -128,14 +113,7 @@ module.exports = function(Topics) { return callback(err); } - async.parallel([ - function(next) { - db.sortedSetRemove('categories:' + topicData.cid + ':tid', tid, next); - }, - function(next) { - db.sortedSetRemove('uid:' + topicData.uid + ':topics', tid, next); - } - ], function(err) { + db.sortedSetsRemove(['categories:' + topicData.cid + ':tid', 'uid:' + topicData.uid + ':topics'], tid, function(err) { if (err) { return callback(err); }