Merge pull request #5467 from NodeBB/upgrades-refactor

Refactor upgrade scripts to use individual files in src/upgrades/ as source of schema changes

app.js

@@ -188,13 +188,17 @@ function upgrade() {
 	var meta = require('./src/meta');
 	var upgrade = require('./src/upgrade');
 	var build = require('./src/meta/build');
+	var tasks = [db.init, meta.configs.init, upgrade.run, build.buildAll];
 
-	async.series([
-		async.apply(db.init),
-		async.apply(meta.configs.init),
-		async.apply(upgrade.upgrade),
-		async.apply(build.buildAll),
-	], function (err) {
+	if (nconf.get('upgrade') !== true) {
+		// Likely an upgrade script name passed in
+		tasks[2] = async.apply(upgrade.runSingle, nconf.get('upgrade'));
+
+		// Skip build
+		tasks.pop();
+	}
+
+	async.series(tasks, function (err) {
 		if (err) {
 			winston.error(err.stack);
 			process.exit(1);

nodebb

@@ -29,7 +29,7 @@ if (args.dev) {
 
 function getRunningPid(callback) {
 	fs.readFile(__dirname + '/pidfile', {
-		encoding: 'utf-8'
+		encoding: 'utf-8',
 	}, function (err, pid) {
 		if (err) {
 			return callback(err);
@@ -38,7 +38,7 @@ function getRunningPid(callback) {
 		try {
 			process.kill(parseInt(pid, 10), 0);
 			callback(null, parseInt(pid, 10));
-		} catch(e) {
+		} catch (e) {
 			callback(e);
 		}
 	});
@@ -52,7 +52,7 @@ function getCurrentVersion(callback) {
 		try {
 			pkg = JSON.parse(pkg);
 			return callback(null, pkg.version);
-		} catch(err) {
+		} catch (err) {
 			return callback(err);
 		}
 	});
@@ -60,20 +60,21 @@ function getCurrentVersion(callback) {
 function fork(args) {
 	return cproc.fork('app.js', args, {
 		cwd: __dirname,
-		silent: false
+		silent: false,
 	});
 }
 function getInstalledPlugins(callback) {
 	async.parallel({
 		files: async.apply(fs.readdir, path.join(__dirname, 'node_modules')),
-		deps: async.apply(fs.readFile, path.join(__dirname, 'package.json'), { encoding: 'utf-8' })
+		deps: async.apply(fs.readFile, path.join(__dirname, 'package.json'), { encoding: 'utf-8' }),
 	}, function (err, payload) {
 		if (err) {
 			return callback(err);
 		}
 
-		var isNbbModule = /^nodebb-(?:plugin|theme|widget|rewards)-[\w\-]+$/,
-			moduleName, isGitRepo;
+		var isNbbModule = /^nodebb-(?:plugin|theme|widget|rewards)-[\w\-]+$/;
+		var moduleName;
+		var isGitRepo;
 
 		payload.files = payload.files.filter(function (file) {
 			return isNbbModule.test(file);
@@ -98,7 +99,7 @@ function getInstalledPlugins(callback) {
 			try {
 				fs.accessSync(path.join(__dirname, 'node_modules/' + moduleName, '.git'));
 				isGitRepo = true;
-			} catch(e) {
+			} catch (e) {
 				isGitRepo = false;
 			}
 
@@ -144,7 +145,7 @@ function checkPlugins(standalone, callback) {
 	async.waterfall([
 		async.apply(async.parallel, {
 			plugins: async.apply(getInstalledPlugins),
-			version: async.apply(getCurrentVersion)
+			version: async.apply(getCurrentVersion),
 		}),
 		function (payload, next) {
 			var toCheck = Object.keys(payload.plugins);
@@ -157,7 +158,7 @@ function checkPlugins(standalone, callback) {
 			request({
 				method: 'GET',
 				url: 'https://packages.nodebb.org/api/v1/suggest?version=' + payload.version + '&package[]=' + toCheck.join('&package[]='),
-				json: true
+				json: true,
 			}, function (err, res, body) {
 				if (err) {
 					process.stdout.write('error'.red + '\n'.reset);
@@ -169,7 +170,8 @@ function checkPlugins(standalone, callback) {
 					body = [body];
 				}
 
-				var current, suggested,
+				var current,
+					suggested,
 					upgradable = body.map(function (suggestObj) {
 						current = payload.plugins[suggestObj.package];
 						suggested = suggestObj.version;
@@ -178,16 +180,15 @@ function checkPlugins(standalone, callback) {
 							return {
 								name: suggestObj.package,
 								current: current,
-								suggested: suggested
+								suggested: suggested,
 							};
-						} else {
-							return null;
 						}
+						return null;
 					}).filter(Boolean);
 
 				next(null, upgradable);
 			});
-		}
+		},
 	], callback);
 }
 function upgradePlugins(callback) {
@@ -223,7 +224,7 @@ function upgradePlugins(callback) {
 		prompt.get({
 			name: 'upgrade',
 			description: 'Proceed with upgrade (y|n)?'.reset,
-			type: 'string'
+			type: 'string',
 		}, function (err, result) {
 			if (err) {
 				return callback(err);
@@ -280,7 +281,7 @@ var commands = {
 
 			// Spawn a new NodeBB process
 			cproc.fork(__dirname + '/loader.js', {
-				env: process.env
+				env: process.env,
 			});
 		},
 	},
@@ -320,7 +321,7 @@ var commands = {
 			process.stdout.write('\n\n'.reset);
 			cproc.spawn('tail', ['-F', './logs/output.log'], {
 				cwd: __dirname,
-				stdio: 'inherit'
+				stdio: 'inherit',
 			});
 		},
 	},
@@ -334,11 +335,11 @@ var commands = {
 
 			// Spawn a new NodeBB process
 			cproc.fork(__dirname + '/loader.js', {
-				env: process.env
+				env: process.env,
 			});
 			cproc.spawn('tail', ['-F', './logs/output.log'], {
 				cwd: __dirname,
-				stdio: 'inherit'
+				stdio: 'inherit',
 			});
 		},
 	},
@@ -348,13 +349,13 @@ var commands = {
 		handler: function () {
 			process.env.NODE_ENV = 'development';
 			cproc.fork(__dirname + '/loader.js', ['--no-daemon', '--no-silent'], {
-				env: process.env
+				env: process.env,
 			});
 		},
 	},
 	build: {
 		description: 'Compile static assets (CSS, Javascript, etc)',
-		usage: 'Usage: ' + './nodebb build'.yellow + ' [js,clientCSS,acpCSS,tpl,lang]'.red + '\n' + 
+		usage: 'Usage: ' + './nodebb build'.yellow + ' [js,clientCSS,acpCSS,tpl,lang]'.red + '\n' +
 			'    e.g. ' + './nodebb build js,tpl'.yellow + '\tbuilds JS and templates\n' +
 			'         ' + './nodebb build'.yellow + '\t\tbuilds all targets\n',
 		handler: function () {
@@ -406,6 +407,18 @@ var commands = {
 		description: 'Run NodeBB upgrade scripts, ensure packages are up-to-date',
 		usage: 'Usage: ' + './nodebb upgrade'.yellow,
 		handler: function () {
+			if (process.argv[3]) {
+				process.stdout.write('\nUpdating NodeBB data store schema...\n'.yellow);
+				var arr = ['--upgrade'].concat(process.argv.slice(3));
+				var upgradeProc = fork(arr);
+
+				return upgradeProc.on('close', function (err) {
+					if (err) {
+						process.stdout.write('\nError'.red + ': ' + err.message + '\n');
+					}
+				});
+			}
+
 			async.series([
 				function (next) {
 					process.stdout.write('1. '.bold + 'Bringing base dependencies up to date... '.yellow);
@@ -422,7 +435,7 @@ var commands = {
 					var upgradeProc = fork(arr);
 
 					upgradeProc.on('close', next);
-				}
+				},
 			], function (err) {
 				if (err) {
 					process.stdout.write('\nError'.red + ': ' + err.message + '\n');
@@ -430,7 +443,7 @@ var commands = {
 					var message = 'NodeBB Upgrade Complete!';
 					// some consoles will return undefined/zero columns, so just use 2 spaces in upgrade script if we can't get our column count
 					var columns = process.stdout.columns;
-					var spaces = columns ? new Array(Math.floor(columns / 2) - (message.length / 2) + 1).join(' ') : "  ";
+					var spaces = columns ? new Array(Math.floor(columns / 2) - (message.length / 2) + 1).join(' ') : '  ';
 
 					process.stdout.write('OK\n'.green);
 					process.stdout.write('\n' + spaces + message.green.bold + '\n\n'.reset);

src/install.js

@@ -507,12 +507,11 @@ install.setup = function (callback) {
 		setCopyrightWidget,
 		function (next) {
 			var upgrade = require('./upgrade');
-			upgrade.check(function (err, uptodate) {
-				if (err) {
+			upgrade.check(function (err) {
+				if (err && err.message === 'schema-out-of-date') {
+					upgrade.run(next);
+				} else if (err) {
 					return next(err);
-				}
-				if (!uptodate) {
-					upgrade.upgrade(next);
 				} else {
 					next();
 				}

src/start.js

@@ -58,16 +58,16 @@ start.start = function () {
 		if (err) {
 			switch (err.message) {
 			case 'schema-out-of-date':
-				winston.warn('Your NodeBB schema is out-of-date. Please run the following command to bring your dataset up to spec:');
-				winston.warn('    ./nodebb upgrade');
+				winston.error('Your NodeBB schema is out-of-date. Please run the following command to bring your dataset up to spec:');
+				winston.error('    ./nodebb upgrade');
 				break;
 			case 'dependencies-out-of-date':
-				winston.warn('One or more of NodeBB\'s dependent packages are out-of-date. Please run the following command to update them:');
-				winston.warn('    ./nodebb upgrade');
+				winston.error('One or more of NodeBB\'s dependent packages are out-of-date. Please run the following command to update them:');
+				winston.error('    ./nodebb upgrade');
 				break;
 			case 'dependencies-missing':
-				winston.warn('One or more of NodeBB\'s dependent packages are missing. Please run the following command to update them:');
-				winston.warn('    ./nodebb upgrade');
+				winston.error('One or more of NodeBB\'s dependent packages are missing. Please run the following command to update them:');
+				winston.error('    ./nodebb upgrade');
 				break;
 			default:
 				winston.error(err);

src/upgrade.js

@@ -1,584 +1,181 @@
+/* jslint node: true */
+
 'use strict';
 
+var async = require('async');
+var path = require('path');
+var semver = require('semver');
 
 var db = require('./database');
-var async = require('async');
-var winston = require('winston');
+var utils = require('../public/src/utils');
 
-var Upgrade = {};
+/*
+ * Need to write an upgrade script for NodeBB? Cool.
+ *
+ * 1. Copy TEMPLATE to a file name of your choice. Try to be succinct.
+ * 2. Open up that file and change the user-friendly name (can be longer/more descriptive than the file name)
+ *    and timestamp
+ * 3. Add your script under the "method" property
+ * 4. Append your filename to the array below for the next NodeBB version.
+ */
 
-var minSchemaDate = Date.UTC(2016, 8, 7);		// This value gets updated every new MAJOR version
-var schemaDate;
-var thisSchemaDate;
+var Upgrade = {
+	available: [
+		{
+			version: '1.0.0',
+			upgrades: ['chat_upgrade', 'chat_room_hashes', 'theme_to_active_plugins', 'user_best_posts', 'users_notvalidated', 'global_moderators', 'social_post_sharing'],
+		},
+		{
+			version: '1.1.0',
+			upgrades: ['group_title_update', 'user_post_count_per_tid', 'dismiss_flags_from_deleted_topics', 'assign_topic_read_privilege', 'separate_upvote_downvote'],
+		},
+		{
+			version: '1.1.1',
+			upgrades: ['upload_privileges', 'remove_negative_best_posts'],
+		},
+		{
+			version: '1.2.0',
+			upgrades: ['category_recent_tids', 'edit_delete_deletetopic_privileges'],
+		},
+		{
+			version: '1.3.0',
+			upgrades: ['favourites_to_bookmarks', 'sorted_sets_for_post_replies'],
+		},
+		{
+			version: '1.4.0',
+			upgrades: ['global_and_user_language_keys', 'sorted_set_for_pinned_topics'],
+		},
+		{
+			version: 'master',	// rename this to whenever the next NodeBB version is (non-breaking)
+			upgrades: ['sound_settings', 'config_urls_update'],
+		},
+		{
+			version: 'develop',	// rename this to whatever the next NodeBB version is (breaking)
+			upgrades: ['flags_refactor', 'post_votes_zset'],
+		},
+	],
+};
 
-// IMPORTANT: REMEMBER TO UPDATE VALUE OF latestSchema
-var latestSchema = Date.UTC(2017, 1, 28);
+Upgrade.getAll = function (callback) {
+	async.waterfall([
+		async.apply(utils.walk, path.join(__dirname, './upgrades')),
+		function (files, next) {
+			// Sort the upgrade scripts based on version
+			var versionA;
+			var versionB;
+			setImmediate(next, null, files.filter(function (file) {
+				return path.basename(file) !== 'TEMPLATE';
+			}).sort(function (a, b) {
+				versionA = path.dirname(a).split('/').pop();
+				versionB = path.dirname(b).split('/').pop();
+
+				return semver.compare(versionA, versionB);
+			}));
+		},
+	], callback);
+};
 
 Upgrade.check = function (callback) {
-	db.get('schemaDate', function (err, value) {
+	// Throw 'schema-out-of-date' if not all upgrade scripts have run
+	async.waterfall([
+		async.apply(Upgrade.getAll),
+		function (files, next) {
+			db.getSortedSetRange('schemaLog', 0, -1, function (err, executed) {
+				if (err) {
+					return callback(err);
+				}
+
+				var remainder = files.filter(function (name) {
+					return executed.indexOf(path.basename(name, '.js')) === -1;
+				});
+
+				next(remainder.length > 1 ? new Error('schema-out-of-date') : null);
+			});
+		},
+	], callback);
+	// var all = Upgrade.available.reduce(function (memo, current) {
+	// 	memo = memo.concat(current.upgrades);
+	// 	return memo;
+	// }, []);
+};
+
+Upgrade.run = function (callback) {
+	process.stdout.write('\nParsing upgrade scripts... ');
+	var queue = [];
+	var skipped = 0;
+
+	async.parallel({
+		// Retrieve list of upgrades that have already been run
+		completed: async.apply(db.getSortedSetRange, 'schemaLog', 0, -1),
+		// ... and those available to be run
+		available: Upgrade.getAll,
+	}, function (err, data) {
 		if (err) {
 			return callback(err);
 		}
 
-		if (!value) {
-			db.set('schemaDate', latestSchema, function (err) {
-				if (err) {
-					return callback(err);
-				}
-				callback(null);
-			});
-			return;
-		}
+		queue = data.available.reduce(function (memo, cur) {
+			if (data.completed.indexOf(path.basename(cur, '.js')) === -1) {
+				memo.push(cur);
+			} else {
+				skipped += 1;
+			}
 
-		var schema_ok = parseInt(value, 10) >= latestSchema;
-		callback(!schema_ok ? new Error('schema-out-of-date') : null);
+			return memo;
+		}, queue);
+
+		Upgrade.process(queue, skipped, callback);
 	});
 };
 
-Upgrade.update = function (schemaDate, callback) {
-	db.set('schemaDate', schemaDate, callback);
+Upgrade.runSingle = function (query, callback) {
+	process.stdout.write('\nParsing upgrade scripts... ');
+
+	async.waterfall([
+		async.apply(utils.walk, path.join(__dirname, './upgrades')),
+		function (files, next) {
+			next(null, files.filter(function (file) {
+				return path.basename(file, '.js') === query;
+			}));
+		},
+	], function (err, files) {
+		if (err) {
+			return callback(err);
+		}
+
+		Upgrade.process(files, 0, callback);
+	});
 };
 
-Upgrade.upgrade = function (callback) {
-	var updatesMade = false;
+Upgrade.process = function (files, skipCount, callback) {
+	process.stdout.write('OK'.green + ' | '.reset + String(files.length).cyan + ' script(s) found'.cyan + (skipCount > 0 ? ', '.cyan + String(skipCount).cyan + ' skipped'.cyan : '') + '\n'.reset);
 
-	winston.info('Beginning database schema update');
+	async.eachSeries(files, function (file, next) {
+		var scriptExport = require(file);
+		var date = new Date(scriptExport.timestamp);
 
-	async.series([
-		function (next) {
-			// Prepare for upgrade & check to make sure the upgrade is possible
-			db.get('schemaDate', function (err, value) {
-				if (err) {
-					return next(err);
-				}
+		process.stdout.write('  → '.white + String('[' + [date.getUTCFullYear(), date.getUTCMonth() + 1, date.getUTCDate()].join('/') + '] ').gray + String(scriptExport.name).reset + '... ');
 
-				if (!value) {
-					db.set('schemaDate', latestSchema, function () {
-						next();
-					});
-					schemaDate = latestSchema;
-				} else {
-					schemaDate = parseInt(value, 10);
-				}
-
-				if (schemaDate >= minSchemaDate) {
-					next();
-				} else {
-					next(new Error('upgrade-not-possible'));
-				}
-			});
-		},
-		function (next) {
-			thisSchemaDate = Date.UTC(2016, 8, 22);
-
-			if (schemaDate < thisSchemaDate) {
-				updatesMade = true;
-				winston.info('[2016/09/22] Setting category recent tids');
-
-
-				db.getSortedSetRange('categories:cid', 0, -1, function (err, cids) {
-					if (err) {
-						return next(err);
-					}
-
-					async.eachSeries(cids, function (cid, next) {
-						db.getSortedSetRevRange('cid:' + cid + ':pids', 0, 0, function (err, pid) {
-							if (err || !pid) {
-								return next(err);
-							}
-							db.getObjectFields('post:' + pid, ['tid', 'timestamp'], function (err, postData) {
-								if (err || !postData || !postData.tid) {
-									return next(err);
-								}
-								db.sortedSetAdd('cid:' + cid + ':recent_tids', postData.timestamp, postData.tid, next);
-							});
-						});
-					}, function (err) {
-						if (err) {
-							return next(err);
-						}
-
-						winston.info('[2016/09/22] Setting category recent tids - done');
-						Upgrade.update(thisSchemaDate, next);
-					});
-				});
-			} else {
-				winston.info('[2016/09/22] Setting category recent tids - skipped!');
-				next();
-			}
-		},
-		function (next) {
-			function upgradePosts(next) {
-				var batch = require('./batch');
-
-				batch.processSortedSet('posts:pid', function (ids, next) {
-					async.each(ids, function (id, next) {
-						console.log('processing pid ' + id);
-						async.waterfall([
-							function (next) {
-								db.rename('pid:' + id + ':users_favourited', 'pid:' + id + ':users_bookmarked', next);
-							},
-							function (next) {
-								db.getObjectField('post:' + id, 'reputation', next);
-							},
-							function (reputation, next) {
-								if (parseInt(reputation, 10)) {
-									db.setObjectField('post:' + id, 'bookmarks', reputation, next);
-								} else {
-									next();
-								}
-							},
-							function (next) {
-								db.deleteObjectField('post:' + id, 'reputation', next);
-							},
-						], next);
-					}, next);
-				}, {}, next);
+		// Do the upgrade...
+		scriptExport.method(function (err) {
+			if (err) {
+				process.stdout.write('error\n'.red);
+				return next(err);
 			}
 
-			function upgradeUsers(next) {
-				var batch = require('./batch');
+			// Record success in schemaLog
+			db.sortedSetAdd('schemaLog', Date.now(), path.basename(file, '.js'));
 
-				batch.processSortedSet('users:joindate', function (ids, next) {
-					async.each(ids, function (id, next) {
-						console.log('processing uid ' + id);
-						db.rename('uid:' + id + ':favourites', 'uid:' + id + ':bookmarks', next);
-					}, next);
-				}, {}, next);
-			}
-
-			thisSchemaDate = Date.UTC(2016, 9, 8);
-
-			if (schemaDate < thisSchemaDate) {
-				updatesMade = true;
-				winston.info('[2016/10/8] favourite -> bookmark refactor');
-				async.series([upgradePosts, upgradeUsers], function (err) {
-					if (err) {
-						return next(err);
-					}
-					winston.info('[2016/08/05] favourite- bookmark refactor done!');
-					Upgrade.update(thisSchemaDate, next);
-				});
-			} else {
-				winston.info('[2016/10/8] favourite -> bookmark refactor - skipped!');
-				next();
-			}
-		},
-		function (next) {
-			thisSchemaDate = Date.UTC(2016, 9, 14);
-
-			if (schemaDate < thisSchemaDate) {
-				updatesMade = true;
-				winston.info('[2016/10/14] Creating sorted sets for post replies');
-
-				var posts = require('./posts');
-				var batch = require('./batch');
-				batch.processSortedSet('posts:pid', function (ids, next) {
-					posts.getPostsFields(ids, ['pid', 'toPid', 'timestamp'], function (err, data) {
-						if (err) {
-							return next(err);
-						}
-
-						async.eachSeries(data, function (postData, next) {
-							if (!parseInt(postData.toPid, 10)) {
-								return next(null);
-							}
-							console.log('processing pid: ' + postData.pid + ' toPid: ' + postData.toPid);
-							async.parallel([
-								async.apply(db.sortedSetAdd, 'pid:' + postData.toPid + ':replies', postData.timestamp, postData.pid),
-								async.apply(db.incrObjectField, 'post:' + postData.toPid, 'replies'),
-							], next);
-						}, next);
-					});
-				}, function (err) {
-					if (err) {
-						return next(err);
-					}
-
-					winston.info('[2016/10/14] Creating sorted sets for post replies - done');
-					Upgrade.update(thisSchemaDate, next);
-				});
-			} else {
-				winston.info('[2016/10/14] Creating sorted sets for post replies - skipped!');
-				next();
-			}
-		},
-		function (next) {
-			thisSchemaDate = Date.UTC(2016, 10, 22);
-
-			if (schemaDate < thisSchemaDate) {
-				updatesMade = true;
-				winston.info('[2016/11/22] Update global and user language keys');
-
-				var user = require('./user');
-				var meta = require('./meta');
-				var batch = require('./batch');
-				var newLanguage;
-				var i = 0;
-				var j = 0;
-				async.parallel([
-					function (next) {
-						meta.configs.get('defaultLang', function (err, defaultLang) {
-							if (err) {
-								return next(err);
-							}
-
-							if (!defaultLang) {
-								return setImmediate(next);
-							}
-
-							newLanguage = defaultLang.replace('_', '-').replace('@', '-x-');
-							if (newLanguage !== defaultLang) {
-								meta.configs.set('defaultLang', newLanguage, next);
-							} else {
-								setImmediate(next);
-							}
-						});
-					},
-					function (next) {
-						batch.processSortedSet('users:joindate', function (ids, next) {
-							async.each(ids, function (uid, next) {
-								async.waterfall([
-									async.apply(db.getObjectField, 'user:' + uid + ':settings', 'userLang'),
-									function (language, next) {
-										i += 1;
-										if (!language) {
-											return setImmediate(next);
-										}
-
-										newLanguage = language.replace('_', '-').replace('@', '-x-');
-										if (newLanguage !== language) {
-											j += 1;
-											user.setSetting(uid, 'userLang', newLanguage, next);
-										} else {
-											setImmediate(next);
-										}
-									},
-								], next);
-							}, next);
-						}, next);
-					},
-				], function (err) {
-					if (err) {
-						return next(err);
-					}
-
-					winston.info('[2016/11/22] Update global and user language keys - done (' + i + ' processed, ' + j + ' updated)');
-					Upgrade.update(thisSchemaDate, next);
-				});
-			} else {
-				winston.info('[2016/11/22] Update global and user language keys - skipped!');
-				next();
-			}
-		},
-		function (next) {
-			thisSchemaDate = Date.UTC(2016, 10, 25);
-
-			if (schemaDate < thisSchemaDate) {
-				updatesMade = true;
-				winston.info('[2016/11/25] Creating sorted sets for pinned topics');
-
-				var topics = require('./topics');
-				var batch = require('./batch');
-				batch.processSortedSet('topics:tid', function (ids, next) {
-					topics.getTopicsFields(ids, ['tid', 'cid', 'pinned', 'lastposttime'], function (err, data) {
-						if (err) {
-							return next(err);
-						}
-
-						data = data.filter(function (topicData) {
-							return parseInt(topicData.pinned, 10) === 1;
-						});
-
-						async.eachSeries(data, function (topicData, next) {
-							console.log('processing tid: ' + topicData.tid);
-
-							async.parallel([
-								async.apply(db.sortedSetAdd, 'cid:' + topicData.cid + ':tids:pinned', Date.now(), topicData.tid),
-								async.apply(db.sortedSetRemove, 'cid:' + topicData.cid + ':tids', topicData.tid),
-								async.apply(db.sortedSetRemove, 'cid:' + topicData.cid + ':tids:posts', topicData.tid),
-							], next);
-						}, next);
-					});
-				}, function (err) {
-					if (err) {
-						return next(err);
-					}
-
-					winston.info('[2016/11/25] Creating sorted sets for pinned topics - done');
-					Upgrade.update(thisSchemaDate, next);
-				});
-			} else {
-				winston.info('[2016/11/25] Creating sorted sets for pinned topics - skipped!');
-				next();
-			}
-		},
-		function (next) {
-			thisSchemaDate = Date.UTC(2016, 11, 7);
-
-			if (schemaDate < thisSchemaDate) {
-				updatesMade = true;
-				winston.info('[2016/12/07] Migrating flags to new schema (#5232)');
-
-				var batch = require('./batch');
-				var posts = require('./posts');
-				var flags = require('./flags');
-
-				batch.processSortedSet('posts:pid', function (ids, next) {
-					posts.getPostsByPids(ids, 1, function (err, posts) {
-						if (err) {
-							return next(err);
-						}
-
-						posts = posts.filter(function (post) {
-							return post.hasOwnProperty('flags');
-						});
-
-						async.each(posts, function (post, next) {
-							async.parallel({
-								uids: async.apply(db.getSortedSetRangeWithScores, 'pid:' + post.pid + ':flag:uids', 0, -1),
-								reasons: async.apply(db.getSortedSetRange, 'pid:' + post.pid + ':flag:uid:reason', 0, -1),
-							}, function (err, data) {
-								if (err) {
-									return next(err);
-								}
-
-								// Adding in another check here in case a post was improperly dismissed (flag count > 1 but no flags in db)
-								if (!data.uids.length || !data.reasons.length) {
-									return setImmediate(next);
-								}
-
-								// Just take the first entry
-								var datetime = data.uids[0].score;
-								var reason = data.reasons[0].split(':')[1];
-								var flagObj;
-
-								async.waterfall([
-									async.apply(flags.create, 'post', post.pid, data.uids[0].value, reason, datetime),
-									function (_flagObj, next) {
-										flagObj = _flagObj;
-										if (post['flag:state'] || post['flag:assignee']) {
-											flags.update(flagObj.flagId, 1, {
-												state: post['flag:state'],
-												assignee: post['flag:assignee'],
-												datetime: datetime,
-											}, next);
-										} else {
-											setImmediate(next);
-										}
-									},
-									function (next) {
-										if (post.hasOwnProperty('flag:notes') && post['flag:notes'].length) {
-											try {
-												var history = JSON.parse(post['flag:history']);
-												history = history.filter(function (event) {
-													return event.type === 'notes';
-												})[0];
-
-												flags.appendNote(flagObj.flagId, history.uid, post['flag:notes'], history.timestamp, next);
-											} catch (e) {
-												next(e);
-											}
-										} else {
-											setImmediate(next);
-										}
-									},
-								], function (err) {
-									if (err && err.message === '[[error:already-flagged]]') {
-										// Already flagged, no need to parse, but not an error
-										next();
-									} else {
-										next(err);
-									}
-								});
-							});
-						}, next);
-					});
-				}, function (err) {
-					if (err) {
-						return next(err);
-					}
-
-					winston.info('[2016/12/07] Migrating flags to new schema (#5232) - done');
-					Upgrade.update(thisSchemaDate, next);
-				});
-			} else {
-				winston.info('[2016/12/07] Migrating flags to new schema (#5232) - skipped!');
-				next();
-			}
-		},
-		function (next) {
-			thisSchemaDate = Date.UTC(2017, 1, 28);
-			var schemaName = '[2017/2/28] Update urls in config to `/assets`';
-
-			if (schemaDate < thisSchemaDate) {
-				updatesMade = true;
-				winston.info(schemaName);
-				async.waterfall([
-					function (cb) {
-						db.getObject('config', cb);
-					},
-					function (config, cb) {
-						if (!config) {
-							return cb();
-						}
-
-						var keys = ['brand:favicon', 'brand:touchicon', 'og:image', 'brand:logo:url', 'defaultAvatar', 'profile:defaultCovers'];
-
-						keys.forEach(function (key) {
-							var oldValue = config[key];
-
-							if (!oldValue || typeof oldValue !== 'string') {
-								return;
-							}
-
-							config[key] = oldValue.replace(/(?:\/assets)?\/(images|uploads)\//g, '/assets/$1/');
-						});
-
-						db.setObject('config', config, cb);
-					},
-					function (next) {
-						winston.info(schemaName + ' - done');
-						Upgrade.update(thisSchemaDate, next);
-					},
-				], next);
-			} else {
-				winston.info(schemaName + ' - skipped!');
-				next();
-			}
-		},
-		function (next) {
-			thisSchemaDate = Date.UTC(2017, 1, 25);
-			var schemaName = '[2017/2/25] Update global and user sound settings';
-
-			if (schemaDate < thisSchemaDate) {
-				updatesMade = true;
-				winston.verbose(schemaName);
-
-				var meta = require('./meta');
-				var batch = require('./batch');
-
-				var map = {
-					'notification.mp3': 'Default | Deedle-dum',
-					'waterdrop-high.mp3': 'Default | Water drop (high)',
-					'waterdrop-low.mp3': 'Default | Water drop (low)',
-				};
-
-				async.parallel([
-					function (cb) {
-						var keys = ['chat-incoming', 'chat-outgoing', 'notification'];
-
-						db.getObject('settings:sounds', function (err, settings) {
-							if (err || !settings) {
-								return cb(err);
-							}
-
-							keys.forEach(function (key) {
-								if (settings[key] && settings[key].indexOf(' | ') === -1) {
-									settings[key] = map[settings[key]] || '';
-								}
-							});
-
-							meta.configs.setMultiple(settings, cb);
-						});
-					},
-					function (cb) {
-						var keys = ['notificationSound', 'incomingChatSound', 'outgoingChatSound'];
-
-						batch.processSortedSet('users:joindate', function (ids, next) {
-							async.each(ids, function (uid, next) {
-								db.getObject('user:' + uid + ':settings', function (err, settings) {
-									if (err || !settings) {
-										return next(err);
-									}
-									var newSettings = {};
-									keys.forEach(function (key) {
-										if (settings[key] && settings[key].indexOf(' | ') === -1) {
-											newSettings[key] = map[settings[key]] || '';
-										}
-									});
-
-									if (Object.keys(newSettings).length) {
-										db.setObject('user:' + uid + ':settings', newSettings, next);
-									} else {
-										setImmediate(next);
-									}
-								});
-							}, next);
-						}, cb);
-					},
-				], function (err) {
-					if (err) {
-						return next(err);
-					}
-					winston.info(schemaName + ' - done');
-					Upgrade.update(thisSchemaDate, next);
-				});
-			} else {
-				winston.info(schemaName + ' - skipped!');
-				next();
-			}
-		},
-		function (next) {
-			thisSchemaDate = Date.UTC(2017, 1, 27);
-			var schemaName = '[2017/2/27] New sorted set posts:votes';
-
-			if (schemaDate < thisSchemaDate) {
-				updatesMade = true;
-				winston.verbose(schemaName);
-
-				require('./batch').processSortedSet('posts:pid', function (pids, next) {
-					async.each(pids, function (pid, next) {
-						db.getObjectFields('post:' + pid, ['upvotes', 'downvotes'], function (err, postData) {
-							if (err || !postData) {
-								return next(err);
-							}
-
-							var votes = parseInt(postData.upvotes || 0, 10) - parseInt(postData.downvotes || 0, 10);
-							db.sortedSetAdd('posts:votes', votes, pid, next);
-						});
-					}, next);
-				}, {}, function (err) {
-					if (err) {
-						return next(err);
-					}
-
-					winston.info(schemaName + ' - done');
-					Upgrade.update(thisSchemaDate, next);
-				});
-			} else {
-				winston.info(schemaName + ' - skipped!');
-				next();
-			}
-		},
-		// Add new schema updates here
-		// IMPORTANT: REMEMBER TO UPDATE VALUE OF latestSchema IN LINE 24!!!
-	], function (err) {
-		if (!err) {
-			if (updatesMade) {
-				winston.info('[upgrade] Schema update complete!');
-			} else {
-				winston.info('[upgrade] Schema already up to date!');
-			}
-		} else {
-			switch (err.message) {
-			case 'upgrade-not-possible':
-				winston.error('[upgrade] NodeBB upgrade could not complete, as your database schema is too far out of date.');
-				winston.error('[upgrade]   Please ensure that you did not skip any minor version upgrades.');
-				winston.error('[upgrade]   (e.g. v0.1.x directly to v0.3.x)');
-				break;
-
-			default:
-				winston.error('[upgrade] Errors were encountered while updating the NodeBB schema: ' + err.message);
-				break;
-			}
+			process.stdout.write('OK\n'.green);
+			next();
+		});
+	}, function (err) {
+		if (err) {
+			return callback(err);
 		}
 
-		if (typeof callback === 'function') {
-			callback(err);
-		} else {
-			process.exit();
-		}
+		process.stdout.write('Upgrade complete!\n\n'.green);
+		callback();
 	});
 };
 

src/upgrades/1.0.0/chat_room_hashes.js

@@ -0,0 +1,41 @@
+/* jslint node: true */
+
+'use strict';
+
+var db = require('../../database');
+
+var async = require('async');
+
+module.exports = {
+	name: 'Chat room hashes',
+	timestamp: Date.UTC(2015, 11, 23),
+	method: function (callback) {
+		db.getObjectField('global', 'nextChatRoomId', function (err, nextChatRoomId) {
+			if (err) {
+				return callback(err);
+			}
+			var currentChatRoomId = 1;
+			async.whilst(function () {
+				return currentChatRoomId <= nextChatRoomId;
+			}, function (next) {
+				db.getSortedSetRange('chat:room:' + currentChatRoomId + ':uids', 0, 0, function (err, uids) {
+					if (err) {
+						return next(err);
+					}
+					if (!Array.isArray(uids) || !uids.length || !uids[0]) {
+						currentChatRoomId += 1;
+						return next();
+					}
+
+					db.setObject('chat:room:' + currentChatRoomId, { owner: uids[0], roomId: currentChatRoomId }, function (err) {
+						if (err) {
+							return next(err);
+						}
+						currentChatRoomId += 1;
+						next();
+					});
+				});
+			}, callback);
+		});
+	},
+};

src/upgrades/1.0.0/chat_upgrade.js

@@ -0,0 +1,87 @@
+/* jslint node: true */
+
+'use strict';
+
+var db = require('../../database');
+
+var async = require('async');
+var winston = require('winston');
+
+module.exports = {
+	name: 'Upgrading chats',
+	timestamp: Date.UTC(2015, 11, 15),
+	method: function (callback) {
+		db.getObjectFields('global', ['nextMid', 'nextChatRoomId'], function (err, globalData) {
+			if (err) {
+				return callback(err);
+			}
+
+			var rooms = {};
+			var roomId = globalData.nextChatRoomId || 1;
+			var currentMid = 1;
+
+			async.whilst(function () {
+				return currentMid <= globalData.nextMid;
+			}, function (next) {
+				db.getObject('message:' + currentMid, function (err, message) {
+					var msgTime;
+
+					function addMessageToUids(roomId, callback) {
+						async.parallel([
+							function (next) {
+								db.sortedSetAdd('uid:' + message.fromuid + ':chat:room:' + roomId + ':mids', msgTime, currentMid, next);
+							},
+							function (next) {
+								db.sortedSetAdd('uid:' + message.touid + ':chat:room:' + roomId + ':mids', msgTime, currentMid, next);
+							},
+						], callback);
+					}
+
+					if (err || !message) {
+						winston.verbose('skipping chat message ', currentMid);
+						currentMid += 1;
+						return next(err);
+					}
+
+					var pairID = [parseInt(message.fromuid, 10), parseInt(message.touid, 10)].sort().join(':');
+					msgTime = parseInt(message.timestamp, 10);
+
+					if (rooms[pairID]) {
+						winston.verbose('adding message ' + currentMid + ' to existing roomID ' + roomId);
+						addMessageToUids(rooms[pairID], function (err) {
+							if (err) {
+								return next(err);
+							}
+							currentMid += 1;
+							next();
+						});
+					} else {
+						winston.verbose('adding message ' + currentMid + ' to new roomID ' + roomId);
+						async.parallel([
+							function (next) {
+								db.sortedSetAdd('uid:' + message.fromuid + ':chat:rooms', msgTime, roomId, next);
+							},
+							function (next) {
+								db.sortedSetAdd('uid:' + message.touid + ':chat:rooms', msgTime, roomId, next);
+							},
+							function (next) {
+								db.sortedSetAdd('chat:room:' + roomId + ':uids', [msgTime, msgTime + 1], [message.fromuid, message.touid], next);
+							},
+							function (next) {
+								addMessageToUids(roomId, next);
+							},
+						], function (err) {
+							if (err) {
+								return next(err);
+							}
+							rooms[pairID] = roomId;
+							roomId += 1;
+							currentMid += 1;
+							db.setObjectField('global', 'nextChatRoomId', roomId, next);
+						});
+					}
+				});
+			}, callback);
+		});
+	},
+};

src/upgrades/1.0.0/global_moderators.js

@@ -0,0 +1,34 @@
+/* jslint node: true */
+
+'use strict';
+
+var async = require('async');
+
+module.exports = {
+	name: 'Creating Global moderators group',
+	timestamp: Date.UTC(2016, 0, 23),
+	method: function (callback) {
+		var groups = require('../../groups');
+		async.waterfall([
+			function (next) {
+				groups.exists('Global Moderators', next);
+			},
+			function (exists, next) {
+				if (exists) {
+					return next(null, null);
+				}
+				groups.create({
+					name: 'Global Moderators',
+					userTitle: 'Global Moderator',
+					description: 'Forum wide moderators',
+					hidden: 0,
+					private: 1,
+					disableJoinRequests: 1,
+				}, next);
+			},
+			function (groupData, next) {
+				groups.show('Global Moderators', next);
+			},
+		], callback);
+	},
+};

src/upgrades/1.0.0/social_post_sharing.js

@@ -0,0 +1,23 @@
+/* jslint node: true */
+
+'use strict';
+
+var db = require('../../database');
+
+var async = require('async');
+
+module.exports = {
+	name: 'Social: Post Sharing',
+	timestamp: Date.UTC(2016, 1, 25),
+	method: function (callback) {
+		var social = require('../../social');
+		async.parallel([
+			function (next) {
+				social.setActivePostSharingNetworks(['facebook', 'google', 'twitter'], next);
+			},
+			function (next) {
+				db.deleteObjectField('config', 'disableSocialButtons', next);
+			},
+		], callback);
+	},
+};

src/upgrades/1.0.0/theme_to_active_plugins.js

@@ -0,0 +1,18 @@
+/* jslint node: true */
+
+'use strict';
+
+var db = require('../../database');
+
+var async = require('async');
+
+module.exports = {
+	name: 'Adding theme to active plugins sorted set',
+	timestamp: Date.UTC(2015, 11, 23),
+	method: function (callback) {
+		async.waterfall([
+			async.apply(db.getObjectField, 'config', 'theme:id'),
+			async.apply(db.sortedSetAdd, 'plugins:active', 0),
+		], callback);
+	},
+};

src/upgrades/1.0.0/user_best_posts.js

@@ -0,0 +1,31 @@
+/* jslint node: true */
+
+'use strict';
+
+var db = require('../../database');
+
+var async = require('async');
+var winston = require('winston');
+
+module.exports = {
+	name: 'Creating user best post sorted sets',
+	timestamp: Date.UTC(2016, 0, 14),
+	method: function (callback) {
+		var batch = require('../../batch');
+
+		batch.processSortedSet('posts:pid', function (ids, next) {
+			async.eachSeries(ids, function (id, next) {
+				db.getObjectFields('post:' + id, ['pid', 'uid', 'votes'], function (err, postData) {
+					if (err) {
+						return next(err);
+					}
+					if (!postData || !parseInt(postData.votes, 10) || !parseInt(postData.uid, 10)) {
+						return next();
+					}
+					winston.verbose('processing pid: ' + postData.pid + ' uid: ' + postData.uid + ' votes: ' + postData.votes);
+					db.sortedSetAdd('uid:' + postData.uid + ':posts:votes', postData.votes, postData.pid, next);
+				});
+			}, next);
+		}, {}, callback);
+	},
+};

src/upgrades/1.0.0/users_notvalidated.js

@@ -0,0 +1,31 @@
+/* jslint node: true */
+
+'use strict';
+
+var db = require('../../database');
+
+var async = require('async');
+var winston = require('winston');
+
+module.exports = {
+	name: 'Creating users:notvalidated',
+	timestamp: Date.UTC(2016, 0, 20),
+	method: function (callback) {
+		var batch = require('../../batch');
+		var now = Date.now();
+		batch.processSortedSet('users:joindate', function (ids, next) {
+			async.eachSeries(ids, function (id, next) {
+				db.getObjectFields('user:' + id, ['uid', 'email:confirmed'], function (err, userData) {
+					if (err) {
+						return next(err);
+					}
+					if (!userData || !parseInt(userData.uid, 10) || parseInt(userData['email:confirmed'], 10) === 1) {
+						return next();
+					}
+					winston.verbose('processing uid: ' + userData.uid + ' email:confirmed: ' + userData['email:confirmed']);
+					db.sortedSetAdd('users:notvalidated', now, userData.uid, next);
+				});
+			}, next);
+		}, callback);
+	},
+};

src/upgrades/1.1.0/assign_topic_read_privilege.js

@@ -0,0 +1,73 @@
+/* jslint node: true */
+
+'use strict';
+
+var db = require('../../database');
+
+var async = require('async');
+var winston = require('winston');
+
+module.exports = {
+	name: 'Giving topics:read privs to any group that was previously allowed to Find & Access Category',
+	timestamp: Date.UTC(2016, 4, 28),
+	method: function (callback) {
+		var groupsAPI = require('../../groups');
+		var privilegesAPI = require('../../privileges');
+
+		db.getSortedSetRange('categories:cid', 0, -1, function (err, cids) {
+			if (err) {
+				return callback(err);
+			}
+
+			async.eachSeries(cids, function (cid, next) {
+				privilegesAPI.categories.list(cid, function (err, data) {
+					if (err) {
+						return next(err);
+					}
+
+					var groups = data.groups;
+					var users = data.users;
+
+					async.waterfall([
+						function (next) {
+							async.eachSeries(groups, function (group, next) {
+								if (group.privileges['groups:read']) {
+									return groupsAPI.join('cid:' + cid + ':privileges:groups:topics:read', group.name, function (err) {
+										if (!err) {
+											winston.verbose('cid:' + cid + ':privileges:groups:topics:read granted to gid: ' + group.name);
+										}
+
+										return next(err);
+									});
+								}
+
+								next(null);
+							}, next);
+						},
+						function (next) {
+							async.eachSeries(users, function (user, next) {
+								if (user.privileges.read) {
+									return groupsAPI.join('cid:' + cid + ':privileges:topics:read', user.uid, function (err) {
+										if (!err) {
+											winston.verbose('cid:' + cid + ':privileges:topics:read granted to uid: ' + user.uid);
+										}
+
+										return next(err);
+									});
+								}
+
+								next(null);
+							}, next);
+						},
+					], function (err) {
+						if (!err) {
+							winston.verbose('-- cid ' + cid + ' upgraded');
+						}
+
+						next(err);
+					});
+				});
+			}, callback);
+		});
+	},
+};

src/upgrades/1.1.0/dismiss_flags_from_deleted_topics.js

@@ -0,0 +1,43 @@
+/* jslint node: true */
+
+'use strict';
+
+var db = require('../../database');
+
+var async = require('async');
+var winston = require('winston');
+
+module.exports = {
+	name: 'Dismiss flags from deleted topics',
+	timestamp: Date.UTC(2016, 3, 29),
+	method: function (callback) {
+		var posts = require('../../posts');
+		var topics = require('../../topics');
+
+		var pids;
+		var tids;
+
+		async.waterfall([
+			async.apply(db.getSortedSetRange, 'posts:flagged', 0, -1),
+			function (_pids, next) {
+				pids = _pids;
+				posts.getPostsFields(pids, ['tid'], next);
+			},
+			function (_tids, next) {
+				tids = _tids.map(function (a) {
+					return a.tid;
+				});
+
+				topics.getTopicsFields(tids, ['deleted'], next);
+			},
+			function (state, next) {
+				var toDismiss = state.map(function (a, idx) {
+					return parseInt(a.deleted, 10) === 1 ? pids[idx] : null;
+				}).filter(Boolean);
+
+				winston.verbose('[2016/04/29] ' + toDismiss.length + ' dismissable flags found');
+				async.each(toDismiss, posts.dismissFlag, next);
+			},
+		], callback);
+	},
+};

src/upgrades/1.1.0/group_title_update.js

@@ -0,0 +1,34 @@
+/* jslint node: true */
+
+'use strict';
+
+var db = require('../../database');
+
+var async = require('async');
+var winston = require('winston');
+
+module.exports = {
+	name: 'Group title from settings to user profile',
+	timestamp: Date.UTC(2016, 3, 14),
+	method: function (callback) {
+		var user = require('../../user');
+		var batch = require('../../batch');
+		var count = 0;
+		batch.processSortedSet('users:joindate', function (uids, next) {
+			winston.verbose('upgraded ' + count + ' users');
+			user.getMultipleUserSettings(uids, function (err, settings) {
+				if (err) {
+					return next(err);
+				}
+				count += uids.length;
+				settings = settings.filter(function (setting) {
+					return setting && setting.groupTitle;
+				});
+
+				async.each(settings, function (setting, next) {
+					db.setObjectField('user:' + setting.uid, 'groupTitle', setting.groupTitle, next);
+				}, next);
+			});
+		}, {}, callback);
+	},
+};

src/upgrades/1.1.0/separate_upvote_downvote.js

@@ -0,0 +1,50 @@
+/* jslint node: true */
+
+'use strict';
+
+var db = require('../../database');
+
+var async = require('async');
+var winston = require('winston');
+
+module.exports = {
+	name: 'Store upvotes/downvotes separately',
+	timestamp: Date.UTC(2016, 5, 13),
+	method: function (callback) {
+		var batch = require('../../batch');
+		var posts = require('../../posts');
+		var count = 0;
+		batch.processSortedSet('posts:pid', function (pids, next) {
+			winston.verbose('upgraded ' + count + ' posts');
+			count += pids.length;
+			async.each(pids, function (pid, next) {
+				async.parallel({
+					upvotes: function (next) {
+						db.setCount('pid:' + pid + ':upvote', next);
+					},
+					downvotes: function (next) {
+						db.setCount('pid:' + pid + ':downvote', next);
+					},
+				}, function (err, results) {
+					if (err) {
+						return next(err);
+					}
+					var data = {};
+
+					if (parseInt(results.upvotes, 10) > 0) {
+						data.upvotes = results.upvotes;
+					}
+					if (parseInt(results.downvotes, 10) > 0) {
+						data.downvotes = results.downvotes;
+					}
+
+					if (Object.keys(data).length) {
+						posts.setPostFields(pid, data, next);
+					} else {
+						next();
+					}
+				}, next);
+			}, next);
+		}, {}, callback);
+	},
+};

src/upgrades/1.1.0/user_post_count_per_tid.js

@@ -0,0 +1,50 @@
+/* jslint node: true */
+
+'use strict';
+
+var db = require('../../database');
+
+var async = require('async');
+var winston = require('winston');
+
+module.exports = {
+	name: 'Users post count per tid',
+	timestamp: Date.UTC(2016, 3, 19),
+	method: function (callback) {
+		var batch = require('../../batch');
+		var topics = require('../../topics');
+		var count = 0;
+		batch.processSortedSet('topics:tid', function (tids, next) {
+			winston.verbose('upgraded ' + count + ' topics');
+			count += tids.length;
+			async.each(tids, function (tid, next) {
+				db.delete('tid:' + tid + ':posters', function (err) {
+					if (err) {
+						return next(err);
+					}
+					topics.getPids(tid, function (err, pids) {
+						if (err) {
+							return next(err);
+						}
+
+						if (!pids.length) {
+							return next();
+						}
+
+						async.eachSeries(pids, function (pid, next) {
+							db.getObjectField('post:' + pid, 'uid', function (err, uid) {
+								if (err) {
+									return next(err);
+								}
+								if (!parseInt(uid, 10)) {
+									return next();
+								}
+								db.sortedSetIncrBy('tid:' + tid + ':posters', 1, uid, next);
+							});
+						}, next);
+					});
+				});
+			}, next);
+		}, {}, callback);
+	},
+};

src/upgrades/1.1.1/remove_negative_best_posts.js

@@ -0,0 +1,22 @@
+/* jslint node: true */
+
+'use strict';
+
+var db = require('../../database');
+
+var async = require('async');
+var winston = require('winston');
+
+module.exports = {
+	name: 'Removing best posts with negative scores',
+	timestamp: Date.UTC(2016, 7, 5),
+	method: function (callback) {
+		var batch = require('../../batch');
+		batch.processSortedSet('users:joindate', function (ids, next) {
+			async.each(ids, function (id, next) {
+				winston.verbose('processing uid ' + id);
+				db.sortedSetsRemoveRangeByScore(['uid:' + id + ':posts:votes'], '-inf', 0, next);
+			}, next);
+		}, {}, callback);
+	},
+};

src/upgrades/1.1.1/upload_privileges.js

@@ -0,0 +1,40 @@
+/* jslint node: true */
+
+'use strict';
+
+var db = require('../../database');
+
+var async = require('async');
+
+module.exports = {
+	name: 'Giving upload privileges',
+	timestamp: Date.UTC(2016, 6, 12),
+	method: function (callback) {
+		var privilegesAPI = require('../../privileges');
+		var meta = require('../../meta');
+
+		db.getSortedSetRange('categories:cid', 0, -1, function (err, cids) {
+			if (err) {
+				return callback(err);
+			}
+
+			async.eachSeries(cids, function (cid, next) {
+				privilegesAPI.categories.list(cid, function (err, data) {
+					if (err) {
+						return next(err);
+					}
+					async.eachSeries(data.groups, function (group, next) {
+						if (group.name === 'guests' && parseInt(meta.config.allowGuestUploads, 10) !== 1) {
+							return next();
+						}
+						if (group.privileges['groups:read']) {
+							privilegesAPI.categories.give(['upload:post:image'], cid, group.name, next);
+						} else {
+							next();
+						}
+					}, next);
+				});
+			}, callback);
+		});
+	},
+};

src/upgrades/1.2.0/category_recent_tids.js

@@ -0,0 +1,33 @@
+/* jslint node: true */
+
+'use strict';
+
+var db = require('../../database');
+
+var async = require('async');
+
+module.exports = {
+	name: 'Category recent tids',
+	timestamp: Date.UTC(2016, 8, 22),
+	method: function (callback) {
+		db.getSortedSetRange('categories:cid', 0, -1, function (err, cids) {
+			if (err) {
+				return callback(err);
+			}
+
+			async.eachSeries(cids, function (cid, next) {
+				db.getSortedSetRevRange('cid:' + cid + ':pids', 0, 0, function (err, pid) {
+					if (err || !pid) {
+						return next(err);
+					}
+					db.getObjectFields('post:' + pid, ['tid', 'timestamp'], function (err, postData) {
+						if (err || !postData || !postData.tid) {
+							return next(err);
+						}
+						db.sortedSetAdd('cid:' + cid + ':recent_tids', postData.timestamp, postData.tid, next);
+					});
+				});
+			}, callback);
+		});
+	},
+};

src/upgrades/1.2.0/edit_delete_deletetopic_privileges.js

@@ -0,0 +1,109 @@
+/* jslint node: true */
+
+'use strict';
+
+var db = require('../../database');
+
+var async = require('async');
+var winston = require('winston');
+
+module.exports = {
+	name: 'Granting edit/delete/delete topic on existing categories',
+	timestamp: Date.UTC(2016, 7, 7),
+	method: function (callback) {
+		var groupsAPI = require('../../groups');
+		var privilegesAPI = require('../../privileges');
+
+		db.getSortedSetRange('categories:cid', 0, -1, function (err, cids) {
+			if (err) {
+				return callback(err);
+			}
+
+			async.eachSeries(cids, function (cid, next) {
+				privilegesAPI.categories.list(cid, function (err, data) {
+					if (err) {
+						return next(err);
+					}
+
+					var groups = data.groups;
+					var users = data.users;
+
+					async.waterfall([
+						function (next) {
+							async.eachSeries(groups, function (group, next) {
+								if (group.privileges['groups:topics:reply']) {
+									return async.parallel([
+										async.apply(groupsAPI.join, 'cid:' + cid + ':privileges:groups:posts:edit', group.name),
+										async.apply(groupsAPI.join, 'cid:' + cid + ':privileges:groups:posts:delete', group.name),
+									], function (err) {
+										if (!err) {
+											winston.verbose('cid:' + cid + ':privileges:groups:posts:edit, cid:' + cid + ':privileges:groups:posts:delete granted to gid: ' + group.name);
+										}
+
+										return next(err);
+									});
+								}
+
+								next(null);
+							}, next);
+						},
+						function (next) {
+							async.eachSeries(groups, function (group, next) {
+								if (group.privileges['groups:topics:create']) {
+									return groupsAPI.join('cid:' + cid + ':privileges:groups:topics:delete', group.name, function (err) {
+										if (!err) {
+											winston.verbose('cid:' + cid + ':privileges:groups:topics:delete granted to gid: ' + group.name);
+										}
+
+										return next(err);
+									});
+								}
+
+								next(null);
+							}, next);
+						},
+						function (next) {
+							async.eachSeries(users, function (user, next) {
+								if (user.privileges['topics:reply']) {
+									return async.parallel([
+										async.apply(groupsAPI.join, 'cid:' + cid + ':privileges:posts:edit', user.uid),
+										async.apply(groupsAPI.join, 'cid:' + cid + ':privileges:posts:delete', user.uid),
+									], function (err) {
+										if (!err) {
+											winston.verbose('cid:' + cid + ':privileges:posts:edit, cid:' + cid + ':privileges:posts:delete granted to uid: ' + user.uid);
+										}
+
+										return next(err);
+									});
+								}
+
+								next(null);
+							}, next);
+						},
+						function (next) {
+							async.eachSeries(users, function (user, next) {
+								if (user.privileges['topics:create']) {
+									return groupsAPI.join('cid:' + cid + ':privileges:topics:delete', user.uid, function (err) {
+										if (!err) {
+											winston.verbose('cid:' + cid + ':privileges:topics:delete granted to uid: ' + user.uid);
+										}
+
+										return next(err);
+									});
+								}
+
+								next(null);
+							}, next);
+						},
+					], function (err) {
+						if (!err) {
+							winston.verbose('-- cid ' + cid + ' upgraded');
+						}
+
+						next(err);
+					});
+				});
+			}, callback);
+		});
+	},
+};

src/upgrades/1.3.0/favourites_to_bookmarks.js

@@ -0,0 +1,52 @@
+/* jslint node: true */
+
+'use strict';
+
+var db = require('../../database');
+
+var async = require('async');
+
+module.exports = {
+	name: 'Favourites to Bookmarks',
+	timestamp: Date.UTC(2016, 9, 8),
+	method: function (callback) {
+		function upgradePosts(next) {
+			var batch = require('../../batch');
+
+			batch.processSortedSet('posts:pid', function (ids, next) {
+				async.each(ids, function (id, next) {
+					async.waterfall([
+						function (next) {
+							db.rename('pid:' + id + ':users_favourited', 'pid:' + id + ':users_bookmarked', next);
+						},
+						function (next) {
+							db.getObjectField('post:' + id, 'reputation', next);
+						},
+						function (reputation, next) {
+							if (parseInt(reputation, 10)) {
+								db.setObjectField('post:' + id, 'bookmarks', reputation, next);
+							} else {
+								next();
+							}
+						},
+						function (next) {
+							db.deleteObjectField('post:' + id, 'reputation', next);
+						},
+					], next);
+				}, next);
+			}, {}, next);
+		}
+
+		function upgradeUsers(next) {
+			var batch = require('../../batch');
+
+			batch.processSortedSet('users:joindate', function (ids, next) {
+				async.each(ids, function (id, next) {
+					db.rename('uid:' + id + ':favourites', 'uid:' + id + ':bookmarks', next);
+				}, next);
+			}, {}, next);
+		}
+
+		async.series([upgradePosts, upgradeUsers], callback);
+	},
+};

src/upgrades/1.3.0/sorted_sets_for_post_replies.js

@@ -0,0 +1,35 @@
+/* jslint node: true */
+
+'use strict';
+
+var db = require('../../database');
+
+var async = require('async');
+var winston = require('winston');
+
+module.exports = {
+	name: 'Sorted sets for post replies',
+	timestamp: Date.UTC(2016, 9, 14),
+	method: function (callback) {
+		var posts = require('../../posts');
+		var batch = require('../../batch');
+		batch.processSortedSet('posts:pid', function (ids, next) {
+			posts.getPostsFields(ids, ['pid', 'toPid', 'timestamp'], function (err, data) {
+				if (err) {
+					return next(err);
+				}
+
+				async.eachSeries(data, function (postData, next) {
+					if (!parseInt(postData.toPid, 10)) {
+						return next(null);
+					}
+					winston.verbose('processing pid: ' + postData.pid + ' toPid: ' + postData.toPid);
+					async.parallel([
+						async.apply(db.sortedSetAdd, 'pid:' + postData.toPid + ':replies', postData.timestamp, postData.pid),
+						async.apply(db.incrObjectField, 'post:' + postData.toPid, 'replies'),
+					], next);
+				}, next);
+			});
+		}, callback);
+	},
+};

src/upgrades/1.4.0/global_and_user_language_keys.js

@@ -0,0 +1,59 @@
+/* jslint node: true */
+
+'use strict';
+
+var db = require('../../database');
+
+var async = require('async');
+
+module.exports = {
+	name: 'Update global and user language keys',
+	timestamp: Date.UTC(2016, 10, 22),
+	method: function (callback) {
+		var user = require('../../user');
+		var meta = require('../../meta');
+		var batch = require('../../batch');
+		var newLanguage;
+		async.parallel([
+			function (next) {
+				meta.configs.get('defaultLang', function (err, defaultLang) {
+					if (err) {
+						return next(err);
+					}
+
+					if (!defaultLang) {
+						return setImmediate(next);
+					}
+
+					newLanguage = defaultLang.replace('_', '-').replace('@', '-x-');
+					if (newLanguage !== defaultLang) {
+						meta.configs.set('defaultLang', newLanguage, next);
+					} else {
+						setImmediate(next);
+					}
+				});
+			},
+			function (next) {
+				batch.processSortedSet('users:joindate', function (ids, next) {
+					async.each(ids, function (uid, next) {
+						async.waterfall([
+							async.apply(db.getObjectField, 'user:' + uid + ':settings', 'userLang'),
+							function (language, next) {
+								if (!language) {
+									return setImmediate(next);
+								}
+
+								newLanguage = language.replace('_', '-').replace('@', '-x-');
+								if (newLanguage !== language) {
+									user.setSetting(uid, 'userLang', newLanguage, next);
+								} else {
+									setImmediate(next);
+								}
+							},
+						], next);
+					}, next);
+				}, next);
+			},
+		], callback);
+	},
+};

src/upgrades/1.4.0/sorted_set_for_pinned_topics.js

@@ -0,0 +1,38 @@
+/* jslint node: true */
+
+'use strict';
+
+var db = require('../../database');
+
+var async = require('async');
+var winston = require('winston');
+
+module.exports = {
+	name: 'Sorted set for pinned topics',
+	timestamp: Date.UTC(2016, 10, 25),
+	method: function (callback) {
+		var topics = require('../../topics');
+		var batch = require('../../batch');
+		batch.processSortedSet('topics:tid', function (ids, next) {
+			topics.getTopicsFields(ids, ['tid', 'cid', 'pinned', 'lastposttime'], function (err, data) {
+				if (err) {
+					return next(err);
+				}
+
+				data = data.filter(function (topicData) {
+					return parseInt(topicData.pinned, 10) === 1;
+				});
+
+				async.eachSeries(data, function (topicData, next) {
+					winston.verbose('processing tid: ' + topicData.tid);
+
+					async.parallel([
+						async.apply(db.sortedSetAdd, 'cid:' + topicData.cid + ':tids:pinned', Date.now(), topicData.tid),
+						async.apply(db.sortedSetRemove, 'cid:' + topicData.cid + ':tids', topicData.tid),
+						async.apply(db.sortedSetRemove, 'cid:' + topicData.cid + ':tids:posts', topicData.tid),
+					], next);
+				}, next);
+			});
+		}, callback);
+	},
+};

src/upgrades/1.4.4/config_urls_update.js

@@ -0,0 +1,38 @@
+/* jslint node: true */
+
+'use strict';
+
+var db = require('../../database');
+
+var async = require('async');
+
+module.exports = {
+	name: 'User_friendly_upgrade_script_name',
+	timestamp: Date.UTC(2017, 0, 1),
+	method: function (callback) {
+		async.waterfall([
+			function (cb) {
+				db.getObject('config', cb);
+			},
+			function (config, cb) {
+				if (!config) {
+					return cb();
+				}
+
+				var keys = ['brand:favicon', 'brand:touchicon', 'og:image', 'brand:logo:url', 'defaultAvatar', 'profile:defaultCovers'];
+
+				keys.forEach(function (key) {
+					var oldValue = config[key];
+
+					if (!oldValue || typeof oldValue !== 'string') {
+						return;
+					}
+
+					config[key] = oldValue.replace(/(?:\/assets)?\/(images|uploads)\//g, '/assets/$1/');
+				});
+
+				db.setObject('config', config, cb);
+			},
+		], callback);
+	},
+};

src/upgrades/1.4.4/sound_settings.js

@@ -0,0 +1,67 @@
+/* jslint node: true */
+
+'use strict';
+
+var db = require('../../database');
+
+var async = require('async');
+
+module.exports = {
+	name: 'Update global and user sound settings',
+	timestamp: Date.UTC(2017, 1, 25),
+	method: function (callback) {
+		var meta = require('../../meta');
+		var batch = require('../../batch');
+
+		var map = {
+			'notification.mp3': 'Default | Deedle-dum',
+			'waterdrop-high.mp3': 'Default | Water drop (high)',
+			'waterdrop-low.mp3': 'Default | Water drop (low)',
+		};
+
+		async.parallel([
+			function (cb) {
+				var keys = ['chat-incoming', 'chat-outgoing', 'notification'];
+
+				db.getObject('settings:sounds', function (err, settings) {
+					if (err || !settings) {
+						return cb(err);
+					}
+
+					keys.forEach(function (key) {
+						if (settings[key] && settings[key].indexOf(' | ') === -1) {
+							settings[key] = map[settings[key]] || '';
+						}
+					});
+
+					meta.configs.setMultiple(settings, cb);
+				});
+			},
+			function (cb) {
+				var keys = ['notificationSound', 'incomingChatSound', 'outgoingChatSound'];
+
+				batch.processSortedSet('users:joindate', function (ids, next) {
+					async.each(ids, function (uid, next) {
+						db.getObject('user:' + uid + ':settings', function (err, settings) {
+							if (err || !settings) {
+								return next(err);
+							}
+							var newSettings = {};
+							keys.forEach(function (key) {
+								if (settings[key] && settings[key].indexOf(' | ') === -1) {
+									newSettings[key] = map[settings[key]] || '';
+								}
+							});
+
+							if (Object.keys(newSettings).length) {
+								db.setObject('user:' + uid + ':settings', newSettings, next);
+							} else {
+								setImmediate(next);
+							}
+						});
+					}, next);
+				}, cb);
+			},
+		], callback);
+	},
+};

src/upgrades/1.5.0/flags_refactor.js

@@ -0,0 +1,89 @@
+/* jslint node: true */
+
+'use strict';
+
+var db = require('../../database');
+
+var async = require('async');
+
+module.exports = {
+	name: 'Migrating flags to new schema',
+	timestamp: Date.UTC(2016, 11, 7),
+	method: function (callback) {
+		var batch = require('../../batch');
+		var posts = require('../../posts');
+		var flags = require('../../flags');
+
+		batch.processSortedSet('posts:pid', function (ids, next) {
+			posts.getPostsByPids(ids, 1, function (err, posts) {
+				if (err) {
+					return next(err);
+				}
+
+				posts = posts.filter(function (post) {
+					return post.hasOwnProperty('flags');
+				});
+
+				async.each(posts, function (post, next) {
+					async.parallel({
+						uids: async.apply(db.getSortedSetRangeWithScores, 'pid:' + post.pid + ':flag:uids', 0, -1),
+						reasons: async.apply(db.getSortedSetRange, 'pid:' + post.pid + ':flag:uid:reason', 0, -1),
+					}, function (err, data) {
+						if (err) {
+							return next(err);
+						}
+
+						// Adding in another check here in case a post was improperly dismissed (flag count > 1 but no flags in db)
+						if (!data.uids.length || !data.reasons.length) {
+							return setImmediate(next);
+						}
+
+						// Just take the first entry
+						var datetime = data.uids[0].score;
+						var reason = data.reasons[0].split(':')[1];
+						var flagObj;
+
+						async.waterfall([
+							async.apply(flags.create, 'post', post.pid, data.uids[0].value, reason, datetime),
+							function (_flagObj, next) {
+								flagObj = _flagObj;
+								if (post['flag:state'] || post['flag:assignee']) {
+									flags.update(flagObj.flagId, 1, {
+										state: post['flag:state'],
+										assignee: post['flag:assignee'],
+										datetime: datetime,
+									}, next);
+								} else {
+									setImmediate(next);
+								}
+							},
+							function (next) {
+								if (post.hasOwnProperty('flag:notes') && post['flag:notes'].length) {
+									try {
+										var history = JSON.parse(post['flag:history']);
+										history = history.filter(function (event) {
+											return event.type === 'notes';
+										})[0];
+
+										flags.appendNote(flagObj.flagId, history.uid, post['flag:notes'], history.timestamp, next);
+									} catch (e) {
+										next(e);
+									}
+								} else {
+									setImmediate(next);
+								}
+							},
+						], function (err) {
+							if (err && err.message === '[[error:already-flagged]]') {
+								// Already flagged, no need to parse, but not an error
+								next();
+							} else {
+								next(err);
+							}
+						});
+					});
+				}, next);
+			});
+		}, callback);
+	},
+};

src/upgrades/1.5.0/post_votes_zset.js

@@ -0,0 +1,26 @@
+/* jslint node: true */
+
+'use strict';
+
+var db = require('../../database');
+
+var async = require('async');
+
+module.exports = {
+	name: 'New sorted set posts:votes',
+	timestamp: Date.UTC(2017, 1, 27),
+	method: function (callback) {
+		require('../../batch').processSortedSet('posts:pid', function (pids, next) {
+			async.each(pids, function (pid, next) {
+				db.getObjectFields('post:' + pid, ['upvotes', 'downvotes'], function (err, postData) {
+					if (err || !postData) {
+						return next(err);
+					}
+
+					var votes = parseInt(postData.upvotes || 0, 10) - parseInt(postData.downvotes || 0, 10);
+					db.sortedSetAdd('posts:votes', votes, pid, next);
+				});
+			}, next);
+		}, {}, callback);
+	},
+};

src/upgrades/TEMPLATE

@@ -0,0 +1,16 @@
+/* jslint node: true */
+
+'use strict';
+
+var db = require('../../database');
+
+var async = require('async');
+var winston = require('winston');
+
+module.exports = {
+	name: 'User_friendly_upgrade_script_name',
+	timestamp: Date.UTC(2017, 0, 1),
+	method: function (callback) {
+		// Do stuff here...
+	},
+};