Async refactor in place (#7736)

* feat: allow both callback&and await

* feat: ignore async key

* feat: callbackify and promisify in same file

* Revert "feat: callbackify and promisify in same file"

This reverts commit cea206a9b8.

* feat: no need to store .callbackify

* feat: change getTopics to async

* feat: remove .async

* fix: byScore

* feat: rewrite topics/index and social with async/await

* fix: rewrite topics/data.js

fix issue with async.waterfall, only pass result if its not undefined

* feat: add callbackify to redis/psql

* feat: psql use await

* fix: redis 🌋

* feat: less returns

* feat: more await rewrite

* fix: redis tests

* feat: convert sortedSetAdd

rewrite psql transaction to async/await

* feat: 🐶

* feat: test

* feat: log client and query

* feat: log bind

* feat: more logs

* feat: more logs

* feat: check perform

* feat: dont callbackify transaction

* feat: remove logs

* fix: main functions

* feat: more logs

* fix: increment

* fix: rename

* feat: remove cls

* fix: remove console.log

* feat: add deprecation message to .async usage

* feat: update more dbal methods

* fix: redis :voodoo:

* feat:  fix redis zrem, convert setObject

* feat: upgrade getObject methods

* fix: psql getObjectField

* fix: redis tests

* feat: getObjectKeys

* feat: getObjectValues

* feat: isObjectField

* fix: add missing return

* feat: delObjectField

* feat: incrObjectField

* fix: add missing await

* feat: remove exposed helpers

* feat: list methods

* feat: flush/empty

* feat: delete

* fix: redis delete all

* feat: get/set

* feat: incr/rename

* feat: type

* feat: expire

* feat: setAdd

* feat: setRemove

* feat: isSetMember

* feat: getSetMembers

* feat: setCount, setRemoveRandom

* feat: zcard,zcount

* feat: sortedSetRank

* feat: isSortedSetMember

* feat: zincrby

* feat: sortedSetLex

* feat: processSortedSet

* fix: add mising await

* feat: debug psql

* fix: psql test

* fix: test

* fix: another test

* fix: test fix

* fix: psql tests

* feat: remove logs

* feat: user arrow func

use builtin async promises

* feat: topic bookmarks

* feat: topic.delete

* feat: topic.restore

* feat: topics.purge

* feat: merge

* feat: suggested

* feat: topics/user.js

* feat: topics modules

* feat: topics/follow

* fix: deprecation msg

* feat: fork

* feat: topics/posts

* feat: sorted/recent

* feat: topic/teaser

* feat: topics/tools

* feat: topics/unread

* feat: add back node versions

disable deprecation notice
wrap async controllers in try/catch

* feat: use db directly

* feat: promisify in place

* fix: redis/psql

* feat: deprecation message

logs for psql

* feat: more logs

* feat: more logs

* feat: logs again

* feat: more logs

* fix: call release

* feat: restore travis, remove logs

* fix: loops

* feat: remove .async. usage

src/database/postgres/sorted/add.js

@@ -1,134 +1,112 @@
 'use strict';
 
-var async = require('async');
-
 module.exports = function (db, module) {
-	var helpers = module.helpers.postgres;
+	var helpers = require('../helpers');
 	var utils = require('../../../utils');
 
-	module.sortedSetAdd = function (key, score, value, callback) {
-		callback = callback || helpers.noop;
-
+	module.sortedSetAdd = async function (key, score, value) {
 		if (!key) {
-			return callback();
+			return;
 		}
 
 		if (Array.isArray(score) && Array.isArray(value)) {
-			return sortedSetAddBulk(key, score, value, callback);
+			return await sortedSetAddBulk(key, score, value);
 		}
 		if (!utils.isNumber(score)) {
-			return setImmediate(callback, new Error('[[error:invalid-score, ' + score + ']]'));
+			throw new Error('[[error:invalid-score, ' + score + ']]');
 		}
 		value = helpers.valueToString(value);
 		score = parseFloat(score);
 
-		module.transaction(function (tx, done) {
-			var query = tx.client.query.bind(tx.client);
+		await module.transaction(async function (client) {
+			var query = client.query.bind(client);
 
-			async.series([
-				async.apply(helpers.ensureLegacyObjectType, tx.client, key, 'zset'),
-				async.apply(query, {
-					name: 'sortedSetAdd',
-					text: `
-INSERT INTO "legacy_zset" ("_key", "value", "score")
-VALUES ($1::TEXT, $2::TEXT, $3::NUMERIC)
-    ON CONFLICT ("_key", "value")
-    DO UPDATE SET "score" = $3::NUMERIC`,
-					values: [key, value, score],
-				}),
-			], function (err) {
-				done(err);
+			await helpers.ensureLegacyObjectType(client, key, 'zset');
+			await query({
+				name: 'sortedSetAdd',
+				text: `
+	INSERT INTO "legacy_zset" ("_key", "value", "score")
+	VALUES ($1::TEXT, $2::TEXT, $3::NUMERIC)
+	ON CONFLICT ("_key", "value")
+	DO UPDATE SET "score" = $3::NUMERIC`,
+				values: [key, value, score],
 			});
-		}, callback);
+		});
 	};
 
-	function sortedSetAddBulk(key, scores, values, callback) {
+	async function sortedSetAddBulk(key, scores, values) {
 		if (!scores.length || !values.length) {
-			return callback();
+			return;
 		}
 		if (scores.length !== values.length) {
-			return callback(new Error('[[error:invalid-data]]'));
+			throw new Error('[[error:invalid-data]]');
 		}
 		for (let i = 0; i < scores.length; i += 1) {
 			if (!utils.isNumber(scores[i])) {
-				return setImmediate(callback, new Error('[[error:invalid-score, ' + scores[i] + ']]'));
+				throw new Error('[[error:invalid-score, ' + scores[i] + ']]');
 			}
 		}
 		values = values.map(helpers.valueToString);
-		scores = scores.map(function (score) {
-			return parseFloat(score);
-		});
+		scores = scores.map(score => parseFloat(score));
 
 		helpers.removeDuplicateValues(values, scores);
 
-		module.transaction(function (tx, done) {
-			var query = tx.client.query.bind(tx.client);
-
-			async.series([
-				async.apply(helpers.ensureLegacyObjectType, tx.client, key, 'zset'),
-				async.apply(query, {
-					name: 'sortedSetAddBulk',
-					text: `
+		await module.transaction(async function (client) {
+			var query = client.query.bind(client);
+			await helpers.ensureLegacyObjectType(client, key, 'zset');
+			await query({
+				name: 'sortedSetAddBulk',
+				text: `
 INSERT INTO "legacy_zset" ("_key", "value", "score")
 SELECT $1::TEXT, v, s
-  FROM UNNEST($2::TEXT[], $3::NUMERIC[]) vs(v, s)
-    ON CONFLICT ("_key", "value")
-    DO UPDATE SET "score" = EXCLUDED."score"`,
-					values: [key, values, scores],
-				}),
-			], function (err) {
-				done(err);
+FROM UNNEST($2::TEXT[], $3::NUMERIC[]) vs(v, s)
+ON CONFLICT ("_key", "value")
+DO UPDATE SET "score" = EXCLUDED."score"`,
+				values: [key, values, scores],
 			});
-		}, callback);
+		});
 	}
 
-	module.sortedSetsAdd = function (keys, scores, value, callback) {
-		callback = callback || helpers.noop;
-
+	module.sortedSetsAdd = async function (keys, scores, value) {
 		if (!Array.isArray(keys) || !keys.length) {
-			return callback();
+			return;
 		}
 		const isArrayOfScores = Array.isArray(scores);
 		if (!isArrayOfScores && !utils.isNumber(scores)) {
-			return setImmediate(callback, new Error('[[error:invalid-score, ' + scores + ']]'));
+			throw new Error('[[error:invalid-score, ' + scores + ']]');
 		}
 
 		if (isArrayOfScores && scores.length !== keys.length) {
-			return setImmediate(callback, new Error('[[error:invalid-data]]'));
+			throw new Error('[[error:invalid-data]]');
 		}
 
 		value = helpers.valueToString(value);
 		scores = isArrayOfScores ? scores.map(score => parseFloat(score)) : parseFloat(scores);
 
-		module.transaction(function (tx, done) {
-			var query = tx.client.query.bind(tx.client);
-
-			async.series([
-				async.apply(helpers.ensureLegacyObjectsType, tx.client, keys, 'zset'),
-				async.apply(query, {
-					name: isArrayOfScores ? 'sortedSetsAddScores' : 'sortedSetsAdd',
-					text: isArrayOfScores ? `
+		await module.transaction(async function (client) {
+			var query = client.query.bind(client);
+			await helpers.ensureLegacyObjectsType(client, keys, 'zset');
+			await query({
+				name: isArrayOfScores ? 'sortedSetsAddScores' : 'sortedSetsAdd',
+				text: isArrayOfScores ? `
 INSERT INTO "legacy_zset" ("_key", "value", "score")
 SELECT k, $2::TEXT, s
-  FROM UNNEST($1::TEXT[], $3::NUMERIC[]) vs(k, s)
-    ON CONFLICT ("_key", "value")
-		DO UPDATE SET "score" = EXCLUDED."score"` : `
+FROM UNNEST($1::TEXT[], $3::NUMERIC[]) vs(k, s)
+ON CONFLICT ("_key", "value")
+	DO UPDATE SET "score" = EXCLUDED."score"` : `
 INSERT INTO "legacy_zset" ("_key", "value", "score")
-		SELECT k, $2::TEXT, $3::NUMERIC
-			FROM UNNEST($1::TEXT[]) k
-				ON CONFLICT ("_key", "value")
-				DO UPDATE SET "score" = $3::NUMERIC`,
-					values: [keys, value, scores],
-				}),
-			], function (err) {
-				done(err);
+	SELECT k, $2::TEXT, $3::NUMERIC
+		FROM UNNEST($1::TEXT[]) k
+			ON CONFLICT ("_key", "value")
+			DO UPDATE SET "score" = $3::NUMERIC`,
+				values: [keys, value, scores],
 			});
-		}, callback);
+		});
 	};
 
-	module.sortedSetAddBulk = function (data, callback) {
+	module.sortedSetAddBulk = async function (data) {
 		if (!Array.isArray(data) || !data.length) {
-			return setImmediate(callback);
+			return;
 		}
 		const keys = [];
 		const values = [];
@@ -138,24 +116,19 @@ INSERT INTO "legacy_zset" ("_key", "value", "score")
 			scores.push(item[1]);
 			values.push(item[2]);
 		});
-		module.transaction(function (tx, done) {
-			var query = tx.client.query.bind(tx.client);
-
-			async.series([
-				async.apply(helpers.ensureLegacyObjectsType, tx.client, keys, 'zset'),
-				async.apply(query, {
-					name: 'sortedSetAddBulk2',
-					text: `
+		await module.transaction(async function (client) {
+			var query = client.query.bind(client);
+			await helpers.ensureLegacyObjectsType(client, keys, 'zset');
+			await query({
+				name: 'sortedSetAddBulk2',
+				text: `
 INSERT INTO "legacy_zset" ("_key", "value", "score")
 SELECT k, v, s
-  FROM UNNEST($1::TEXT[], $2::TEXT[], $3::NUMERIC[]) vs(k, v, s)
-    ON CONFLICT ("_key", "value")
-    DO UPDATE SET "score" = EXCLUDED."score"`,
-					values: [keys, values, scores],
-				}),
-			], function (err) {
-				done(err);
+FROM UNNEST($1::TEXT[], $2::TEXT[], $3::NUMERIC[]) vs(k, v, s)
+ON CONFLICT ("_key", "value")
+DO UPDATE SET "score" = EXCLUDED."score"`,
+				values: [keys, values, scores],
 			});
-		}, callback);
+		});
 	};
 };

src/database/postgres/sorted/intersect.js

@@ -1,12 +1,12 @@
 'use strict';
 
 module.exports = function (db, module) {
-	module.sortedSetIntersectCard = function (keys, callback) {
+	module.sortedSetIntersectCard = async function (keys) {
 		if (!Array.isArray(keys) || !keys.length) {
-			return callback(null, 0);
+			return 0;
 		}
 
-		db.query({
+		const res = await db.query({
 			name: 'sortedSetIntersectCard',
 			text: `
 WITH A AS (SELECT z."value" v,
@@ -21,27 +21,22 @@ SELECT COUNT(*) c
   FROM A
  WHERE A.c = array_length($1::TEXT[], 1)`,
 			values: [keys],
-		}, function (err, res) {
-			if (err) {
-				return callback(err);
-			}
-
-			callback(null, parseInt(res.rows[0].c, 10));
 		});
+
+		return parseInt(res.rows[0].c, 10);
 	};
 
-
-	module.getSortedSetIntersect = function (params, callback) {
+	module.getSortedSetIntersect = async function (params) {
 		params.sort = 1;
-		getSortedSetIntersect(params, callback);
+		return await getSortedSetIntersect(params);
 	};
 
-	module.getSortedSetRevIntersect = function (params, callback) {
+	module.getSortedSetRevIntersect = async function (params) {
 		params.sort = -1;
-		getSortedSetIntersect(params, callback);
+		return await getSortedSetIntersect(params);
 	};
 
-	function getSortedSetIntersect(params, callback) {
+	async function getSortedSetIntersect(params) {
 		var sets = params.sets;
 		var start = params.hasOwnProperty('start') ? params.start : 0;
 		var stop = params.hasOwnProperty('stop') ? params.stop : -1;
@@ -60,7 +55,7 @@ SELECT COUNT(*) c
 			limit = null;
 		}
 
-		db.query({
+		const res = await db.query({
 			name: 'getSortedSetIntersect' + aggregate + (params.sort > 0 ? 'Asc' : 'Desc') + 'WithScores',
 			text: `
 WITH A AS (SELECT z."value",
@@ -81,25 +76,19 @@ SELECT A."value",
  LIMIT $4::INTEGER
 OFFSET $3::INTEGER`,
 			values: [sets, weights, start, limit],
-		}, function (err, res) {
-			if (err) {
-				return callback(err);
-			}
-
-			if (params.withScores) {
-				res.rows = res.rows.map(function (r) {
-					return {
-						value: r.value,
-						score: parseFloat(r.score),
-					};
-				});
-			} else {
-				res.rows = res.rows.map(function (r) {
-					return r.value;
-				});
-			}
-
-			callback(null, res.rows);
 		});
+
+		if (params.withScores) {
+			res.rows = res.rows.map(function (r) {
+				return {
+					value: r.value,
+					score: parseFloat(r.score),
+				};
+			});
+		} else {
+			res.rows = res.rows.map(r => r.value);
+		}
+
+		return res.rows;
 	}
 };

src/database/postgres/sorted/remove.js

@@ -1,64 +1,55 @@
 'use strict';
 
 module.exports = function (db, module) {
-	var helpers = module.helpers.postgres;
-
-	module.sortedSetRemove = function (key, value, callback) {
-		function done(err) {
-			if (callback) {
-				callback(err);
-			}
-		}
+	var helpers = require('../helpers');
 
+	module.sortedSetRemove = async function (key, value) {
 		if (!key) {
-			return done();
+			return;
+		}
+		const isValueArray = Array.isArray(value);
+		if (!value || (isValueArray && !value.length)) {
+			return;
 		}
 
 		if (!Array.isArray(key)) {
 			key = [key];
 		}
 
-		if (!Array.isArray(value)) {
+		if (!isValueArray) {
 			value = [value];
 		}
 		value = value.map(helpers.valueToString);
-
-		db.query({
+		await db.query({
 			name: 'sortedSetRemove',
 			text: `
 DELETE FROM "legacy_zset"
  WHERE "_key" = ANY($1::TEXT[])
    AND "value" = ANY($2::TEXT[])`,
 			values: [key, value],
-		}, done);
+		});
 	};
 
-	module.sortedSetsRemove = function (keys, value, callback) {
-		callback = callback || helpers.noop;
-
+	module.sortedSetsRemove = async function (keys, value) {
 		if (!Array.isArray(keys) || !keys.length) {
-			return callback();
+			return;
 		}
 
 		value = helpers.valueToString(value);
 
-		db.query({
+		await db.query({
 			name: 'sortedSetsRemove',
 			text: `
 DELETE FROM "legacy_zset"
  WHERE "_key" = ANY($1::TEXT[])
    AND "value" = $2::TEXT`,
 			values: [keys, value],
-		}, function (err) {
-			callback(err);
 		});
 	};
 
-	module.sortedSetsRemoveRangeByScore = function (keys, min, max, callback) {
-		callback = callback || helpers.noop;
-
+	module.sortedSetsRemoveRangeByScore = async function (keys, min, max) {
 		if (!Array.isArray(keys) || !keys.length) {
-			return callback();
+			return;
 		}
 
 		if (min === '-inf') {
@@ -68,7 +59,7 @@ DELETE FROM "legacy_zset"
 			max = null;
 		}
 
-		db.query({
+		await db.query({
 			name: 'sortedSetsRemoveRangeByScore',
 			text: `
 DELETE FROM "legacy_zset"
@@ -76,8 +67,6 @@ DELETE FROM "legacy_zset"
    AND ("score" >= $2::NUMERIC OR $2::NUMERIC IS NULL)
    AND ("score" <= $3::NUMERIC OR $3::NUMERIC IS NULL)`,
 			values: [keys, min, max],
-		}, function (err) {
-			callback(err);
 		});
 	};
 };

src/database/postgres/sorted/union.js

@@ -1,12 +1,12 @@
 'use strict';
 
 module.exports = function (db, module) {
-	module.sortedSetUnionCard = function (keys, callback) {
+	module.sortedSetUnionCard = async function (keys) {
 		if (!Array.isArray(keys) || !keys.length) {
-			return callback(null, 0);
+			return 0;
 		}
 
-		db.query({
+		const res = await db.query({
 			name: 'sortedSetUnionCard',
 			text: `
 SELECT COUNT(DISTINCT z."value") c
@@ -16,26 +16,21 @@ SELECT COUNT(DISTINCT z."value") c
         AND o."type" = z."type"
  WHERE o."_key" = ANY($1::TEXT[])`,
 			values: [keys],
-		}, function (err, res) {
-			if (err) {
-				return callback(err);
-			}
-
-			callback(null, parseInt(res.rows[0].c, 10));
 		});
+		return res.rows[0].c;
 	};
 
-	module.getSortedSetUnion = function (params, callback) {
+	module.getSortedSetUnion = async function (params) {
 		params.sort = 1;
-		getSortedSetUnion(params, callback);
+		return await getSortedSetUnion(params);
 	};
 
-	module.getSortedSetRevUnion = function (params, callback) {
+	module.getSortedSetRevUnion = async function (params) {
 		params.sort = -1;
-		getSortedSetUnion(params, callback);
+		return await getSortedSetUnion(params);
 	};
 
-	function getSortedSetUnion(params, callback) {
+	async function getSortedSetUnion(params) {
 		var sets = params.sets;
 		var start = params.hasOwnProperty('start') ? params.start : 0;
 		var stop = params.hasOwnProperty('stop') ? params.stop : -1;
@@ -54,7 +49,7 @@ SELECT COUNT(DISTINCT z."value") c
 			limit = null;
 		}
 
-		db.query({
+		const res = await db.query({
 			name: 'getSortedSetUnion' + aggregate + (params.sort > 0 ? 'Asc' : 'Desc') + 'WithScores',
 			text: `
 WITH A AS (SELECT z."value",
@@ -73,25 +68,18 @@ SELECT A."value",
  LIMIT $4::INTEGER
 OFFSET $3::INTEGER`,
 			values: [sets, weights, start, limit],
-		}, function (err, res) {
-			if (err) {
-				return callback(err);
-			}
-
-			if (params.withScores) {
-				res.rows = res.rows.map(function (r) {
-					return {
-						value: r.value,
-						score: parseFloat(r.score),
-					};
-				});
-			} else {
-				res.rows = res.rows.map(function (r) {
-					return r.value;
-				});
-			}
-
-			callback(null, res.rows);
 		});
+
+		if (params.withScores) {
+			res.rows = res.rows.map(function (r) {
+				return {
+					value: r.value,
+					score: parseFloat(r.score),
+				};
+			});
+		} else {
+			res.rows = res.rows.map(r => r.value);
+		}
+		return res.rows;
 	}
 };