diff --git a/install/package.json b/install/package.json index d817989349..a42eb52182 100644 --- a/install/package.json +++ b/install/package.json @@ -98,7 +98,7 @@ "nconf": "0.13.0", "nodebb-plugin-2factor": "7.5.10", "nodebb-plugin-composer-default": "10.2.51", - "nodebb-plugin-dbsearch": "6.2.19", + "nodebb-plugin-dbsearch": "6.3.0", "nodebb-plugin-emoji": "6.0.3", "nodebb-plugin-emoji-android": "4.1.1", "nodebb-plugin-markdown": "13.2.1", @@ -122,7 +122,7 @@ "postcss-clean": "1.2.0", "progress-webpack-plugin": "1.0.16", "prompt": "1.3.0", - "ioredis": "5.6.1", + "redis": "5.5.6", "rimraf": "6.0.1", "rss": "1.2.2", "rtlcss": "4.3.0", diff --git a/src/activitypub/index.js b/src/activitypub/index.js index 80ec4a40f5..78cb3f26f6 100644 --- a/src/activitypub/index.js +++ b/src/activitypub/index.js @@ -132,7 +132,6 @@ ActivityPub.resolveInboxes = async (ids) => { }, [[], []]); const categoryData = await categories.getCategoriesFields(cids, ['inbox', 'sharedInbox']); const userData = await user.getUsersFields(uids, ['inbox', 'sharedInbox']); - currentIds.forEach((id) => { if (cids.includes(id)) { const data = categoryData[cids.indexOf(id)]; diff --git a/src/controllers/activitypub/index.js b/src/controllers/activitypub/index.js index de478a6021..6751cb30cd 100644 --- a/src/controllers/activitypub/index.js +++ b/src/controllers/activitypub/index.js @@ -145,14 +145,15 @@ Controller.postInbox = async (req, res) => { const method = String(req.body.type).toLowerCase(); if (!activitypub.inbox.hasOwnProperty(method)) { winston.warn(`[activitypub/inbox] Received Activity of type ${method} but unable to handle. Ignoring.`); + console.log('[activitypub/inbox] method not found', method, req.body); return res.sendStatus(200); } try { await activitypub.inbox[method](req); await activitypub.record(req.body); - helpers.formatApiResponse(202, res); + await helpers.formatApiResponse(202, res); } catch (e) { - helpers.formatApiResponse(500, res, e); + helpers.formatApiResponse(500, res, e).catch(err => winston.error(err.stack)); } }; diff --git a/src/controllers/helpers.js b/src/controllers/helpers.js index a6ade8c73b..e11692867e 100644 --- a/src/controllers/helpers.js +++ b/src/controllers/helpers.js @@ -448,6 +448,10 @@ helpers.getHomePageRoutes = async function (uid) { }; helpers.formatApiResponse = async (statusCode, res, payload) => { + if (!res.hasOwnProperty('req')) { + console.log('formatApiResponse', statusCode, payload); + } + if (res.req.method === 'HEAD') { return res.sendStatus(statusCode); } diff --git a/src/database/redis.js b/src/database/redis.js index f73ee79313..472dae8de6 100644 --- a/src/database/redis.js +++ b/src/database/redis.js @@ -61,7 +61,7 @@ redisModule.checkCompatibilityVersion = function (version, callback) { }; redisModule.close = async function () { - await redisModule.client.quit(); + await redisModule.client.close(); if (redisModule.objectCache) { redisModule.objectCache.reset(); } diff --git a/src/database/redis/connection.js b/src/database/redis/connection.js index a4ba757ef6..2a38cf1a79 100644 --- a/src/database/redis/connection.js +++ b/src/database/redis/connection.js @@ -1,7 +1,7 @@ 'use strict'; const nconf = require('nconf'); -const Redis = require('ioredis'); +const { createClient, createCluster, createSentinel } = require('redis'); const winston = require('winston'); const connection = module.exports; @@ -13,28 +13,40 @@ connection.connect = async function (options) { let cxn; if (options.cluster) { - cxn = new Redis.Cluster(options.cluster, options.options); - } else if (options.sentinels) { - cxn = new Redis({ - sentinels: options.sentinels, + const rootNodes = options.cluster.map(node => ({ url : `redis://${node.host}:${node.port}` })); + cxn = createCluster({ ...options.options, + rootNodes: rootNodes, + }); + } else if (options.sentinels) { + const sentinelRootNodes = options.sentinels.map(sentinel => ({ host: sentinel.host, port: sentinel.port })); + cxn = createSentinel({ + ...options.options, + name: 'sentinel-db', + sentinelRootNodes, }); } else if (redis_socket_or_host && String(redis_socket_or_host).indexOf('/') >= 0) { // If redis.host contains a path name character, use the unix dom sock connection. ie, /tmp/redis.sock - cxn = new Redis({ + cxn = createClient({ ...options.options, - path: redis_socket_or_host, password: options.password, - db: options.database, + database: options.database, + socket: { + path: redis_socket_or_host, + reconnectStrategy: 3000, + }, }); } else { // Else, connect over tcp/ip - cxn = new Redis({ + cxn = createClient({ ...options.options, host: redis_socket_or_host, port: options.port, password: options.password, - db: options.database, + database: options.database, + socket: { + reconnectStrategy: 3000, + }, }); } @@ -49,9 +61,14 @@ connection.connect = async function (options) { }); cxn.on('ready', () => { // back-compat with node_redis - cxn.batch = cxn.pipeline; + cxn.batch = cxn.multi; resolve(cxn); }); + cxn.connect().then(() => { + winston.info('Connected to Redis successfully'); + }).catch((err) => { + winston.error('Error connecting to Redis:', err); + }); if (options.password) { cxn.auth(options.password); diff --git a/src/database/redis/hash.js b/src/database/redis/hash.js index 4c6e7b374f..c03bff055b 100644 --- a/src/database/redis/hash.js +++ b/src/database/redis/hash.js @@ -25,12 +25,13 @@ module.exports = function (module) { if (!Object.keys(data).length) { return; } + const strObj = helpers.objectFieldsToString(data); if (Array.isArray(key)) { const batch = module.client.batch(); - key.forEach(k => batch.hmset(k, data)); + key.forEach(k => batch.hSet(k, strObj)); await helpers.execBatch(batch); } else { - await module.client.hmset(key, data); + await module.client.hSet(key, strObj); } cache.del(key); @@ -49,10 +50,16 @@ module.exports = function (module) { const batch = module.client.batch(); data.forEach((item) => { + Object.keys(item[1]).forEach((key) => { + if (item[1][key] === undefined || item[1][key] === null) { + delete item[1][key]; + } + }); if (Object.keys(item[1]).length) { - batch.hmset(item[0], item[1]); + batch.hSet(item[0], helpers.objectFieldsToString(item[1])); } }); + await helpers.execBatch(batch); cache.del(data.map(item => item[0])); }; @@ -61,12 +68,15 @@ module.exports = function (module) { if (!field) { return; } + if (value === null || value === undefined) { + return; + } if (Array.isArray(key)) { const batch = module.client.batch(); - key.forEach(k => batch.hset(k, field, value)); + key.forEach(k => batch.hSet(k, field, String(value))); await helpers.execBatch(batch); } else { - await module.client.hset(key, field, value); + await module.client.hSet(key, field, String(value)); } cache.del(key); @@ -92,9 +102,9 @@ module.exports = function (module) { const cachedData = {}; cache.getUnCachedKeys([key], cachedData); if (cachedData[key]) { - return cachedData[key].hasOwnProperty(field) ? cachedData[key][field] : null; + return Object.hasOwn(cachedData[key], field) ? cachedData[key][field] : null; } - return await module.client.hget(key, String(field)); + return await module.client.hGet(key, String(field)); }; module.getObjectFields = async function (key, fields) { @@ -116,10 +126,10 @@ module.exports = function (module) { let data = []; if (unCachedKeys.length > 1) { const batch = module.client.batch(); - unCachedKeys.forEach(k => batch.hgetall(k)); + unCachedKeys.forEach(k => batch.hGetAll(k)); data = await helpers.execBatch(batch); } else if (unCachedKeys.length === 1) { - data = [await module.client.hgetall(unCachedKeys[0])]; + data = [await module.client.hGetAll(unCachedKeys[0])]; } // convert empty objects into null for back-compat with node_redis @@ -149,21 +159,21 @@ module.exports = function (module) { }; module.getObjectKeys = async function (key) { - return await module.client.hkeys(key); + return await module.client.hKeys(key); }; module.getObjectValues = async function (key) { - return await module.client.hvals(key); + return await module.client.hVals(key); }; module.isObjectField = async function (key, field) { - const exists = await module.client.hexists(key, field); + const exists = await module.client.hExists(key, String(field)); return exists === 1; }; module.isObjectFields = async function (key, fields) { const batch = module.client.batch(); - fields.forEach(f => batch.hexists(String(key), String(f))); + fields.forEach(f => batch.hExists(String(key), String(f))); const results = await helpers.execBatch(batch); return Array.isArray(results) ? helpers.resultsToBool(results) : null; }; @@ -174,7 +184,7 @@ module.exports = function (module) { } field = field.toString(); if (field) { - await module.client.hdel(key, field); + await module.client.hDel(key, field); cache.del(key); } }; @@ -189,10 +199,10 @@ module.exports = function (module) { } if (Array.isArray(key)) { const batch = module.client.batch(); - key.forEach(k => batch.hdel(k, fields)); + key.forEach(k => batch.hDel(k, fields)); await helpers.execBatch(batch); } else { - await module.client.hdel(key, fields); + await module.client.hDel(key, fields); } cache.del(key); @@ -214,10 +224,10 @@ module.exports = function (module) { let result; if (Array.isArray(key)) { const batch = module.client.batch(); - key.forEach(k => batch.hincrby(k, field, value)); + key.forEach(k => batch.hIncrBy(k, field, value)); result = await helpers.execBatch(batch); } else { - result = await module.client.hincrby(key, field, value); + result = await module.client.hIncrBy(key, field, value); } cache.del(key); return Array.isArray(result) ? result.map(value => parseInt(value, 10)) : parseInt(result, 10); @@ -231,7 +241,7 @@ module.exports = function (module) { const batch = module.client.batch(); data.forEach((item) => { for (const [field, value] of Object.entries(item[1])) { - batch.hincrby(item[0], field, value); + batch.hIncrBy(item[0], field, value); } }); await helpers.execBatch(batch); diff --git a/src/database/redis/helpers.js b/src/database/redis/helpers.js index 8961da8255..39585b1f88 100644 --- a/src/database/redis/helpers.js +++ b/src/database/redis/helpers.js @@ -5,13 +5,8 @@ const helpers = module.exports; helpers.noop = function () {}; helpers.execBatch = async function (batch) { - const results = await batch.exec(); - return results.map(([err, res]) => { - if (err) { - throw err; - } - return res; - }); + const results = await batch.execAsPipeline(); + return results; }; helpers.resultsToBool = function (results) { @@ -21,10 +16,29 @@ helpers.resultsToBool = function (results) { return results; }; -helpers.zsetToObjectArray = function (data) { - const objects = new Array(data.length / 2); - for (let i = 0, k = 0; i < objects.length; i += 1, k += 2) { - objects[i] = { value: data[k], score: parseFloat(data[k + 1]) }; - } - return objects; +helpers.objectFieldsToString = function (obj) { + const stringified = Object.fromEntries( + Object.entries(obj).map(([key, value]) => [key, String(value)]) + ); + return stringified; +}; + +helpers.normalizeLexRange = function (min, max, reverse) { + let minmin; + let maxmax; + if (reverse) { + minmin = '+'; + maxmax = '-'; + } else { + minmin = '-'; + maxmax = '+'; + } + + if (min !== minmin && !min.match(/^[[(]/)) { + min = `[${min}`; + } + if (max !== maxmax && !max.match(/^[[(]/)) { + max = `[${max}`; + } + return { lmin: min, lmax: max }; }; diff --git a/src/database/redis/list.js b/src/database/redis/list.js index 101ef178e3..229069b6ed 100644 --- a/src/database/redis/list.js +++ b/src/database/redis/list.js @@ -1,27 +1,25 @@ 'use strict'; module.exports = function (module) { - const helpers = require('./helpers'); - module.listPrepend = async function (key, value) { if (!key) { return; } - await module.client.lpush(key, value); + await module.client.lPush(key, Array.isArray(value) ? value.map(String) : String(value)); }; module.listAppend = async function (key, value) { if (!key) { return; } - await module.client.rpush(key, value); + await module.client.rPush(key, Array.isArray(value) ? value.map(String) : String(value)); }; module.listRemoveLast = async function (key) { if (!key) { return; } - return await module.client.rpop(key); + return await module.client.rPop(key); }; module.listRemoveAll = async function (key, value) { @@ -29,11 +27,11 @@ module.exports = function (module) { return; } if (Array.isArray(value)) { - const batch = module.client.batch(); - value.forEach(value => batch.lrem(key, 0, value)); - await helpers.execBatch(batch); + const batch = module.client.multi(); + value.forEach(value => batch.lRem(key, 0, value)); + await batch.execAsPipeline(); } else { - await module.client.lrem(key, 0, value); + await module.client.lRem(key, 0, value); } }; @@ -41,17 +39,17 @@ module.exports = function (module) { if (!key) { return; } - await module.client.ltrim(key, start, stop); + await module.client.lTrim(key, start, stop); }; module.getListRange = async function (key, start, stop) { if (!key) { return; } - return await module.client.lrange(key, start, stop); + return await module.client.lRange(key, start, stop); }; module.listLength = async function (key) { - return await module.client.llen(key); + return await module.client.lLen(key); }; }; diff --git a/src/database/redis/main.js b/src/database/redis/main.js index b849361a8e..a6506ba701 100644 --- a/src/database/redis/main.js +++ b/src/database/redis/main.js @@ -4,7 +4,7 @@ module.exports = function (module) { const helpers = require('./helpers'); module.flushdb = async function () { - await module.client.send_command('flushdb', []); + await module.client.sendCommand(['FLUSHDB']); }; module.emptydb = async function () { @@ -32,9 +32,9 @@ module.exports = function (module) { const seen = Object.create(null); do { /* eslint-disable no-await-in-loop */ - const res = await module.client.scan(cursor, 'MATCH', params.match, 'COUNT', 10000); - cursor = res[0]; - const values = res[1].filter((value) => { + const res = await module.client.scan(cursor, { MATCH: params.match, COUNT: 10000 }); + cursor = res.cursor; + const values = res.keys.filter((value) => { const isSeen = !!seen[value]; if (!isSeen) { seen[value] = 1; @@ -67,7 +67,7 @@ module.exports = function (module) { if (!keys || !Array.isArray(keys) || !keys.length) { return []; } - return await module.client.mget(keys); + return await module.client.mGet(keys); }; module.set = async function (key, value) { @@ -96,26 +96,26 @@ module.exports = function (module) { }; module.expire = async function (key, seconds) { - await module.client.expire(key, seconds); + await module.client.EXPIRE(key, seconds); }; module.expireAt = async function (key, timestamp) { - await module.client.expireat(key, timestamp); + await module.client.EXPIREAT(key, timestamp); }; module.pexpire = async function (key, ms) { - await module.client.pexpire(key, ms); + await module.client.PEXPIRE(key, ms); }; module.pexpireAt = async function (key, timestamp) { - await module.client.pexpireat(key, timestamp); + await module.client.PEXPIREAT(key, timestamp); }; module.ttl = async function (key) { - return await module.client.ttl(key); + return await module.client.TTL(key); }; module.pttl = async function (key) { - return await module.client.pttl(key); + return await module.client.PTTL(key); }; }; diff --git a/src/database/redis/sets.js b/src/database/redis/sets.js index b2b390598b..dd7e484325 100644 --- a/src/database/redis/sets.js +++ b/src/database/redis/sets.js @@ -10,7 +10,7 @@ module.exports = function (module) { if (!value.length) { return; } - await module.client.sadd(key, value); + await module.client.sAdd(key, value.map(String)); }; module.setsAdd = async function (keys, value) { @@ -18,7 +18,7 @@ module.exports = function (module) { return; } const batch = module.client.batch(); - keys.forEach(k => batch.sadd(String(k), String(value))); + keys.forEach(k => batch.sAdd(String(k), String(value))); await helpers.execBatch(batch); }; @@ -34,57 +34,57 @@ module.exports = function (module) { } const batch = module.client.batch(); - key.forEach(k => batch.srem(String(k), value)); + key.forEach(k => batch.sRem(String(k), value.map(String))); await helpers.execBatch(batch); }; module.setsRemove = async function (keys, value) { const batch = module.client.batch(); - keys.forEach(k => batch.srem(String(k), value)); + keys.forEach(k => batch.sRem(String(k), String(value))); await helpers.execBatch(batch); }; module.isSetMember = async function (key, value) { - const result = await module.client.sismember(key, value); + const result = await module.client.sIsMember(key, String(value)); return result === 1; }; module.isSetMembers = async function (key, values) { const batch = module.client.batch(); - values.forEach(v => batch.sismember(String(key), String(v))); + values.forEach(v => batch.sIsMember(String(key), String(v))); const results = await helpers.execBatch(batch); return results ? helpers.resultsToBool(results) : null; }; module.isMemberOfSets = async function (sets, value) { const batch = module.client.batch(); - sets.forEach(s => batch.sismember(String(s), String(value))); + sets.forEach(s => batch.sIsMember(String(s), String(value))); const results = await helpers.execBatch(batch); return results ? helpers.resultsToBool(results) : null; }; module.getSetMembers = async function (key) { - return await module.client.smembers(key); + return await module.client.sMembers(key); }; module.getSetsMembers = async function (keys) { const batch = module.client.batch(); - keys.forEach(k => batch.smembers(String(k))); + keys.forEach(k => batch.sMembers(String(k))); return await helpers.execBatch(batch); }; module.setCount = async function (key) { - return await module.client.scard(key); + return await module.client.sCard(key); }; module.setsCount = async function (keys) { const batch = module.client.batch(); - keys.forEach(k => batch.scard(String(k))); + keys.forEach(k => batch.sCard(String(k))); return await helpers.execBatch(batch); }; module.setRemoveRandom = async function (key) { - return await module.client.spop(key); + return await module.client.sPop(key); }; return module; diff --git a/src/database/redis/sorted.js b/src/database/redis/sorted.js index 013477da5a..d8613a3a68 100644 --- a/src/database/redis/sorted.js +++ b/src/database/redis/sorted.js @@ -11,34 +11,74 @@ module.exports = function (module) { require('./sorted/intersect')(module); module.getSortedSetRange = async function (key, start, stop) { - return await sortedSetRange('zrange', key, start, stop, '-inf', '+inf', false); + return await sortedSetRange(key, start, stop, '-inf', '+inf', false, false, false); }; module.getSortedSetRevRange = async function (key, start, stop) { - return await sortedSetRange('zrevrange', key, start, stop, '-inf', '+inf', false); + return await sortedSetRange(key, start, stop, '-inf', '+inf', false, true, false); }; module.getSortedSetRangeWithScores = async function (key, start, stop) { - return await sortedSetRange('zrange', key, start, stop, '-inf', '+inf', true); + return await sortedSetRange(key, start, stop, '-inf', '+inf', true, false, false); }; module.getSortedSetRevRangeWithScores = async function (key, start, stop) { - return await sortedSetRange('zrevrange', key, start, stop, '-inf', '+inf', true); + return await sortedSetRange(key, start, stop, '-inf', '+inf', true, true, false); }; - async function sortedSetRange(method, key, start, stop, min, max, withScores) { + module.getSortedSetRangeByScore = async function (key, start, count, min, max) { + return await sortedSetRangeByScore(key, start, count, min, max, false, false); + }; + + module.getSortedSetRevRangeByScore = async function (key, start, count, max, min) { + return await sortedSetRangeByScore(key, start, count, max, min, false, true); + }; + + module.getSortedSetRangeByScoreWithScores = async function (key, start, count, min, max) { + return await sortedSetRangeByScore(key, start, count, min, max, true, false); + }; + + module.getSortedSetRevRangeByScoreWithScores = async function (key, start, count, max, min) { + return await sortedSetRangeByScore(key, start, count, max, min, true, true); + }; + + async function sortedSetRangeByScore(key, start, count, min, max, withScores, rev) { + if (parseInt(count, 10) === 0) { + return []; + } + const stop = (parseInt(count, 10) === -1) ? -1 : (start + count - 1); + return await sortedSetRange(key, start, stop, min, max, withScores, rev, true); + } + + async function sortedSetRange(key, start, stop, min, max, withScores, rev, byScore) { + const opts = {}; + const cmd = withScores ? 'zRangeWithScores' : 'zRange'; + if (byScore) { + opts.BY = 'SCORE'; + opts.LIMIT = { offset: start, count: stop !== -1 ? stop + 1 : stop }; + } + if (rev) { + opts.REV = true; + } + if (Array.isArray(key)) { if (!key.length) { return []; } const batch = module.client.batch(); - key.forEach(key => batch[method](genParams(method, key, 0, stop, min, max, true))); + + if (byScore) { + key.forEach(key => batch.zRangeWithScores(key, min, max, { + ...opts, + LIMIT: { offset: 0, count: stop !== -1 ? stop + 1 : stop }, + })); + } else { + key.forEach(key => batch.zRangeWithScores(key, 0, stop, { ...opts })); + } + const data = await helpers.execBatch(batch); - - const batchData = data.map(setData => helpers.zsetToObjectArray(setData)); - - let objects = dbHelpers.mergeBatch(batchData, 0, stop, method === 'zrange' ? 1 : -1); - + const batchData = data; + let objects = dbHelpers.mergeBatch(batchData, 0, stop, rev ? -1 : 1); if (start > 0) { objects = objects.slice(start, stop !== -1 ? stop + 1 : undefined); } @@ -48,63 +88,25 @@ module.exports = function (module) { return objects; } - const params = genParams(method, key, start, stop, min, max, withScores); - const data = await module.client[method](params); + let data; + if (byScore) { + data = await module.client[cmd](key, min, max, opts); + } else { + data = await module.client[cmd](key, start, stop, opts); + } + if (!withScores) { return data; } - const objects = helpers.zsetToObjectArray(data); - return objects; - } - - function genParams(method, key, start, stop, min, max, withScores) { - const params = { - zrevrange: [key, start, stop], - zrange: [key, start, stop], - zrangebyscore: [key, min, max], - zrevrangebyscore: [key, max, min], - }; - if (withScores) { - params[method].push('WITHSCORES'); - } - - if (method === 'zrangebyscore' || method === 'zrevrangebyscore') { - const count = stop !== -1 ? stop - start + 1 : stop; - params[method].push('LIMIT', start, count); - } - return params[method]; - } - - module.getSortedSetRangeByScore = async function (key, start, count, min, max) { - return await sortedSetRangeByScore('zrangebyscore', key, start, count, min, max, false); - }; - - module.getSortedSetRevRangeByScore = async function (key, start, count, max, min) { - return await sortedSetRangeByScore('zrevrangebyscore', key, start, count, min, max, false); - }; - - module.getSortedSetRangeByScoreWithScores = async function (key, start, count, min, max) { - return await sortedSetRangeByScore('zrangebyscore', key, start, count, min, max, true); - }; - - module.getSortedSetRevRangeByScoreWithScores = async function (key, start, count, max, min) { - return await sortedSetRangeByScore('zrevrangebyscore', key, start, count, min, max, true); - }; - - async function sortedSetRangeByScore(method, key, start, count, min, max, withScores) { - if (parseInt(count, 10) === 0) { - return []; - } - const stop = (parseInt(count, 10) === -1) ? -1 : (start + count - 1); - return await sortedSetRange(method, key, start, stop, min, max, withScores); + return data; } module.sortedSetCount = async function (key, min, max) { - return await module.client.zcount(key, min, max); + return await module.client.zCount(key, min, max); }; module.sortedSetCard = async function (key) { - return await module.client.zcard(key); + return await module.client.zCard(key); }; module.sortedSetsCard = async function (keys) { @@ -112,7 +114,7 @@ module.exports = function (module) { return []; } const batch = module.client.batch(); - keys.forEach(k => batch.zcard(String(k))); + keys.forEach(k => batch.zCard(String(k))); return await helpers.execBatch(batch); }; @@ -125,26 +127,26 @@ module.exports = function (module) { } const batch = module.client.batch(); if (min !== '-inf' || max !== '+inf') { - keys.forEach(k => batch.zcount(String(k), min, max)); + keys.forEach(k => batch.zCount(String(k), min, max)); } else { - keys.forEach(k => batch.zcard(String(k))); + keys.forEach(k => batch.zCard(String(k))); } const counts = await helpers.execBatch(batch); return counts.reduce((acc, val) => acc + val, 0); }; module.sortedSetRank = async function (key, value) { - return await module.client.zrank(key, value); + return await module.client.zRank(key, String(value)); }; module.sortedSetRevRank = async function (key, value) { - return await module.client.zrevrank(key, value); + return await module.client.zRevRank(key, String(value)); }; module.sortedSetsRanks = async function (keys, values) { const batch = module.client.batch(); for (let i = 0; i < values.length; i += 1) { - batch.zrank(keys[i], String(values[i])); + batch.zRank(keys[i], String(values[i])); } return await helpers.execBatch(batch); }; @@ -152,7 +154,7 @@ module.exports = function (module) { module.sortedSetsRevRanks = async function (keys, values) { const batch = module.client.batch(); for (let i = 0; i < values.length; i += 1) { - batch.zrevrank(keys[i], String(values[i])); + batch.zRevRank(keys[i], String(values[i])); } return await helpers.execBatch(batch); }; @@ -160,7 +162,7 @@ module.exports = function (module) { module.sortedSetRanks = async function (key, values) { const batch = module.client.batch(); for (let i = 0; i < values.length; i += 1) { - batch.zrank(key, String(values[i])); + batch.zRank(key, String(values[i])); } return await helpers.execBatch(batch); }; @@ -168,7 +170,7 @@ module.exports = function (module) { module.sortedSetRevRanks = async function (key, values) { const batch = module.client.batch(); for (let i = 0; i < values.length; i += 1) { - batch.zrevrank(key, String(values[i])); + batch.zRevRank(key, String(values[i])); } return await helpers.execBatch(batch); }; @@ -177,8 +179,7 @@ module.exports = function (module) { if (!key || value === undefined) { return null; } - - const score = await module.client.zscore(key, value); + const score = await module.client.zScore(key, String(value)); return score === null ? score : parseFloat(score); }; @@ -187,7 +188,7 @@ module.exports = function (module) { return []; } const batch = module.client.batch(); - keys.forEach(key => batch.zscore(String(key), String(value))); + keys.forEach(key => batch.zScore(String(key), String(value))); const scores = await helpers.execBatch(batch); return scores.map(d => (d === null ? d : parseFloat(d))); }; @@ -197,7 +198,7 @@ module.exports = function (module) { return []; } const batch = module.client.batch(); - values.forEach(value => batch.zscore(String(key), String(value))); + values.forEach(value => batch.zScore(String(key), String(value))); const scores = await helpers.execBatch(batch); return scores.map(d => (d === null ? d : parseFloat(d))); }; @@ -211,9 +212,9 @@ module.exports = function (module) { if (!values.length) { return []; } - const batch = module.client.batch(); - values.forEach(v => batch.zscore(key, String(v))); - const results = await helpers.execBatch(batch); + const batch = module.client.multi(); + values.forEach(v => batch.zScore(key, String(v))); + const results = await batch.execAsPipeline(); return results.map(utils.isNumber); }; @@ -221,20 +222,18 @@ module.exports = function (module) { if (!Array.isArray(keys) || !keys.length) { return []; } - const batch = module.client.batch(); - keys.forEach(k => batch.zscore(k, String(value))); - const results = await helpers.execBatch(batch); + const batch = module.client.multi(); + keys.forEach(k => batch.zScore(k, String(value))); + const results = await batch.execAsPipeline(); return results.map(utils.isNumber); }; module.getSortedSetMembers = async function (key) { - return await module.client.zrange(key, 0, -1); + return await module.client.zRange(key, 0, -1); }; module.getSortedSetMembersWithScores = async function (key) { - return helpers.zsetToObjectArray( - await module.client.zrange(key, 0, -1, 'WITHSCORES') - ); + return await module.client.zRangeWithScores(key, 0, -1); }; module.getSortedSetsMembers = async function (keys) { @@ -242,7 +241,7 @@ module.exports = function (module) { return []; } const batch = module.client.batch(); - keys.forEach(k => batch.zrange(k, 0, -1)); + keys.forEach(k => batch.zRange(k, 0, -1)); return await helpers.execBatch(batch); }; @@ -251,65 +250,52 @@ module.exports = function (module) { return []; } const batch = module.client.batch(); - keys.forEach(k => batch.zrange(k, 0, -1, 'WITHSCORES')); + keys.forEach(k => batch.zRangeWithScores(k, 0, -1)); const res = await helpers.execBatch(batch); - return res.map(helpers.zsetToObjectArray); + return res; }; module.sortedSetIncrBy = async function (key, increment, value) { - const newValue = await module.client.zincrby(key, increment, value); + const newValue = await module.client.zIncrBy(key, increment, String(value)); return parseFloat(newValue); }; module.sortedSetIncrByBulk = async function (data) { const multi = module.client.multi(); data.forEach((item) => { - multi.zincrby(item[0], item[1], item[2]); + multi.zIncrBy(item[0], item[1], String(item[2])); }); const result = await multi.exec(); - return result.map(item => item && parseFloat(item[1])); + return result; }; - module.getSortedSetRangeByLex = async function (key, min, max, start, count) { - return await sortedSetLex('zrangebylex', false, key, min, max, start, count); + module.getSortedSetRangeByLex = async function (key, min, max, start = 0, count = -1) { + const { lmin, lmax } = helpers.normalizeLexRange(min, max, false); + return await module.client.zRange(key, lmin, lmax, { + BY: 'LEX', + LIMIT: { offset: start, count: count }, + }); }; - module.getSortedSetRevRangeByLex = async function (key, max, min, start, count) { - return await sortedSetLex('zrevrangebylex', true, key, max, min, start, count); + module.getSortedSetRevRangeByLex = async function (key, max, min, start = 0, count = -1) { + const { lmin, lmax } = helpers.normalizeLexRange(max, min, true); + return await module.client.zRange(key, lmin, lmax, { + REV: true, + BY: 'LEX', + LIMIT: { offset: start, count: count }, + }); }; module.sortedSetRemoveRangeByLex = async function (key, min, max) { - await sortedSetLex('zremrangebylex', false, key, min, max); + const { lmin, lmax } = helpers.normalizeLexRange(min, max, false); + await module.client.zRemRangeByLex(key, lmin, lmax); }; module.sortedSetLexCount = async function (key, min, max) { - return await sortedSetLex('zlexcount', false, key, min, max); + const { lmin, lmax } = helpers.normalizeLexRange(min, max, false); + return await module.client.zLexCount(key, lmin, lmax); }; - async function sortedSetLex(method, reverse, key, min, max, start, count) { - let minmin; - let maxmax; - if (reverse) { - minmin = '+'; - maxmax = '-'; - } else { - minmin = '-'; - maxmax = '+'; - } - - if (min !== minmin && !min.match(/^[[(]/)) { - min = `[${min}`; - } - if (max !== maxmax && !max.match(/^[[(]/)) { - max = `[${max}`; - } - const args = [key, min, max]; - if (count) { - args.push('LIMIT', start, count); - } - return await module.client[method](args); - } - module.getSortedSetScan = async function (params) { let cursor = '0'; @@ -318,20 +304,19 @@ module.exports = function (module) { const seen = Object.create(null); do { /* eslint-disable no-await-in-loop */ - const res = await module.client.zscan(params.key, cursor, 'MATCH', params.match, 'COUNT', 5000); - cursor = res[0]; + const res = await module.client.zScan(params.key, cursor, { MATCH: params.match, COUNT: 5000 }); + cursor = res.cursor; done = cursor === '0'; - const data = res[1]; - for (let i = 0; i < data.length; i += 2) { - const value = data[i]; - if (!seen[value]) { - seen[value] = 1; + for (let i = 0; i < res.members.length; i ++) { + const item = res.members[i]; + if (!seen[item.value]) { + seen[item.value] = 1; if (params.withScores) { - returnData.push({ value: value, score: parseFloat(data[i + 1]) }); + returnData.push({ value: item.value, score: parseFloat(item.score) }); } else { - returnData.push(value); + returnData.push(item.value); } if (params.limit && returnData.length >= params.limit) { done = true; diff --git a/src/database/redis/sorted/add.js b/src/database/redis/sorted/add.js index 660618b8a4..264c877845 100644 --- a/src/database/redis/sorted/add.js +++ b/src/database/redis/sorted/add.js @@ -1,7 +1,6 @@ 'use strict'; module.exports = function (module) { - const helpers = require('../helpers'); const utils = require('../../../utils'); module.sortedSetAdd = async function (key, score, value) { @@ -14,7 +13,8 @@ module.exports = function (module) { if (!utils.isNumber(score)) { throw new Error(`[[error:invalid-score, ${score}]]`); } - await module.client.zadd(key, score, String(value)); + + await module.client.zAdd(key, { score, value: String(value) }); }; async function sortedSetAddMulti(key, scores, values) { @@ -30,11 +30,8 @@ module.exports = function (module) { throw new Error(`[[error:invalid-score, ${scores[i]}]]`); } } - const args = [key]; - for (let i = 0; i < scores.length; i += 1) { - args.push(scores[i], String(values[i])); - } - await module.client.zadd(args); + const members = scores.map((score, i) => ({ score, value: String(values[i])})); + await module.client.zAdd(key, members); } module.sortedSetsAdd = async function (keys, scores, value) { @@ -51,13 +48,16 @@ module.exports = function (module) { throw new Error('[[error:invalid-data]]'); } - const batch = module.client.batch(); + const batch = module.client.multi(); for (let i = 0; i < keys.length; i += 1) { if (keys[i]) { - batch.zadd(keys[i], isArrayOfScores ? scores[i] : scores, String(value)); + batch.zAdd(keys[i], { + score: isArrayOfScores ? scores[i] : scores, + value: String(value), + }); } } - await helpers.execBatch(batch); + await batch.execAsPipeline(); }; module.sortedSetAddBulk = async function (data) { @@ -69,8 +69,8 @@ module.exports = function (module) { if (!utils.isNumber(item[1])) { throw new Error(`[[error:invalid-score, ${item[1]}]]`); } - batch.zadd(item[0], item[1], item[2]); + batch.zAdd(item[0], { score: item[1], value: String(item[2]) }); }); - await helpers.execBatch(batch); + await batch.execAsPipeline(); }; }; diff --git a/src/database/redis/sorted/intersect.js b/src/database/redis/sorted/intersect.js index 2b2ed1fe90..983c11abc4 100644 --- a/src/database/redis/sorted/intersect.js +++ b/src/database/redis/sorted/intersect.js @@ -8,52 +8,46 @@ module.exports = function (module) { return 0; } const tempSetName = `temp_${Date.now()}`; - - const interParams = [tempSetName, keys.length].concat(keys); - const multi = module.client.multi(); - multi.zinterstore(interParams); - multi.zcard(tempSetName); + multi.zInterStore(tempSetName, keys); + multi.zCard(tempSetName); multi.del(tempSetName); const results = await helpers.execBatch(multi); return results[1] || 0; }; module.getSortedSetIntersect = async function (params) { - params.method = 'zrange'; + params.reverse = false; return await getSortedSetRevIntersect(params); }; module.getSortedSetRevIntersect = async function (params) { - params.method = 'zrevrange'; + params.reverse = true; return await getSortedSetRevIntersect(params); }; async function getSortedSetRevIntersect(params) { - const { sets } = params; + let { sets } = params; const start = params.hasOwnProperty('start') ? params.start : 0; const stop = params.hasOwnProperty('stop') ? params.stop : -1; const weights = params.weights || []; const tempSetName = `temp_${Date.now()}`; - let interParams = [tempSetName, sets.length].concat(sets); + const interParams = {}; if (weights.length) { - interParams = interParams.concat(['WEIGHTS'].concat(weights)); + sets = sets.map((set, index) => ({ key: set, weight: weights[index] })); } if (params.aggregate) { - interParams = interParams.concat(['AGGREGATE', params.aggregate]); + interParams['AGGREGATE'] = params.aggregate.toUpperCase(); } - const rangeParams = [tempSetName, start, stop]; - if (params.withScores) { - rangeParams.push('WITHSCORES'); - } + const rangeCmd = params.withScores ? 'zRangeWithScores' : 'zRange'; const multi = module.client.multi(); - multi.zinterstore(interParams); - multi[params.method](rangeParams); + multi.zInterStore(tempSetName, sets, interParams); + multi[rangeCmd](tempSetName, start, stop, { REV: params.reverse}); multi.del(tempSetName); let results = await helpers.execBatch(multi); @@ -61,6 +55,6 @@ module.exports = function (module) { return results ? results[1] : null; } results = results[1] || []; - return helpers.zsetToObjectArray(results); + return results; } }; diff --git a/src/database/redis/sorted/remove.js b/src/database/redis/sorted/remove.js index 0c2b0164b0..df4c980b11 100644 --- a/src/database/redis/sorted/remove.js +++ b/src/database/redis/sorted/remove.js @@ -18,10 +18,10 @@ module.exports = function (module) { if (Array.isArray(key)) { const batch = module.client.batch(); - key.forEach(k => batch.zrem(k, value)); + key.forEach(k => batch.zRem(k, value.map(String))); await helpers.execBatch(batch); } else { - await module.client.zrem(key, value); + await module.client.zRem(key, value.map(String)); } }; @@ -31,7 +31,7 @@ module.exports = function (module) { module.sortedSetsRemoveRangeByScore = async function (keys, min, max) { const batch = module.client.batch(); - keys.forEach(k => batch.zremrangebyscore(k, min, max)); + keys.forEach(k => batch.zRemRangeByScore(k, min, max)); await helpers.execBatch(batch); }; @@ -40,7 +40,7 @@ module.exports = function (module) { return; } const batch = module.client.batch(); - data.forEach(item => batch.zrem(item[0], item[1])); + data.forEach(item => batch.zRem(item[0], String(item[1]))); await helpers.execBatch(batch); }; }; diff --git a/src/database/redis/sorted/union.js b/src/database/redis/sorted/union.js index acd57c2db0..329c5f125f 100644 --- a/src/database/redis/sorted/union.js +++ b/src/database/redis/sorted/union.js @@ -4,25 +4,20 @@ module.exports = function (module) { const helpers = require('../helpers'); module.sortedSetUnionCard = async function (keys) { - const tempSetName = `temp_${Date.now()}`; if (!keys.length) { return 0; } - const multi = module.client.multi(); - multi.zunionstore([tempSetName, keys.length].concat(keys)); - multi.zcard(tempSetName); - multi.del(tempSetName); - const results = await helpers.execBatch(multi); - return Array.isArray(results) && results.length ? results[1] : 0; + const results = await module.client.zUnion(keys); + return results ? results.length : 0; }; module.getSortedSetUnion = async function (params) { - params.method = 'zrange'; + params.reverse = false; return await module.sortedSetUnion(params); }; module.getSortedSetRevUnion = async function (params) { - params.method = 'zrevrange'; + params.reverse = true; return await module.sortedSetUnion(params); }; @@ -32,21 +27,16 @@ module.exports = function (module) { } const tempSetName = `temp_${Date.now()}`; - - const rangeParams = [tempSetName, params.start, params.stop]; - if (params.withScores) { - rangeParams.push('WITHSCORES'); - } - + const rangeCmd = params.withScores ? 'zRangeWithScores' : 'zRange'; const multi = module.client.multi(); - multi.zunionstore([tempSetName, params.sets.length].concat(params.sets)); - multi[params.method](rangeParams); + multi.zUnionStore(tempSetName, params.sets); + multi[rangeCmd](tempSetName, params.start, params.stop, { REV: params.reverse }); multi.del(tempSetName); let results = await helpers.execBatch(multi); if (!params.withScores) { return results ? results[1] : null; } results = results[1] || []; - return helpers.zsetToObjectArray(results); + return results; }; }; diff --git a/src/topics/posts.js b/src/topics/posts.js index e32c18e727..8201bcad02 100644 --- a/src/topics/posts.js +++ b/src/topics/posts.js @@ -442,7 +442,7 @@ module.exports = function (Topics) { let { content } = postData; // ignore lines that start with `>` - content = content.split('\n').filter(line => !line.trim().startsWith('>')).join('\n'); + content = (content || '').split('\n').filter(line => !line.trim().startsWith('>')).join('\n'); // Scan post content for topic links const matches = [...content.matchAll(backlinkRegex)]; if (!matches) { diff --git a/test/activitypub/actors.js b/test/activitypub/actors.js index 37fc74280d..ac2d11e5da 100644 --- a/test/activitypub/actors.js +++ b/test/activitypub/actors.js @@ -449,7 +449,8 @@ describe('Inbox resolution', () => { await activitypub.actors.assert(id); const inboxes = await activitypub.resolveInboxes([id]); - + console.log('inboxes', inboxes); + console.log('actor', actor); assert(inboxes && Array.isArray(inboxes)); assert.strictEqual(inboxes.length, 1); assert.strictEqual(inboxes[0], actor.inbox); diff --git a/test/activitypub/notes.js b/test/activitypub/notes.js index fbbf0c59ec..57ffce4259 100644 --- a/test/activitypub/notes.js +++ b/test/activitypub/notes.js @@ -650,7 +650,7 @@ describe('Notes', () => { it('should upvote an asserted remote post', async () => { const { id } = helpers.mocks.note(); - await activitypub.notes.assert(0, [id], { skipChecks: true }); + await activitypub.notes.assert(0, id, { skipChecks: true }); const { activity: like } = helpers.mocks.like({ object: id, }); @@ -672,7 +672,7 @@ describe('Notes', () => { it('should update a note\'s content', async () => { const { id: actor } = helpers.mocks.person(); const { id, note } = helpers.mocks.note({ attributedTo: actor }); - await activitypub.notes.assert(0, [id], { skipChecks: true }); + await activitypub.notes.assert(0, id, { skipChecks: true }); note.content = utils.generateUUID(); const { activity: update } = helpers.mocks.update({ object: note }); const { activity } = helpers.mocks.announce({ object: update }); diff --git a/test/database/sorted.js b/test/database/sorted.js index b98d969730..a375b2ec48 100644 --- a/test/database/sorted.js +++ b/test/database/sorted.js @@ -501,7 +501,9 @@ NUMERIC)-- WsPn&query[cid]=-1&parentCid=0&selectedCids[]=-1&privilege=topics:rea ['byScoreWithScoresKeys1', 1, 'value1'], ['byScoreWithScoresKeys2', 2, 'value2'], ]); - const data = await db.getSortedSetRevRangeByScoreWithScores(['byScoreWithScoresKeys1', 'byScoreWithScoresKeys2'], 0, -1, 5, -5); + const data = await db.getSortedSetRevRangeByScoreWithScores([ + 'byScoreWithScoresKeys1', 'byScoreWithScoresKeys2', + ], 0, -1, 5, -5); assert.deepStrictEqual(data, [{ value: 'value2', score: 2 }, { value: 'value1', score: 1 }]); }); }); @@ -1144,23 +1146,17 @@ NUMERIC)-- WsPn&query[cid]=-1&parentCid=0&selectedCids[]=-1&privilege=topics:rea assert.strictEqual(await db.exists('sorted3'), false); }); - it('should remove multiple values from multiple keys', (done) => { - db.sortedSetAdd('multiTest1', [1, 2, 3, 4], ['one', 'two', 'three', 'four'], (err) => { - assert.ifError(err); - db.sortedSetAdd('multiTest2', [3, 4, 5, 6], ['three', 'four', 'five', 'six'], (err) => { - assert.ifError(err); - db.sortedSetRemove(['multiTest1', 'multiTest2'], ['two', 'three', 'four', 'five', 'doesnt exist'], (err) => { - assert.ifError(err); - db.getSortedSetsMembers(['multiTest1', 'multiTest2'], (err, members) => { - assert.ifError(err); - assert.equal(members[0].length, 1); - assert.equal(members[1].length, 1); - assert.deepEqual(members, [['one'], ['six']]); - done(); - }); - }); - }); - }); + it('should remove multiple values from multiple keys', async () => { + await db.sortedSetAdd('multiTest1', [1, 2, 3, 4], ['one', 'two', 'three', 'four']); + await db.sortedSetAdd('multiTest2', [3, 4, 5, 6], ['three', 'four', 'five', 'six']); + + await db.sortedSetRemove(['multiTest1', 'multiTest2'], ['two', 'three', 'four', 'five', 'doesnt exist']); + + const members = await db.getSortedSetsMembers(['multiTest1', 'multiTest2']); + + assert.equal(members[0].length, 1); + assert.equal(members[1].length, 1); + assert.deepEqual(members, [['one'], ['six']]); }); it('should remove value from multiple keys', async () => { @@ -1171,24 +1167,15 @@ NUMERIC)-- WsPn&query[cid]=-1&parentCid=0&selectedCids[]=-1&privilege=topics:rea assert.deepStrictEqual(await db.getSortedSetRange('multiTest4', 0, -1), ['four', 'five', 'six']); }); - it('should remove multiple values from multiple keys', (done) => { - db.sortedSetAdd('multiTest5', [1], ['one'], (err) => { - assert.ifError(err); - db.sortedSetAdd('multiTest6', [2], ['two'], (err) => { - assert.ifError(err); - db.sortedSetAdd('multiTest7', [3], [333], (err) => { - assert.ifError(err); - db.sortedSetRemove(['multiTest5', 'multiTest6', 'multiTest7'], ['one', 'two', 333], (err) => { - assert.ifError(err); - db.getSortedSetsMembers(['multiTest5', 'multiTest6', 'multiTest7'], (err, members) => { - assert.ifError(err); - assert.deepEqual(members, [[], [], []]); - done(); - }); - }); - }); - }); - }); + it('should remove multiple values from multiple keys', async () => { + await db.sortedSetAdd('multiTest5', [1], ['one']); + await db.sortedSetAdd('multiTest6', [2], ['two']); + await db.sortedSetAdd('multiTest7', [3], [333]); + + await db.sortedSetRemove(['multiTest5', 'multiTest6', 'multiTest7'], ['one', 'two', 333]); + + const members = await db.getSortedSetsMembers(['multiTest5', 'multiTest6', 'multiTest7']); + assert.deepEqual(members, [[], [], []]); }); it('should not remove anything if values is empty array', (done) => { @@ -1379,7 +1366,10 @@ NUMERIC)-- WsPn&query[cid]=-1&parentCid=0&selectedCids[]=-1&privilege=topics:rea weights: [1, 0.5], }, (err, data) => { assert.ifError(err); - assert.deepEqual([{ value: 'value2', score: 4 }, { value: 'value3', score: 5.5 }], data); + assert.deepEqual([ + { value: 'value2', score: 4 }, + { value: 'value3', score: 5.5 }, + ], data); done(); }); }); diff --git a/test/mocks/databasemock.js b/test/mocks/databasemock.js index 6a568109b7..9416962b07 100644 --- a/test/mocks/databasemock.js +++ b/test/mocks/databasemock.js @@ -171,7 +171,6 @@ before(async function () { require('../../src/user').startJobs(); await webserver.listen(); - // Iterate over all of the test suites/contexts this.test.parent.suites.forEach((suite) => { // Attach an afterAll listener that resets the defaults